add a readme, remove junk

master
Ondřej Hruška 4 years ago
parent 0ecfdbb7f9
commit 6435a0a372
Signed by: MightyPork
GPG Key ID: 2C5FD5035250423D
  1. 1
      .gitignore
  2. 72
      Cargo.lock
  3. 4
      Cargo.toml
  4. 79
      README.md
  5. BIN
      postit.db
  6. 2
      src/config.rs
  7. 219
      src/main.rs
  8. 43
      src/well_known_mime.rs

1
.gitignore vendored

@ -1,3 +1,4 @@
/target
.idea/
postit.json
postit.db

72
Cargo.lock generated

@ -576,52 +576,6 @@ dependencies = [
"memchr 1.0.2",
]
[[package]]
name = "num"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg 1.0.0",
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
"autocfg 1.0.0",
"num-traits",
]
[[package]]
name = "num-derive"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "num-integer"
version = "0.1.42"
@ -632,29 +586,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00"
dependencies = [
"autocfg 1.0.0",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg 1.0.0",
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.11"
@ -811,9 +742,8 @@ dependencies = [
"chrono",
"clappconfig",
"flate2",
"lazy_static",
"log 0.4.8",
"num",
"num-derive",
"parking_lot",
"rand 0.7.3",
"rouille",

@ -3,6 +3,7 @@ name = "postit"
version = "0.1.0"
authors = ["Ondřej Hruška <ondra@ondrovo.com>"]
edition = "2018"
publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -21,5 +22,4 @@ bincode = "1.2.1"
flate2 = "1.0.14"
anyhow = "1.0.28"
tree_magic = { version = "0.2.3", default_features = false, features = ["staticmime"] }
num = "0.2.1"
num-derive = "0.3.0"
lazy_static = "1.4.0"

@ -0,0 +1,79 @@
# PostIt file sharing server
PostIt is designed to work as a temporary public storage for text (and other)
files uploaded to it by software that need a publicly reachable page without
hosting its own server or even having a public IP.
The primary use case is to share diagnostic and contextual information
produced by Fediverse bots (think an interactive game where the game board
is rendered to an image or text file on demand). There are sure to be many
other uses I didn't think of.
The uploaded files have a lifetime of 10 minutes, which can be shortened or
extended up to 1 hour (or more, as configured).
## Uploading a file
To upload a file, send a POST request to the running PostIt server.
```none
$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i
HTTP/1.1 200 OK
X-Secret: 5273d775746e393b
X-Expire: 599
Content-Length: 16
421d082ef85827ea
```
Take note of the `X-Secret` header, you will need it to update or delete the file.
If you only want to share the file, this is all you need. Grab the file ID from the response body
and share it. The URL is `/<FILE_ID>`, e.g.
```none
$ curl -X GET 0.0.0.0:7745/421d082ef85827ea -i
HTTP/1.1 200 OK
Content-Type: text/plain; charset=utf8
X-Expire: 459
Content-Length: 688
File content here...
```
### Content type
The server attempts to auto-detect the file's `Content-Type`. The fallback is `text/plain`.
If you wish to set a custom type, use the `Content-Type` header when uploading the file, e.g.
```
$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i -H 'Content-Type: application/json'
```
### Expiration time
To customize the expiration time, use the header `X-Expire: <seconds>`, or a GET argument `?expire=<seconds>` e.g.
```
$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i -H 'X-Expire: 60'
```
## Updating a file
A file you uploaded can be deleted or modified using the **secret token** obtained in respose to its upload.
Send the token as the `X-Secret` header, or GET argument `?secret=....`
File is updated by sending a `PUT` request to the file's URL.
The `PUT` request can change file expiration (`X-Expire: <secs>` or GET arg `expire`),
update its `Content-Type`, or replace its content.
Note that sending `PUT` with empty body will *not* clear the file, in that case the file content is
not changed at all. This can be used to extend file's expiration without changing it in any other way
(by sending the `X-Expire` header).
## Deleting a file
The `DELETE` verb, unsurprisingly, deletes a file. As with `PUT`, the secret token is required.
.

Binary file not shown.

@ -40,7 +40,7 @@ pub(crate) struct Config {
pub(crate) compression: bool,
/// Persistence file
pub(crate) persist_file : String,
pub(crate) persist_file: String,
}
impl Default for Config {

@ -2,36 +2,38 @@
extern crate serde_derive;
#[macro_use]
extern crate log;
#[macro_use]
extern crate num_derive;
use crate::config::Config;
use crate::well_known_mime::Mime;
use chrono::{DateTime, Utc};
use clappconfig::{anyhow, AppConfig};
use parking_lot::Mutex;
use rand::rngs::OsRng;
use rand::Rng;
use rouille::{Request, Response, ResponseBody};
use std::borrow::Cow;
use std::collections::HashMap;
use std::fs::OpenOptions;
use std::hash::{Hash, Hasher};
use std::io::Read;
use std::time::Duration;
use chrono::{Utc, DateTime};
use std::fs::OpenOptions;
use std::fmt::Display;
use serde::export::Formatter;
use std::fmt;
use serde::{Serialize, Serializer, Deserialize, Deserializer};
use serde::de::{DeserializeOwned, Visitor};
use crate::well_known_mime::Mime;
use siphasher::sip::SipHasher;
mod config;
mod well_known_mime;
const HDR_EXPIRES : &str = "X-Expires";
const HDR_SECRET : &str = "X-Secret";
/// Header to set expiry (seconds)
const HDR_EXPIRY: &str = "X-Expire";
/// Header to pass secret token for update/delete
const HDR_SECRET: &str = "X-Secret";
/// GET param to pass secret token (as a substitute for header)
const GET_EXPIRY: &str = "expire";
/// GET param to pass secret token (as a substitute for header)
const GET_SECRET: &str = "secret";
const FAVICON : &[u8] = include_bytes!("favicon.ico");
const FAVICON: &[u8] = include_bytes!("favicon.ico");
/// Post ID (represented as a 16-digit hex string)
type PostId = u64;
@ -41,7 +43,7 @@ type Secret = u64;
type DataHash = u64;
/// Post stored in the repository
#[derive(Debug,Serialize,Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
struct Post {
/// Content-Type
mime: Mime,
@ -59,6 +61,17 @@ impl Post {
pub fn is_expired(&self) -> bool {
self.expires < Utc::now()
}
/// Get remaining lifetime
pub fn time_remains(&self) -> Duration {
let seconds_remains = self.expires.signed_duration_since(Utc::now())
.num_seconds();
if seconds_remains < 0 {
Duration::from_secs(0)
} else {
Duration::from_secs(seconds_remains as u64)
}
}
}
fn main() -> anyhow::Result<()> {
@ -72,6 +85,7 @@ fn main() -> anyhow::Result<()> {
error!("Load failed: {}", e);
}
}
store.gc_expired_posts();
store
});
@ -82,7 +96,7 @@ fn main() -> anyhow::Result<()> {
info!("{} {}", method, req.raw_url());
if req.url() == "/favicon.ico" {
return Response::from_data("image/vnd.microsoft.icon", FAVICON);
return decorate_response(Response::from_data("image/vnd.microsoft.icon", FAVICON));
}
store_w.gc_expired_posts_if_needed();
@ -104,14 +118,21 @@ fn main() -> anyhow::Result<()> {
warn!("Error resp: {}", resp.status_code);
}
resp
decorate_response(resp)
});
}
fn decorate_response(resp : Response) -> Response {
resp.without_header("Server")
.with_additional_header("Server", "postit.rs")
.with_additional_header("Access-Control-Allow-Origin", "*")
.with_additional_header("X-Version", env!("CARGO_PKG_VERSION"))
}
type PostsMap = HashMap<PostId, Post>;
type DataMap = HashMap<DataHash, (usize, Vec<u8>)>;
#[derive(Debug,Serialize,Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
struct Repository {
#[serde(skip)]
config: Config,
@ -175,7 +196,7 @@ impl Repository {
.read(true)
.open(&self.config.persist_file)?;
let result : Repository = if self.config.compression {
let result: Repository = if self.config.compression {
let flate = flate2::read::DeflateDecoder::new(file);
bincode::deserialize_from(flate)?
} else {
@ -193,7 +214,7 @@ impl Repository {
fn serve_delete(&mut self, req: &Request) -> Response {
let post_id = match self.request_to_post_id(req, true) {
Ok(Some(pid)) => pid,
Ok(None) => return error_with_text(400, "Post ID required."),
Ok(None) => return error_with_text(400, "File ID required."),
Err(resp) => return resp,
};
@ -207,13 +228,16 @@ impl Repository {
/// POST inserts a new record
/// PUT updates a record
fn serve_post_put(&mut self, req: &Request) -> Response {
let is_post = req.method() == "POST";
let is_put = req.method() == "PUT";
// Post ID is empty for POST, set for PUT
let post_id = match self.request_to_post_id(req, true) {
Ok(pid) => {
if req.method() == "PUT" && pid.is_none() {
if is_put && pid.is_none() {
warn!("PUT without ID!");
return error_with_text(400, "PUT requires a file ID!");
} else if req.method() == "POST" && pid.is_some() {
} else if is_post && pid.is_some() {
warn!("POST with ID!");
return error_with_text(400, "Use PUT to update a file!");
}
@ -232,11 +256,17 @@ impl Repository {
body.take(self.config.max_file_size as u64 + 1)
.read_to_end(&mut data)
.unwrap();
if data.len() > self.config.max_file_size {
if is_post && data.len() == 0 {
warn!("Empty body!");
return error_with_text(400, "Empty body!");
} else if data.len() > self.config.max_file_size {
warn!("Upload too large!");
return empty_error(413);
}
} else {
return error_with_text(400, "Empty body!");
// Should not be possible
panic!("Req data None!");
}
// Convert "application/x-www-form-urlencoded" to text/plain (CURL uses this)
@ -247,7 +277,13 @@ impl Repository {
Some(v) => Some(v),
};
let expiry = match req.header(HDR_EXPIRES) {
let expiry = req.get_param(GET_EXPIRY);
let mut expiry_s = expiry.as_ref().map(|s| s.as_str());
if expiry_s.is_none() {
expiry_s = req.header(HDR_EXPIRY);
}
let expiry = match expiry_s {
Some(text) => match text.parse() {
Ok(v) => {
let dur = Duration::from_secs(v);
@ -266,31 +302,38 @@ impl Repository {
Err(_) => {
return error_with_text(
400,
"Malformed \"X-Expires\", use relative time in seconds.",
"Malformed expiration, use relative time in seconds.",
);
}
},
None => None,
};
if let Some(id) = post_id {
let the_id;
let resp = if let Some(id) = post_id {
// UPDATE
self.update(id, data, mime, expiry);
the_id = id;
Response::text("Updated OK.")
} else {
// INSERT
let (id, token) = self.insert(data, mime, expiry.unwrap_or(self.config.default_expiry));
the_id = self.insert(data, mime, expiry.unwrap_or(self.config.default_expiry));
Response::text(format!("{:016x}", the_id))
};
Response::text(format!("{:016x}", id))
.with_additional_header("X-Secret", format!("{:016x}", token))
}
let post = self.posts.get(&the_id).unwrap();
resp
.with_additional_header(HDR_SECRET, format!("{:016x}", post.secret))
.with_additional_header(HDR_EXPIRY, post.time_remains().as_secs().to_string())
}
/// Serve a GET or HEAD request
fn serve_get_head(&mut self, req: &Request) -> Response {
let post_id = match self.request_to_post_id(req, false) {
Ok(Some(pid)) => pid,
Ok(None) => return error_with_text(400, "Post ID required."),
Ok(None) => return error_with_text(400, "File ID required."),
Err(resp) => return resp,
};
@ -305,12 +348,25 @@ impl Repository {
return error_with_text(500, "File data lost.");
}
let seconds_remains = post.expires.signed_duration_since(Utc::now())
.num_seconds();
Response {
status_code: 200,
headers: vec![(
"Content-Type".into(),
format!("{}; charset=utf8", post.mime).into(),
)],
headers: vec![
(
"Content-Type".into(),
format!("{}; charset=utf8", post.mime).into(),
),
(
"Cache-Control".into(),
format!("public, max-age={}", seconds_remains).into()
),
(
HDR_EXPIRY.into(),
seconds_remains.to_string().into()
)
],
data: if req.method() == "HEAD" {
ResponseBody::empty()
} else {
@ -361,30 +417,38 @@ impl Repository {
None => {
warn!("ID {} does not exist!", id);
return Err(error_with_text(404, "No file with this ID!"));
},
}
Some(post) => {
if post.is_expired() {
warn!("Access of expired file {}!", id);
return Err(error_with_text(404, "No file with this ID!"));
}
let secret: u64 = match req.header(HDR_SECRET).map(|v| u64::from_str_radix(v, 16)) {
Some(Ok(bytes)) => bytes,
None => {
warn!("Missing secret token!");
return Err(error_with_text(400, "Secret token required!"));
}
Some(Err(e)) => {
warn!("Token parse error: {:?}", e);
return Err(error_with_text(400, "Bad secret token format!"));
},
};
let secret = req.get_param(GET_SECRET);
let mut secret_str = secret.as_ref().map(|s| s.as_str());
if secret_str.is_none() {
secret_str = req.header(HDR_SECRET);
}
let secret: u64 =
match secret_str.map(|v| u64::from_str_radix(v, 16)) {
Some(Ok(bytes)) => bytes,
None => {
warn!("Missing secret token!");
return Err(error_with_text(400, "Secret token required!"));
}
Some(Err(e)) => {
warn!("Token parse error: {:?}", e);
return Err(error_with_text(400, "Bad secret token format!"));
}
};
if post.secret != secret {
warn!("Secret token mismatch");
return Err(error_with_text(401, "Invalid secret token!"));
}
},
}
}
}
@ -394,7 +458,12 @@ impl Repository {
/// Drop expired posts, if cleaning is due
fn gc_expired_posts_if_needed(&mut self) {
if Utc::now().signed_duration_since(self.last_gc_time).to_std().unwrap_or_default() > self.config.expired_gc_interval {
if Utc::now()
.signed_duration_since(self.last_gc_time)
.to_std()
.unwrap_or_default()
> self.config.expired_gc_interval
{
self.gc_expired_posts();
self.last_gc_time = Utc::now();
}
@ -425,7 +494,7 @@ impl Repository {
/// Get hash of a byte vector (for deduplication)
fn hash_data(data: &Vec<u8>) -> DataHash {
let mut hasher = siphasher::sip::SipHasher::new();
let mut hasher = SipHasher::new();
data.hash(&mut hasher);
hasher.finish()
}
@ -462,7 +531,7 @@ impl Repository {
}
/// Insert a post
fn insert(&mut self, data: Vec<u8>, mime: Option<&str>, expires: Duration) -> (PostId, Secret) {
fn insert(&mut self, data: Vec<u8>, mime: Option<&str>, expires: Duration) -> PostId {
info!(
"Insert post with data of len {} bytes, mime {}, expiry {:?}",
data.len(),
@ -473,12 +542,8 @@ impl Repository {
let hash = Self::hash_data(&data);
let mime = match mime {
None => {
Mime::from(tree_magic::from_u8(&data))
},
Some(explicit) => {
Mime::from(explicit)
},
None => Mime::from(tree_magic::from_u8(&data)),
Some(explicit) => Mime::from(explicit),
};
Self::store_data_or_increment_rc(&mut self.data, hash, data);
@ -492,7 +557,7 @@ impl Repository {
let secret = OsRng.gen();
debug!("Post ID = #{:016x}", post_id);
debug!("File ID = #{:016x} (http://{}:{}/{:016x})", post_id, self.config.host, self.config.port, post_id);
debug!("Data hash = #{:016x}, mime {}", hash, mime);
debug!("Secret = #{:016x}", secret);
@ -508,7 +573,7 @@ impl Repository {
self.dirty = true;
(post_id, secret)
post_id
}
/// Update a post by ID
@ -523,18 +588,21 @@ impl Repository {
.unwrap_or("unchanged".into())
);
let hash = Self::hash_data(&data);
let post = self.posts.get_mut(&id).unwrap(); // post existence was checked before
if hash != post.hash {
debug!("Data hash = #{:016x} (content changed)", hash);
if !data.is_empty() {
let hash = Self::hash_data(&data);
Self::drop_data_or_decrement_rc(&mut self.data, post.hash);
Self::store_data_or_increment_rc(&mut self.data, hash, data);
post.hash = hash;
self.dirty = true;
} else {
debug!("Data hash = #{:016x} (no change)", hash);
if hash != post.hash {
debug!("Data hash = #{:016x} (content changed)", hash);
Self::drop_data_or_decrement_rc(&mut self.data, post.hash);
Self::store_data_or_increment_rc(&mut self.data, hash, data);
post.hash = hash;
self.dirty = true;
} else {
debug!("Data hash = #{:016x} (no change)", hash);
}
}
if let Some(mime) = mime {
@ -566,22 +634,25 @@ impl Repository {
/// Serialize chrono unix timestamp as seconds
mod serde_chrono_datetime_as_unix {
use chrono::{DateTime, NaiveDateTime, Utc};
use serde::{self, Deserialize, Deserializer, Serializer};
use chrono::{DateTime, Utc, NaiveDateTime};
pub fn serialize<S>(value: &DateTime<Utc>, se: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
where
S: Serializer,
{
se.serialize_i64(value.naive_utc().timestamp())
}
pub fn deserialize<'de, D>(de: D) -> Result<DateTime<Utc>, D::Error>
where
D: Deserializer<'de>,
where
D: Deserializer<'de>,
{
let ts: i64 = i64::deserialize(de)?;
Ok(DateTime::from_utc(NaiveDateTime::from_timestamp(ts, 0), Utc))
Ok(DateTime::from_utc(
NaiveDateTime::from_timestamp(ts, 0),
Utc,
))
}
}

@ -1,18 +1,20 @@
use serde::{Serialize, Serializer, Deserialize, Deserializer};
use std::fmt::{Write, Display, Formatter};
use std::fmt;
use serde::de::Visitor;
use std::fmt::{Display, Formatter};
use lazy_static::lazy_static;
#[derive(Serialize,Deserialize,Debug,PartialEq,Eq,Hash)]
/// Mime type
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub enum Mime {
/// Well-known mime
WellKnown(usize),
/// Custom mime
Custom(String),
}
impl From<String> for Mime {
fn from(s: String) -> Self {
if let Ok(index) = WELL_KNOWN.binary_search(&s.as_str()) {
Mime::WellKnown(index)
if let Some(index) = MIME_LOOKUP.get(&s.as_str()) {
Mime::WellKnown(*index)
} else {
Mime::Custom(s)
}
@ -21,8 +23,8 @@ impl From<String> for Mime {
impl<'a> From<&'a str> for Mime {
fn from(s: &'a str) -> Self {
if let Ok(index) = WELL_KNOWN.binary_search(&s) {
Mime::WellKnown(index)
if let Some(index) = MIME_LOOKUP.get(&s) {
Mime::WellKnown(*index)
} else {
Mime::Custom(s.to_string())
}
@ -38,15 +40,16 @@ impl Display for Mime {
} else {
f.write_str("application/octet-stream")
}
},
Mime::Custom(s) => {
f.write_str(s)
},
}
Mime::Custom(s) => f.write_str(s),
}
}
}
// CAUTION!!!!! This list must be alphabetically sorted!
// The positions in this list must be kept stable - otherwise the persistence file may
// deserialize to the wrong type.
//
// If a new type needs to be added, add it at the end.
const WELL_KNOWN : &[&str] = &[
"application/andrew-inset",
"application/applixware",
@ -739,4 +742,18 @@ const WELL_KNOWN : &[&str] = &[
"video/x-msvideo",
"video/x-sgi-movie",
"x-conference/x-cooltalk",
// Extras
"application/toml",
"application/json5",
];
use std::collections::HashMap;
lazy_static!{
static ref MIME_LOOKUP : HashMap<&'static str, usize> = {
let mut map = HashMap::new();
for (n, entry) in WELL_KNOWN.iter().enumerate() {
map.insert(*entry, n);
}
map
};
}

Loading…
Cancel
Save