Lua runner with rich builtin stdlib
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
a/src/stdlib/http.rs

683 lines
27 KiB

use std::collections::HashMap;
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use mlua::prelude::LuaResult;
use mlua::{Lua, Table as LuaTable, Value as LuaValue};
use crate::stdlib::utils::lua_to_json;
const HTTP_LUA: &str = include_str!("../../lua/stdlib/http.lua");
// ---------------------------------------------------------------------------
// Cookie jar
// ---------------------------------------------------------------------------
/// A minimal in-memory cookie jar: bare domain (no leading dot) → name → value.
/// Deliberately simple — it covers the >95% case of `Set-Cookie` flows without
/// pulling in the `cookie_store` crate, and serializes cleanly to JSONL.
#[derive(Default)]
struct CookieJar {
cookies: HashMap<String, HashMap<String, String>>,
}
impl CookieJar {
/// All cookies (name, value) whose stored domain matches `host`: either an
/// exact match or `host` being a subdomain of the stored domain.
fn cookies_for(&self, host: &str) -> Vec<(String, String)> {
let mut out = Vec::new();
for (domain, names) in &self.cookies {
let suffix = format!(".{domain}");
if host == domain || host.ends_with(&suffix) {
for (name, value) in names {
out.push((name.clone(), value.clone()));
}
}
}
out
}
/// Parse a single `Set-Cookie` header value and store the cookie. Extracts
/// `name=value` (first segment) and an optional `Domain=` attribute, falling
/// back to the request host. Malformed headers are ignored.
fn set_from_header(&mut self, host: &str, header: &str) {
let mut segments = header.split(';');
let first = match segments.next() {
Some(s) => s.trim(),
None => return,
};
let (name, value) = match first.split_once('=') {
Some((n, v)) => (n.trim(), v.trim()),
None => return,
};
if name.is_empty() {
return;
}
let request_host = host.to_ascii_lowercase();
let mut domain = request_host.clone();
for seg in segments {
if let Some((k, v)) = seg.split_once('=')
&& k.trim().eq_ignore_ascii_case("domain")
{
let d = v.trim().trim_start_matches('.').to_ascii_lowercase();
// Only accept a Domain the request host actually belongs to
// (exact match or a subdomain). Otherwise a response could plant
// a cookie scoped to an unrelated domain ("cookie tossing"); such
// an attribute is ignored and the cookie stays host-scoped.
if !d.is_empty() && (request_host == d || request_host.ends_with(&format!(".{d}"))) {
domain = d;
}
}
}
self.cookies
.entry(domain)
.or_default()
.insert(name.to_string(), value.to_string());
}
/// Serialize as JSONL — one `{"domain","name","value"}` object per line.
fn to_jsonl(&self) -> String {
let mut out = String::new();
for (domain, names) in &self.cookies {
for (name, value) in names {
let obj = serde_json::json!({
"domain": domain,
"name": name,
"value": value,
});
out.push_str(&obj.to_string());
out.push('\n');
}
}
out
}
/// Merge cookies from JSONL produced by `to_jsonl`. Bad lines are skipped.
fn merge_jsonl(&mut self, src: &str) {
for line in src.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else {
continue;
};
let domain = v.get("domain").and_then(|x| x.as_str());
let name = v.get("name").and_then(|x| x.as_str());
let value = v.get("value").and_then(|x| x.as_str());
if let (Some(d), Some(n), Some(val)) = (domain, name, value) {
self.cookies
.entry(d.to_string())
.or_default()
.insert(n.to_string(), val.to_string());
}
}
}
}
// ---------------------------------------------------------------------------
// Request execution
// ---------------------------------------------------------------------------
/// `application/x-www-form-urlencoded` body from key/value pairs. Implemented
/// inline so the build needs no optional reqwest features.
fn form_urlencode(pairs: &[(String, String)]) -> String {
fn encode(s: &str) -> String {
let mut out = String::new();
for b in s.bytes() {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
out.push(b as char)
}
b' ' => out.push('+'),
_ => out.push_str(&format!("%{b:02X}")),
}
}
out
}
pairs
.iter()
.map(|(k, v)| format!("{}={}", encode(k), encode(v)))
.collect::<Vec<_>>()
.join("&")
}
const MAX_REDIRECTS: u32 = 10;
/// The origin of a URL: (scheme, host, effective port). Two URLs share an origin
/// when all three match — the boundary across which credentials and secret
/// headers must not be replayed on a redirect.
fn origin_of(u: &reqwest::Url) -> (String, Option<String>, Option<u16>) {
(
u.scheme().to_ascii_lowercase(),
u.host_str().map(|h| h.to_ascii_lowercase()),
u.port_or_known_default(),
)
}
/// Shared by the stateless `http.request` and a session's `:request`. When `jar`
/// is `Some`, the matching jar cookies are sent and any `Set-Cookie` responses
/// are stored back.
///
/// Redirects are followed manually (the client is built with
/// `redirect::Policy::none`) so that `Set-Cookie` headers on 30x responses —
/// the common login → redirect → dashboard pattern — are captured into the jar,
/// which reqwest's transparent redirect following would otherwise hide.
async fn execute_request(
lua: Lua,
client: reqwest::Client,
jar: Option<Arc<Mutex<CookieJar>>>,
method: String,
url: String,
opts: Option<LuaTable>,
) -> LuaResult<LuaTable> {
// Every request funnels through here (http.request, the shorthands, and
// all session methods) — the one choke point for --sandbox.
super::fs::ensure_net(&lua)?;
let mut method = reqwest::Method::from_bytes(method.to_ascii_uppercase().as_bytes())
.map_err(|e| mlua::Error::external(format!("http: invalid method '{method}': {e}")))?;
// Parse opts once into owned pieces so each redirect hop can rebuild the
// request (a reqwest RequestBuilder is single-use).
let mut timeout: Option<Duration> = None;
let mut custom_headers: Vec<(String, String)> = Vec::new();
let mut opts_cookies: Vec<(String, String)> = Vec::new();
// Body, with the Content-Type it implies (None for a raw body).
let mut body: Option<(Vec<u8>, Option<&'static str>)> = None;
// Auth: (is_digest, username, password).
let mut auth: Option<(bool, String, String)> = None;
if let Some(opts) = opts.as_ref() {
if let Some(t) = opts.get::<Option<f64>>("timeout")? {
timeout = Some(Duration::try_from_secs_f64(t).map_err(|_| {
mlua::Error::external(
"http: timeout must be a non-negative finite number of seconds",
)
})?);
}
if let Some(headers) = opts.get::<Option<LuaTable>>("headers")? {
for pair in headers.pairs::<String, String>() {
custom_headers.push(pair?);
}
}
if let Some(cookies) = opts.get::<Option<LuaTable>>("cookies")? {
for pair in cookies.pairs::<String, String>() {
opts_cookies.push(pair?);
}
}
// Body: json > form > raw body (first one present wins).
if let Some(json_val) = opts.get::<Option<LuaValue>>("json")? {
let json = lua_to_json(json_val, 0)?;
let s = serde_json::to_string(&json).map_err(mlua::Error::external)?;
body = Some((s.into_bytes(), Some("application/json")));
} else if let Some(form) = opts.get::<Option<LuaTable>>("form")? {
let mut pairs = Vec::new();
for pair in form.pairs::<String, String>() {
pairs.push(pair?);
}
body = Some((
form_urlencode(&pairs).into_bytes(),
Some("application/x-www-form-urlencoded"),
));
} else if let Some(b) = opts.get::<Option<mlua::String>>("body")? {
body = Some((b.as_bytes().to_vec(), None));
}
// Auth: { username, password, scheme = "basic" (default) | "digest" }.
if let Some(auth_tbl) = opts.get::<Option<LuaTable>>("auth")? {
let username = auth_tbl
.get::<Option<String>>("username")?
.ok_or_else(|| mlua::Error::external("http: auth.username is required"))?;
let password = auth_tbl
.get::<Option<String>>("password")?
.ok_or_else(|| mlua::Error::external("http: auth.password is required"))?;
let is_digest = match auth_tbl.get::<Option<String>>("scheme")?.as_deref() {
None | Some("basic") => false,
Some("digest") => true,
Some(other) => {
return Err(mlua::Error::external(format!(
"http: auth.scheme must be \"basic\" or \"digest\", got \"{other}\""
)));
}
};
auth = Some((is_digest, username, password));
}
}
// Did the caller supply their own Content-Type? If so it wins over the one a
// json/form body would otherwise imply (no silent override, no duplicate).
let has_user_content_type = custom_headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case("content-type"));
let is_digest = matches!(auth.as_ref(), Some((true, _, _)));
let mut digest_header: Option<String> = None;
let mut digest_tried = false;
let mut url = url;
let mut redirects_left = MAX_REDIRECTS;
// Origin (scheme, host, port) of the ORIGINAL request. Once a redirect
// leaves this origin we stop attaching credentials, custom headers, and
// per-request cookies, mirroring what reqwest's own redirect policy does:
// a redirect to an attacker-controlled host must not receive the caller's
// Authorization, Cookie, or secret headers.
let origin = reqwest::Url::parse(&url).ok().map(|u| origin_of(&u));
let resp = loop {
// Host used for cookie matching and as the Set-Cookie domain fallback;
// recomputed each hop since a redirect may cross hosts.
let parsed = reqwest::Url::parse(&url).ok();
let host = parsed
.as_ref()
.and_then(|u| u.host_str().map(|h| h.to_ascii_lowercase()));
// Are we still on the origin the request was addressed to?
let same_origin = match (origin.as_ref(), parsed.as_ref()) {
(Some(o), Some(u)) => *o == origin_of(u),
_ => false,
};
let mut req = client.request(method.clone(), &url);
if let Some(t) = timeout {
req = req.timeout(t);
}
// Custom headers carry only while on the original origin.
if same_origin {
for (k, v) in &custom_headers {
req = req.header(k, v);
}
}
// Cookie header: jar cookies for this host (the jar is domain-scoped, so
// this is always safe), then per-request cookies which override on name
// collision — but the explicit per-request cookies stay on-origin only.
let mut cookie_map: HashMap<String, String> = HashMap::new();
if let (Some(jar), Some(host)) = (jar.as_ref(), host.as_ref()) {
for (n, v) in jar.lock().unwrap_or_else(|e| e.into_inner()).cookies_for(host) {
cookie_map.insert(n, v);
}
}
if same_origin {
for (k, v) in &opts_cookies {
cookie_map.insert(k.clone(), v.clone());
}
}
if !cookie_map.is_empty() {
let header = cookie_map
.iter()
.map(|(k, v)| format!("{k}={v}"))
.collect::<Vec<_>>()
.join("; ");
req = req.header(reqwest::header::COOKIE, header);
}
if let Some((bytes, ct)) = body.as_ref() {
// Apply the body's implied Content-Type only when the caller didn't
// set their own (and theirs is still in effect, i.e. same origin).
if let Some(ct) = ct
&& !(same_origin && has_user_content_type)
{
req = req.header(reqwest::header::CONTENT_TYPE, *ct);
}
req = req.body(bytes.clone());
}
// Auth. Basic goes out on each hop; digest's Authorization is set only
// after the 401 challenge below has been answered (digest_header). Both
// are withheld once a redirect leaves the original origin, so credentials
// never reach a host the caller did not address.
if same_origin
&& let Some((digest, username, password)) = auth.as_ref()
{
if *digest {
if let Some(h) = digest_header.as_ref() {
req = req.header(reqwest::header::AUTHORIZATION, h);
}
} else {
req = req.basic_auth(username, Some(password));
}
}
let resp = req
.send()
.await
.map_err(|e| mlua::Error::external(format!("http: {e}")))?;
let status = resp.status();
// Store Set-Cookie from this hop into the jar.
if let (Some(jar), Some(host)) = (jar.as_ref(), host.as_ref()) {
let mut j = jar.lock().unwrap_or_else(|e| e.into_inner());
for value in resp.headers().get_all(reqwest::header::SET_COOKIE).iter() {
if let Ok(s) = value.to_str() {
j.set_from_header(host, s);
}
}
}
// Digest auth: answer a 401 challenge once, then retry the same request
// with the computed Authorization header.
if is_digest && !digest_tried && status == reqwest::StatusCode::UNAUTHORIZED {
let challenge = resp
.headers()
.get(reqwest::header::WWW_AUTHENTICATE)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
if let Some((_, username, password)) = auth.as_ref()
&& let Some(challenge) = challenge
&& let Ok(mut prompt) = digest_auth::parse(&challenge)
{
let uri = reqwest::Url::parse(&url)
.map(|u| match u.query() {
Some(q) => format!("{}?{}", u.path(), q),
None => u.path().to_string(),
})
.unwrap_or_else(|_| url.clone());
let ctx = digest_auth::AuthContext::new_with_method(
username.as_str(),
password.as_str(),
uri,
body.as_ref().map(|(b, _)| b.as_slice()),
digest_auth::HttpMethod::from(method.as_str()),
);
if let Ok(answer) = prompt.respond(&ctx) {
digest_header = Some(answer.to_header_string());
digest_tried = true;
continue;
}
}
}
// Follow a redirect if there is one to follow. A redirect with a usable
// Location that we can't follow because the budget is spent is a "too
// many redirects" error, not a silent success returning the 3xx.
if status.is_redirection()
&& let Some(next) = resp
.headers()
.get(reqwest::header::LOCATION)
.and_then(|l| l.to_str().ok())
.and_then(|loc| reqwest::Url::parse(&url).and_then(|base| base.join(loc)).ok())
{
if redirects_left == 0 {
return Err(mlua::Error::external(format!(
"http: too many redirects (max {MAX_REDIRECTS})"
)));
}
redirects_left -= 1;
// 303, and 301/302 on a POST, degrade to a bodyless GET — the
// behaviour browsers and reqwest's own redirect policy apply.
let code = status.as_u16();
if code == 303 || ((code == 301 || code == 302) && method == reqwest::Method::POST) {
method = reqwest::Method::GET;
body = None;
}
url = next.to_string();
continue;
}
break resp;
};
let status = resp.status().as_u16();
// Response headers: lowercase names, first value wins.
let headers_tbl = lua.create_table()?;
for (name, value) in resp.headers().iter() {
let lname = name.as_str().to_ascii_lowercase();
if !headers_tbl.contains_key(lname.as_str())? {
headers_tbl.raw_set(lname.as_str(), lua.create_string(value.as_bytes())?)?;
}
}
let bytes = resp
.bytes()
.await
.map_err(|e| mlua::Error::external(format!("http: {e}")))?;
let out = lua.create_table()?;
out.raw_set("status", status)?;
out.raw_set("ok", (200..=299).contains(&status))?;
out.raw_set("headers", headers_tbl)?;
out.raw_set("body", lua.create_string(&bytes)?)?;
Ok(out)
}
// ---------------------------------------------------------------------------
// Session
// ---------------------------------------------------------------------------
/// Build the plain Lua table that backs a session. State (the cookie jar) lives
/// in the closures; the Lua layer applies a metatable for the method shorthands.
/// Methods are called as `s:method(...)`, so each closure receives the session
/// table as a leading `_this` argument that it ignores.
fn make_session(
lua: &Lua,
client: reqwest::Client,
jar: Arc<Mutex<CookieJar>>,
) -> LuaResult<LuaTable> {
let tbl = lua.create_table()?;
tbl.raw_set(
"_request",
lua.create_async_function({
let client = client.clone();
let jar = jar.clone();
move |lua, (_this, method, url, opts): (LuaTable, String, String, Option<LuaTable>)| {
let client = client.clone();
let jar = jar.clone();
async move {
execute_request(lua, client, Some(jar), method, url, opts).await
}
}
})?,
)?;
// Cookie-jar files hold live credentials; both directions go through the
// fs permission system (async only because the permission cycle is).
tbl.raw_set("save", {
let jar = jar.clone();
lua.create_async_function(move |lua, (_this, path): (LuaTable, String)| {
let jar = jar.clone();
async move {
let (dir, target) =
super::fs::write_target("session:save", Path::new(&path)).await?;
super::fs::ensure_access(&lua, "session:save", dir, super::fs::Wants::WRITE)
.await?;
let jsonl = jar.lock().unwrap_or_else(|e| e.into_inner()).to_jsonl();
tokio::fs::write(&target, jsonl)
.await
.map_err(|e| mlua::Error::external(format!("session:save: {e}")))
}
})?
})?;
tbl.raw_set("load", {
let jar = jar.clone();
lua.create_async_function(move |lua, (_this, path): (LuaTable, String)| {
let jar = jar.clone();
async move {
let (dir, target) =
super::fs::read_target("session:load", Path::new(&path)).await?;
super::fs::ensure_access(&lua, "session:load", dir, super::fs::Wants::READ)
.await?;
let src = tokio::fs::read_to_string(&target)
.await
.map_err(|e| mlua::Error::external(format!("session:load: {e}")))?;
jar.lock().unwrap_or_else(|e| e.into_inner()).merge_jsonl(&src);
Ok(())
}
})?
})?;
tbl.raw_set("clearCookies", {
let jar = jar.clone();
lua.create_function(move |_, _this: LuaTable| {
jar.lock().unwrap_or_else(|e| e.into_inner()).cookies.clear();
Ok(())
})?
})?;
tbl.raw_set("cookies", {
let jar = jar.clone();
lua.create_function(move |lua, _this: LuaTable| {
let outer = lua.create_table()?;
let j = jar.lock().unwrap_or_else(|e| e.into_inner());
for (domain, names) in j.cookies.iter() {
let inner = lua.create_table()?;
for (name, value) in names.iter() {
inner.raw_set(name.as_str(), value.as_str())?;
}
outer.raw_set(domain.as_str(), inner)?;
}
Ok(outer)
})?
})?;
Ok(tbl)
}
// ---------------------------------------------------------------------------
// Installation
// ---------------------------------------------------------------------------
pub(super) fn install(lua: &Lua) -> LuaResult<()> {
// reqwest is built with `rustls-no-provider`, so it has no crypto provider
// of its own and panics ("No provider set") unless one is installed as the
// process default before the client is built. `ring` is self-contained
// (compiled in, no system OpenSSL). Idempotent across Lua states — only the
// first install in the process wins, and they would all install ring.
let _ = rustls::crypto::ring::default_provider().install_default();
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
// reqwest sends no User-Agent by default; some edges/CDNs reject
// empty-UA requests outright. A per-request `headers` entry overrides it.
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")))
// Redirects are followed manually in execute_request so Set-Cookie
// headers on 30x responses can be captured into the cookie jar.
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(mlua::Error::external)?;
let http = lua.create_table()?;
// Stateless http.request — no cookie jar.
http.raw_set(
"request",
lua.create_async_function({
let client = client.clone();
move |lua, (method, url, opts): (String, String, Option<LuaTable>)| {
let client = client.clone();
async move { execute_request(lua, client, None, method, url, opts).await }
}
})?,
)?;
// http.session(path?) — constructor returning the raw session table.
http.raw_set(
"session",
lua.create_async_function({
let client = client.clone();
move |lua, path: Option<String>| {
let client = client.clone();
async move {
let mut jar = CookieJar::default();
if let Some(p) = path {
let (dir, target) =
super::fs::read_target("http.session", Path::new(&p)).await?;
super::fs::ensure_access(
&lua,
"http.session",
dir,
super::fs::Wants::READ,
)
.await?;
let src = tokio::fs::read_to_string(&target)
.await
.map_err(|e| mlua::Error::external(format!("http.session: {e}")))?;
jar.merge_jsonl(&src);
}
make_session(&lua, client, Arc::new(Mutex::new(jar)))
}
}
})?,
)?;
lua.globals().raw_set("http", http)?;
// Lua side adds: resp.json(), method shorthands, getJSON/postJSON, and the
// session metatable wrapping http.session().
lua.load(HTTP_LUA).set_name("@[stdlib/http]").exec()
}
#[cfg(test)]
mod tests {
use super::*;
fn cookie(jar: &CookieJar, domain: &str, name: &str) -> Option<String> {
jar.cookies.get(domain).and_then(|m| m.get(name).cloned())
}
#[test]
fn set_cookie_defaults_to_request_host() {
let mut jar = CookieJar::default();
jar.set_from_header("example.com", "sid=abc");
assert_eq!(cookie(&jar, "example.com", "sid").as_deref(), Some("abc"));
}
#[test]
fn set_cookie_honors_domain_the_host_belongs_to() {
let mut jar = CookieJar::default();
// A subdomain may scope a cookie up to its parent domain.
jar.set_from_header("app.example.com", "sid=abc; Domain=example.com");
assert_eq!(cookie(&jar, "example.com", "sid").as_deref(), Some("abc"));
}
#[test]
fn set_cookie_rejects_foreign_domain() {
let mut jar = CookieJar::default();
// Cookie tossing: a response from one host must not plant a cookie scoped
// to an unrelated domain. The bad Domain= is ignored; it stays host-scoped.
jar.set_from_header("127.0.0.1", "evil=1; Domain=other.example");
assert!(cookie(&jar, "other.example", "evil").is_none());
assert_eq!(cookie(&jar, "127.0.0.1", "evil").as_deref(), Some("1"));
}
#[test]
fn set_cookie_rejects_partial_suffix_match() {
let mut jar = CookieJar::default();
// "notexample.com" ends with "example.com" as a raw substring but is not
// a subdomain, so the Domain must be rejected.
jar.set_from_header("notexample.com", "x=1; Domain=example.com");
assert!(cookie(&jar, "example.com", "x").is_none());
assert_eq!(cookie(&jar, "notexample.com", "x").as_deref(), Some("1"));
}
#[test]
fn origins_compare_by_scheme_host_port() {
let u = |s: &str| origin_of(&reqwest::Url::parse(s).unwrap());
assert_eq!(u("https://a.com/x"), u("https://a.com/y")); // path-independent
assert_eq!(u("https://a.com"), u("https://a.com:443")); // default port
assert_ne!(u("https://a.com"), u("http://a.com")); // scheme differs
assert_ne!(u("https://a.com"), u("https://b.com")); // host differs
assert_ne!(u("https://a.com"), u("https://a.com:8443")); // port differs
}
#[test]
fn cookies_for_matches_domain_and_subdomains() {
let mut jar = CookieJar::default();
jar.set_from_header("example.com", "a=1");
assert_eq!(jar.cookies_for("example.com"), vec![("a".into(), "1".into())]);
// a cookie stored on example.com is sent to its subdomains
jar.set_from_header("sub.example.com", "b=2; Domain=example.com");
let mut got = jar.cookies_for("sub.example.com");
got.sort();
assert_eq!(got, vec![("a".into(), "1".into()), ("b".into(), "2".into())]);
// but not to an unrelated host
assert!(jar.cookies_for("other.com").is_empty());
}
}