use std::collections::HashMap; use std::path::Path; use std::sync::{Arc, OnceLock}; use std::time::Duration; use cookie_store::CookieStore; use mlua::prelude::LuaResult; use mlua::{Lua, Table as LuaTable, Value as LuaValue}; use reqwest_cookie_store::CookieStoreMutex; use crate::stdlib::utils::lua_to_json; const HTTP_LUA: &str = include_str!("../../lua/stdlib/http.lua"); const MAX_REDIRECTS: usize = 10; // --------------------------------------------------------------------------- // Cookie store // --------------------------------------------------------------------------- /// Mozilla's Public Suffix List, vendored so cookie domain checks work offline /// (the same list curl and the browsers ship). Refresh with `make update-psl`. fn public_suffix_list() -> &'static publicsuffix::List { static PSL: OnceLock = OnceLock::new(); PSL.get_or_init(|| { include_str!("../../assets/public_suffix_list.dat") .parse() .expect("vendored public suffix list must parse") }) } /// An empty RFC 6265 cookie store with public-suffix checking, so a response /// cannot plant a cookie scoped to a whole TLD (`Domain=com`) or to an /// unrelated domain ("cookie tossing"). Domain, Path, Secure, HttpOnly, /// Expires and Max-Age all behave per spec — this is `cookie_store`, the same /// crate reqwest's own cookie support builds on. fn new_cookie_store() -> CookieStore { CookieStore::new_with_public_suffix(Some(public_suffix_list().clone())) } // --------------------------------------------------------------------------- // Request options // --------------------------------------------------------------------------- /// `opts` parsed into owned values (the request may be built twice: once /// plain, once more to answer a digest challenge). #[derive(Default)] struct RequestOpts { timeout: Option, headers: Vec<(String, String)>, cookies: Vec<(String, String)>, /// Body bytes, with the Content-Type it implies (None for a raw body). body: Option<(Vec, Option<&'static str>)>, /// (is_digest, username, password). auth: Option<(bool, String, String)>, /// The caller set their own Content-Type header; it wins over the one a /// json/form body would otherwise imply (no silent override, no duplicate). has_user_content_type: bool, } fn parse_opts(opts: Option<&LuaTable>) -> LuaResult { let mut o = RequestOpts::default(); let Some(opts) = opts else { return Ok(o) }; if let Some(t) = opts.get::>("timeout")? { o.timeout = Some(Duration::try_from_secs_f64(t).map_err(|_| { mlua::Error::external("http: timeout must be a non-negative finite number of seconds") })?); } if let Some(headers) = opts.get::>("headers")? { for pair in headers.pairs::() { o.headers.push(pair?); } } if let Some(cookies) = opts.get::>("cookies")? { for pair in cookies.pairs::() { o.cookies.push(pair?); } } // Body: json > form > raw body (first one present wins). if let Some(json_val) = opts.get::>("json")? { let json = lua_to_json(json_val, 0)?; let s = serde_json::to_string(&json).map_err(mlua::Error::external)?; o.body = Some((s.into_bytes(), Some("application/json"))); } else if let Some(form) = opts.get::>("form")? { let mut ser = form_urlencoded::Serializer::new(String::new()); for pair in form.pairs::() { let (k, v) = pair?; ser.append_pair(&k, &v); } o.body = Some(( ser.finish().into_bytes(), Some("application/x-www-form-urlencoded"), )); } else if let Some(b) = opts.get::>("body")? { o.body = Some((b.as_bytes().to_vec(), None)); } // Auth: { username, password, scheme = "basic" (default) | "digest" }. if let Some(auth_tbl) = opts.get::>("auth")? { let username = auth_tbl .get::>("username")? .ok_or_else(|| mlua::Error::external("http: auth.username is required"))?; let password = auth_tbl .get::>("password")? .ok_or_else(|| mlua::Error::external("http: auth.password is required"))?; let is_digest = match auth_tbl.get::>("scheme")?.as_deref() { None | Some("basic") => false, Some("digest") => true, Some(other) => { return Err(mlua::Error::external(format!( "http: auth.scheme must be \"basic\" or \"digest\", got \"{other}\"" ))); } }; o.auth = Some((is_digest, username, password)); } o.has_user_content_type = o .headers .iter() .any(|(k, _)| k.eq_ignore_ascii_case("content-type")); // A Cookie header and opts.cookies would produce two Cookie headers (or // silently drop one) — refuse the ambiguity instead. if !o.cookies.is_empty() && o.headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("cookie")) { return Err(mlua::Error::external( "http: pass cookies either in opts.cookies or as a Cookie header, not both", )); } Ok(o) } // --------------------------------------------------------------------------- // Request execution // --------------------------------------------------------------------------- /// reqwest's Display often hides the interesting part ("too many redirects", /// "operation timed out") behind a generic wrapper — surface the source chain. fn req_error(e: reqwest::Error) -> mlua::Error { use std::error::Error as _; let mut msg = format!("http: {e}"); let mut src = e.source(); while let Some(s) = src { msg.push_str(&format!(": {s}")); src = s.source(); } mlua::Error::external(msg) } /// Assemble one request. Redirects, cookie capture (including `Set-Cookie` on /// 30x hops) and re-sending jar cookies to each hop are reqwest's job: the /// session client carries the cookie store as its provider, and its redirect /// policy strips Authorization/Cookie when a redirect leaves the host. fn build_request( client: &reqwest::Client, jar: Option<&CookieStoreMutex>, method: &reqwest::Method, url: &reqwest::Url, o: &RequestOpts, digest_header: Option<&str>, ) -> reqwest::RequestBuilder { let mut req = client.request(method.clone(), url.clone()); if let Some(t) = o.timeout { req = req.timeout(t); } for (k, v) in &o.headers { req = req.header(k, v); } // Per-request cookies. reqwest only injects jar cookies when the request // has no Cookie header, so an explicit header must merge the jar's cookies // itself (per-request wins on a name collision). if !o.cookies.is_empty() { let mut map: HashMap = HashMap::new(); if let Some(jar) = jar { let store = jar.lock().unwrap_or_else(|e| e.into_inner()); for (n, v) in store.get_request_values(url) { map.insert(n.to_string(), v.to_string()); } } for (k, v) in &o.cookies { map.insert(k.clone(), v.clone()); } let header = map .iter() .map(|(k, v)| format!("{k}={v}")) .collect::>() .join("; "); req = req.header(reqwest::header::COOKIE, header); } if let Some((bytes, implied_ct)) = &o.body { if let Some(ct) = implied_ct && !o.has_user_content_type { req = req.header(reqwest::header::CONTENT_TYPE, *ct); } req = req.body(bytes.clone()); } // Basic auth goes out with the request; digest's Authorization exists only // after a 401 challenge has been answered (digest_header). match (&o.auth, digest_header) { (Some((false, username, password)), _) => req = req.basic_auth(username, Some(password)), (Some((true, ..)), Some(h)) => req = req.header(reqwest::header::AUTHORIZATION, h), _ => {} } req } /// Answer a digest challenge: the Authorization header value for retrying the /// request that came back 401, and the URL to retry (the response's final URL, /// so a challenge behind a redirect is answered for the right path). fn digest_answer( resp: &reqwest::Response, method: &reqwest::Method, o: &RequestOpts, ) -> Option<(reqwest::Url, String)> { let (_, username, password) = o.auth.as_ref()?; let challenge = resp .headers() .get(reqwest::header::WWW_AUTHENTICATE)? .to_str() .ok()?; let mut prompt = digest_auth::parse(challenge).ok()?; let url = resp.url().clone(); let uri = match url.query() { Some(q) => format!("{}?{q}", url.path()), None => url.path().to_string(), }; let ctx = digest_auth::AuthContext::new_with_method( username.as_str(), password.as_str(), uri, o.body.as_ref().map(|(b, _)| b.as_slice()), // If a 303 redirect degraded the method before the challenge, this // replays the original method; a digest endpoint behind a // method-changing redirect is not a case worth carrying state for. digest_auth::HttpMethod::from(method.as_str()), ); let answer = prompt.respond(&ctx).ok()?; Some((url, answer.to_header_string())) } /// Shared by the stateless `http.request` and a session's `:request`. A /// session's cookie jar rides along inside `client` (as its cookie provider); /// `jar` is passed separately only so per-request cookies can merge with it. async fn execute_request( lua: Lua, client: reqwest::Client, jar: Option>, method: String, url: String, opts: Option, ) -> LuaResult { // Every request funnels through here (http.request, the shorthands, and // all session methods) — the one choke point for --sandbox. super::fs::ensure_net(&lua)?; let method = reqwest::Method::from_bytes(method.to_ascii_uppercase().as_bytes()) .map_err(|e| mlua::Error::external(format!("http: invalid method '{method}': {e}")))?; let url = reqwest::Url::parse(&url) .map_err(|e| mlua::Error::external(format!("http: invalid URL '{url}': {e}")))?; let o = parse_opts(opts.as_ref())?; let resp = build_request(&client, jar.as_deref(), &method, &url, &o, None) .send() .await .map_err(req_error)?; // Digest auth: answer a 401 challenge once, then retry the same request // with the computed Authorization header. (Each send has its own timeout // window; the challenge round-trip is a second request.) let resp = if matches!(o.auth, Some((true, ..))) && resp.status() == reqwest::StatusCode::UNAUTHORIZED && let Some((retry_url, header)) = digest_answer(&resp, &method, &o) { build_request(&client, jar.as_deref(), &method, &retry_url, &o, Some(&header)) .send() .await .map_err(req_error)? } else { resp }; let status = resp.status().as_u16(); // Response headers, keyed by lowercase name. Repeats are joined with ", " // (RFC 9110 list semantics) — except Set-Cookie, whose values may contain // commas; the headers table keeps its first value and the full list is // exposed as resp.setCookies. let headers_tbl = lua.create_table()?; let set_cookies = lua.create_table()?; for name in resp.headers().keys() { let lname = name.as_str().to_ascii_lowercase(); let mut values = resp.headers().get_all(name).iter(); if name == reqwest::header::SET_COOKIE { if let Some(first) = values.next() { headers_tbl.raw_set(lname.as_str(), lua.create_string(first.as_bytes())?)?; } for v in resp.headers().get_all(name) { set_cookies.raw_push(lua.create_string(v.as_bytes())?)?; } } else { let mut joined: Vec = Vec::new(); for (i, v) in values.enumerate() { if i > 0 { joined.extend_from_slice(b", "); } joined.extend_from_slice(v.as_bytes()); } headers_tbl.raw_set(lname.as_str(), lua.create_string(&joined)?)?; } } let final_url = resp.url().to_string(); let bytes = resp.bytes().await.map_err(req_error)?; let out = lua.create_table()?; out.raw_set("status", status)?; out.raw_set("ok", (200..=299).contains(&status))?; out.raw_set("url", final_url)?; out.raw_set("headers", headers_tbl)?; out.raw_set("setCookies", set_cookies)?; out.raw_set("body", lua.create_string(&bytes)?)?; Ok(out) } // --------------------------------------------------------------------------- // Client construction // --------------------------------------------------------------------------- /// One client per cookie jar: reqwest attaches the store at build time. The /// stateless module shares a single jarless client; each session builds its own. fn build_client(jar: Option>) -> reqwest::Result { let mut builder = reqwest::Client::builder() .timeout(Duration::from_secs(30)) // reqwest sends no User-Agent by default; some edges/CDNs reject // empty-UA requests outright. A per-request `headers` entry overrides it. .user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"))) .redirect(reqwest::redirect::Policy::limited(MAX_REDIRECTS)); if let Some(jar) = jar { builder = builder.cookie_provider(jar); } builder.build() } // --------------------------------------------------------------------------- // Session // --------------------------------------------------------------------------- /// Build the plain Lua table that backs a session. State (the cookie store) /// lives in the closures; the Lua layer applies a metatable for the method /// shorthands. Methods are called as `s:method(...)`, so each closure receives /// the session table as a leading `_this` argument that it ignores. fn make_session( lua: &Lua, client: reqwest::Client, jar: Arc, ) -> LuaResult { let tbl = lua.create_table()?; tbl.raw_set( "_request", lua.create_async_function({ let client = client.clone(); let jar = jar.clone(); move |lua, (_this, method, url, opts): (LuaTable, String, String, Option)| { let client = client.clone(); let jar = jar.clone(); async move { execute_request(lua, client, Some(jar), method, url, opts).await } } })?, )?; // Cookie-jar files hold live credentials; both directions go through the // fs permission system (async only because the permission cycle is). tbl.raw_set("save", { let jar = jar.clone(); lua.create_async_function(move |lua, (_this, path): (LuaTable, String)| { let jar = jar.clone(); async move { let (dir, target) = super::fs::write_target("session:save", Path::new(&path)).await?; super::fs::ensure_access(&lua, "session:save", dir, super::fs::Wants::WRITE) .await?; // Session cookies (no Expires/Max-Age — most login tokens) are // exactly what the caller wants to persist, so include them. let mut buf = Vec::new(); cookie_store::serde::json::save_incl_expired_and_nonpersistent( &jar.lock().unwrap_or_else(|e| e.into_inner()), &mut buf, ) .map_err(|e| mlua::Error::external(format!("session:save: {e}")))?; tokio::fs::write(&target, buf) .await .map_err(|e| mlua::Error::external(format!("session:save: {e}"))) } })? })?; tbl.raw_set("load", { let jar = jar.clone(); lua.create_async_function(move |lua, (_this, path): (LuaTable, String)| { let jar = jar.clone(); async move { let (dir, target) = super::fs::read_target("session:load", Path::new(&path)).await?; super::fs::ensure_access(&lua, "session:load", dir, super::fs::Wants::READ) .await?; let src = tokio::fs::read_to_string(&target) .await .map_err(|e| mlua::Error::external(format!("session:load: {e}")))?; let loaded = load_store("session:load", &src)?; *jar.lock().unwrap_or_else(|e| e.into_inner()) = loaded; Ok(()) } })? })?; tbl.raw_set("clearCookies", { let jar = jar.clone(); lua.create_function(move |_, _this: LuaTable| { jar.lock().unwrap_or_else(|e| e.into_inner()).clear(); Ok(()) })? })?; tbl.raw_set("cookies", { let jar = jar.clone(); lua.create_function(move |lua, _this: LuaTable| { let outer = lua.create_table()?; let store = jar.lock().unwrap_or_else(|e| e.into_inner()); for c in store.iter_unexpired() { let Some(domain) = c.domain.as_cow() else { continue }; let inner = match outer.raw_get::>(domain.as_ref())? { Some(t) => t, None => { let t = lua.create_table()?; outer.raw_set(domain.as_ref(), &t)?; t } }; inner.raw_set(c.name(), c.value())?; } Ok(outer) })? })?; Ok(tbl) } /// Parse a saved jar (JSONL, one cookie per line, `cookie_store`'s format), /// re-attaching the public suffix list — a loaded store must reject bad /// Domain attributes exactly like a fresh one. fn load_store(fname: &str, src: &str) -> LuaResult { cookie_store::serde::json::load_all(src.as_bytes()) .map_err(|e| mlua::Error::external(format!("{fname}: malformed cookie jar: {e}"))) .map(|s| s.with_suffix_list(public_suffix_list().clone())) } // --------------------------------------------------------------------------- // Installation // --------------------------------------------------------------------------- pub(super) fn install(lua: &Lua) -> LuaResult<()> { // reqwest is built with `rustls-no-provider`, so it has no crypto provider // of its own and panics ("No provider set") unless one is installed as the // process default before the client is built. `ring` is self-contained // (compiled in, no system OpenSSL). Idempotent across Lua states — only the // first install in the process wins, and they would all install ring. let _ = rustls::crypto::ring::default_provider().install_default(); let client = build_client(None).map_err(mlua::Error::external)?; let http = lua.create_table()?; // Stateless http.request — no cookie jar. http.raw_set( "request", lua.create_async_function({ let client = client.clone(); move |lua, (method, url, opts): (String, String, Option)| { let client = client.clone(); async move { execute_request(lua, client, None, method, url, opts).await } } })?, )?; // http.session(path?) — constructor returning the raw session table. http.raw_set( "session", lua.create_async_function(move |lua, path: Option| async move { let mut store = new_cookie_store(); if let Some(p) = path { let (dir, target) = super::fs::read_target("http.session", Path::new(&p)).await?; super::fs::ensure_access(&lua, "http.session", dir, super::fs::Wants::READ) .await?; let src = tokio::fs::read_to_string(&target) .await .map_err(|e| mlua::Error::external(format!("http.session: {e}")))?; store = load_store("http.session", &src)?; } let jar = Arc::new(CookieStoreMutex::new(store)); let client = build_client(Some(jar.clone())).map_err(mlua::Error::external)?; make_session(&lua, client, jar) })?, )?; lua.globals().raw_set("http", http)?; // Lua side adds: resp.json(), method shorthands, getJSON/postJSON, and the // session metatable wrapping http.session(). lua.load(HTTP_LUA).set_name("@[stdlib/http]").exec() } #[cfg(test)] mod tests { //! Unit tests for the cookie-store policy this module relies on: these pin //! the security properties (public-suffix, cookie tossing, Secure, //! Max-Age) that motivated using `cookie_store` over a hand-rolled jar. //! End-to-end behaviour is covered in `crate::lua_tests::http_tests`. use super::*; fn url(s: &str) -> reqwest::Url { reqwest::Url::parse(s).unwrap() } fn names_for(store: &CookieStore, u: &str) -> Vec { let mut v: Vec = store.get_request_values(&url(u)).map(|(n, _)| n.to_string()).collect(); v.sort(); v } #[test] fn rejects_public_suffix_domain() { let mut store = new_cookie_store(); // A TLD-scoped cookie would be sent to every .com host ("supercookie"). assert!(store.parse("x=1; Domain=com", &url("http://evil.com/")).is_err()); assert!(store.iter_any().next().is_none()); } #[test] fn rejects_foreign_domain() { let mut store = new_cookie_store(); // Cookie tossing: a response must not plant a cookie for an unrelated // domain, including a raw-substring "suffix" that is not a subdomain. assert!(store.parse("x=1; Domain=other.example.com", &url("http://evil.com/")).is_err()); assert!(store.parse("x=1; Domain=example.com", &url("http://notexample.com/")).is_err()); assert!(store.iter_any().next().is_none()); } #[test] fn subdomain_may_scope_to_parent() { let mut store = new_cookie_store(); store.parse("sid=abc; Domain=example.com", &url("http://app.example.com/")).unwrap(); // Sent to the parent and to sibling subdomains, not to strangers. assert_eq!(names_for(&store, "http://example.com/"), ["sid"]); assert_eq!(names_for(&store, "http://other.example.com/"), ["sid"]); assert!(names_for(&store, "http://example.org/").is_empty()); } #[test] fn secure_cookie_not_sent_over_http() { let mut store = new_cookie_store(); store.parse("s=1; Secure", &url("https://example.com/")).unwrap(); assert_eq!(names_for(&store, "https://example.com/"), ["s"]); assert!(names_for(&store, "http://example.com/").is_empty()); } #[test] fn max_age_zero_expires_cookie() { let u = url("http://example.com/"); let mut store = new_cookie_store(); store.parse("sid=abc", &u).unwrap(); assert_eq!(names_for(&store, "http://example.com/"), ["sid"]); // How servers delete a cookie (e.g. logout). let _ = store.parse("sid=; Max-Age=0", &u); assert!(names_for(&store, "http://example.com/").is_empty()); } #[test] fn save_load_round_trips_session_cookies() { let mut store = new_cookie_store(); // No Expires/Max-Age — a session cookie, which plain save would drop. store.parse("sid=abc", &url("https://example.com/")).unwrap(); let mut buf = Vec::new(); cookie_store::serde::json::save_incl_expired_and_nonpersistent(&store, &mut buf).unwrap(); let loaded = load_store("test", std::str::from_utf8(&buf).unwrap()).unwrap(); assert_eq!(names_for(&loaded, "https://example.com/"), ["sid"]); // The reloaded store must keep rejecting public-suffix domains. let mut loaded = loaded; assert!(loaded.parse("x=1; Domain=com", &url("http://evil.com/")).is_err()); } #[test] fn load_rejects_malformed_jar() { assert!(load_store("test", "not json\n").is_err()); } }