luna/src/stdlib/http.rs

use std::collections::HashMap;
use std::path::Path;
use std::sync::{Arc, OnceLock};
use std::time::Duration;

use cookie_store::CookieStore;
use mlua::prelude::LuaResult;
use mlua::{Lua, Table as LuaTable, Value as LuaValue};
use reqwest_cookie_store::CookieStoreMutex;

use crate::stdlib::utils::lua_to_json;

const HTTP_LUA: &str = include_str!("../../lua/stdlib/http.lua");

const MAX_REDIRECTS: usize = 10;

// ---------------------------------------------------------------------------
// Cookie store
// ---------------------------------------------------------------------------

/// Mozilla's Public Suffix List, vendored so cookie domain checks work offline
/// (the same list curl and the browsers ship). Refresh with `make update-psl`.
fn public_suffix_list() -> &'static publicsuffix::List {
    static PSL: OnceLock<publicsuffix::List> = OnceLock::new();
    PSL.get_or_init(|| {
        include_str!("../../assets/public_suffix_list.dat")
            .parse()
            .expect("vendored public suffix list must parse")
    })
}

/// An empty RFC 6265 cookie store with public-suffix checking, so a response
/// cannot plant a cookie scoped to a whole TLD (`Domain=com`) or to an
/// unrelated domain ("cookie tossing"). Domain, Path, Secure, HttpOnly,
/// Expires and Max-Age all behave per spec — this is `cookie_store`, the same
/// crate reqwest's own cookie support builds on.
fn new_cookie_store() -> CookieStore {
    CookieStore::new_with_public_suffix(Some(public_suffix_list().clone()))
}

// ---------------------------------------------------------------------------
// Request options
// ---------------------------------------------------------------------------

/// `opts` parsed into owned values (the request may be built twice: once
/// plain, once more to answer a digest challenge).
#[derive(Default)]
struct RequestOpts {
    timeout: Option<Duration>,
    headers: Vec<(String, String)>,
    cookies: Vec<(String, String)>,
    /// Body bytes, with the Content-Type it implies (None for a raw body).
    body: Option<(Vec<u8>, Option<&'static str>)>,
    /// (is_digest, username, password).
    auth: Option<(bool, String, String)>,
    /// The caller set their own Content-Type header; it wins over the one a
    /// json/form body would otherwise imply (no silent override, no duplicate).
    has_user_content_type: bool,
}

fn parse_opts(opts: Option<&LuaTable>) -> LuaResult<RequestOpts> {
    let mut o = RequestOpts::default();
    let Some(opts) = opts else { return Ok(o) };

    if let Some(t) = opts.get::<Option<f64>>("timeout")? {
        o.timeout = Some(Duration::try_from_secs_f64(t).map_err(|_| {
            mlua::Error::external("http: timeout must be a non-negative finite number of seconds")
        })?);
    }
    if let Some(headers) = opts.get::<Option<LuaTable>>("headers")? {
        for pair in headers.pairs::<String, String>() {
            o.headers.push(pair?);
        }
    }
    if let Some(cookies) = opts.get::<Option<LuaTable>>("cookies")? {
        for pair in cookies.pairs::<String, String>() {
            o.cookies.push(pair?);
        }
    }

    // Body: json > form > raw body (first one present wins).
    if let Some(json_val) = opts.get::<Option<LuaValue>>("json")? {
        let json = lua_to_json(json_val, 0)?;
        let s = serde_json::to_string(&json).map_err(mlua::Error::external)?;
        o.body = Some((s.into_bytes(), Some("application/json")));
    } else if let Some(form) = opts.get::<Option<LuaTable>>("form")? {
        let mut ser = form_urlencoded::Serializer::new(String::new());
        for pair in form.pairs::<String, String>() {
            let (k, v) = pair?;
            ser.append_pair(&k, &v);
        }
        o.body = Some((
            ser.finish().into_bytes(),
            Some("application/x-www-form-urlencoded"),
        ));
    } else if let Some(b) = opts.get::<Option<mlua::String>>("body")? {
        o.body = Some((b.as_bytes().to_vec(), None));
    }

    // Auth: { username, password, scheme = "basic" (default) | "digest" }.
    if let Some(auth_tbl) = opts.get::<Option<LuaTable>>("auth")? {
        let username = auth_tbl
            .get::<Option<String>>("username")?
            .ok_or_else(|| mlua::Error::external("http: auth.username is required"))?;
        let password = auth_tbl
            .get::<Option<String>>("password")?
            .ok_or_else(|| mlua::Error::external("http: auth.password is required"))?;
        let is_digest = match auth_tbl.get::<Option<String>>("scheme")?.as_deref() {
            None | Some("basic") => false,
            Some("digest") => true,
            Some(other) => {
                return Err(mlua::Error::external(format!(
                    "http: auth.scheme must be \"basic\" or \"digest\", got \"{other}\""
                )));
            }
        };
        o.auth = Some((is_digest, username, password));
    }

    o.has_user_content_type = o
        .headers
        .iter()
        .any(|(k, _)| k.eq_ignore_ascii_case("content-type"));

    // A Cookie header and opts.cookies would produce two Cookie headers (or
    // silently drop one) — refuse the ambiguity instead.
    if !o.cookies.is_empty() && o.headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("cookie")) {
        return Err(mlua::Error::external(
            "http: pass cookies either in opts.cookies or as a Cookie header, not both",
        ));
    }

    Ok(o)
}

// ---------------------------------------------------------------------------
// Request execution
// ---------------------------------------------------------------------------

/// reqwest's Display often hides the interesting part ("too many redirects",
/// "operation timed out") behind a generic wrapper — surface the source chain.
fn req_error(e: reqwest::Error) -> mlua::Error {
    use std::error::Error as _;
    let mut msg = format!("http: {e}");
    let mut src = e.source();
    while let Some(s) = src {
        msg.push_str(&format!(": {s}"));
        src = s.source();
    }
    mlua::Error::external(msg)
}

/// Assemble one request. Redirects, cookie capture (including `Set-Cookie` on
/// 30x hops) and re-sending jar cookies to each hop are reqwest's job: the
/// session client carries the cookie store as its provider, and its redirect
/// policy strips Authorization/Cookie when a redirect leaves the host.
fn build_request(
    client: &reqwest::Client,
    jar: Option<&CookieStoreMutex>,
    method: &reqwest::Method,
    url: &reqwest::Url,
    o: &RequestOpts,
    digest_header: Option<&str>,
) -> reqwest::RequestBuilder {
    let mut req = client.request(method.clone(), url.clone());

    if let Some(t) = o.timeout {
        req = req.timeout(t);
    }
    for (k, v) in &o.headers {
        req = req.header(k, v);
    }

    // Per-request cookies. reqwest only injects jar cookies when the request
    // has no Cookie header, so an explicit header must merge the jar's cookies
    // itself (per-request wins on a name collision).
    if !o.cookies.is_empty() {
        let mut map: HashMap<String, String> = HashMap::new();
        if let Some(jar) = jar {
            let store = jar.lock().unwrap_or_else(|e| e.into_inner());
            for (n, v) in store.get_request_values(url) {
                map.insert(n.to_string(), v.to_string());
            }
        }
        for (k, v) in &o.cookies {
            map.insert(k.clone(), v.clone());
        }
        let header = map
            .iter()
            .map(|(k, v)| format!("{k}={v}"))
            .collect::<Vec<_>>()
            .join("; ");
        req = req.header(reqwest::header::COOKIE, header);
    }

    if let Some((bytes, implied_ct)) = &o.body {
        if let Some(ct) = implied_ct
            && !o.has_user_content_type
        {
            req = req.header(reqwest::header::CONTENT_TYPE, *ct);
        }
        req = req.body(bytes.clone());
    }

    // Basic auth goes out with the request; digest's Authorization exists only
    // after a 401 challenge has been answered (digest_header).
    match (&o.auth, digest_header) {
        (Some((false, username, password)), _) => req = req.basic_auth(username, Some(password)),
        (Some((true, ..)), Some(h)) => req = req.header(reqwest::header::AUTHORIZATION, h),
        _ => {}
    }

    req
}

/// Answer a digest challenge: the Authorization header value for retrying the
/// request that came back 401, and the URL to retry (the response's final URL,
/// so a challenge behind a redirect is answered for the right path).
fn digest_answer(
    resp: &reqwest::Response,
    method: &reqwest::Method,
    o: &RequestOpts,
) -> Option<(reqwest::Url, String)> {
    let (_, username, password) = o.auth.as_ref()?;
    let challenge = resp
        .headers()
        .get(reqwest::header::WWW_AUTHENTICATE)?
        .to_str()
        .ok()?;
    let mut prompt = digest_auth::parse(challenge).ok()?;
    let url = resp.url().clone();
    let uri = match url.query() {
        Some(q) => format!("{}?{q}", url.path()),
        None => url.path().to_string(),
    };
    let ctx = digest_auth::AuthContext::new_with_method(
        username.as_str(),
        password.as_str(),
        uri,
        o.body.as_ref().map(|(b, _)| b.as_slice()),
        // If a 303 redirect degraded the method before the challenge, this
        // replays the original method; a digest endpoint behind a
        // method-changing redirect is not a case worth carrying state for.
        digest_auth::HttpMethod::from(method.as_str()),
    );
    let answer = prompt.respond(&ctx).ok()?;
    Some((url, answer.to_header_string()))
}

/// Shared by the stateless `http.request` and a session's `:request`. A
/// session's cookie jar rides along inside `client` (as its cookie provider);
/// `jar` is passed separately only so per-request cookies can merge with it.
async fn execute_request(
    lua: Lua,
    client: reqwest::Client,
    jar: Option<Arc<CookieStoreMutex>>,
    method: String,
    url: String,
    opts: Option<LuaTable>,
) -> LuaResult<LuaTable> {
    // Every request funnels through here (http.request, the shorthands, and
    // all session methods) — the one choke point for --sandbox.
    super::fs::ensure_net(&lua)?;

    let method = reqwest::Method::from_bytes(method.to_ascii_uppercase().as_bytes())
        .map_err(|e| mlua::Error::external(format!("http: invalid method '{method}': {e}")))?;
    let url = reqwest::Url::parse(&url)
        .map_err(|e| mlua::Error::external(format!("http: invalid URL '{url}': {e}")))?;
    let o = parse_opts(opts.as_ref())?;

    let resp = build_request(&client, jar.as_deref(), &method, &url, &o, None)
        .send()
        .await
        .map_err(req_error)?;

    // Digest auth: answer a 401 challenge once, then retry the same request
    // with the computed Authorization header. (Each send has its own timeout
    // window; the challenge round-trip is a second request.)
    let resp = if matches!(o.auth, Some((true, ..)))
        && resp.status() == reqwest::StatusCode::UNAUTHORIZED
        && let Some((retry_url, header)) = digest_answer(&resp, &method, &o)
    {
        build_request(&client, jar.as_deref(), &method, &retry_url, &o, Some(&header))
            .send()
            .await
            .map_err(req_error)?
    } else {
        resp
    };

    let status = resp.status().as_u16();

    // Response headers, keyed by lowercase name. Repeats are joined with ", "
    // (RFC 9110 list semantics) — except Set-Cookie, whose values may contain
    // commas; the headers table keeps its first value and the full list is
    // exposed as resp.setCookies.
    let headers_tbl = lua.create_table()?;
    let set_cookies = lua.create_table()?;
    for name in resp.headers().keys() {
        let lname = name.as_str().to_ascii_lowercase();
        let mut values = resp.headers().get_all(name).iter();
        if name == reqwest::header::SET_COOKIE {
            if let Some(first) = values.next() {
                headers_tbl.raw_set(lname.as_str(), lua.create_string(first.as_bytes())?)?;
            }
            for v in resp.headers().get_all(name) {
                set_cookies.raw_push(lua.create_string(v.as_bytes())?)?;
            }
        } else {
            let mut joined: Vec<u8> = Vec::new();
            for (i, v) in values.enumerate() {
                if i > 0 {
                    joined.extend_from_slice(b", ");
                }
                joined.extend_from_slice(v.as_bytes());
            }
            headers_tbl.raw_set(lname.as_str(), lua.create_string(&joined)?)?;
        }
    }

    let final_url = resp.url().to_string();
    let bytes = resp.bytes().await.map_err(req_error)?;

    let out = lua.create_table()?;
    out.raw_set("status", status)?;
    out.raw_set("ok", (200..=299).contains(&status))?;
    out.raw_set("url", final_url)?;
    out.raw_set("headers", headers_tbl)?;
    out.raw_set("setCookies", set_cookies)?;
    out.raw_set("body", lua.create_string(&bytes)?)?;
    Ok(out)
}

// ---------------------------------------------------------------------------
// Client construction
// ---------------------------------------------------------------------------

/// One client per cookie jar: reqwest attaches the store at build time. The
/// stateless module shares a single jarless client; each session builds its own.
fn build_client(jar: Option<Arc<CookieStoreMutex>>) -> reqwest::Result<reqwest::Client> {
    let mut builder = reqwest::Client::builder()
        .timeout(Duration::from_secs(30))
        // reqwest sends no User-Agent by default; some edges/CDNs reject
        // empty-UA requests outright. A per-request `headers` entry overrides it.
        .user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")))
        .redirect(reqwest::redirect::Policy::limited(MAX_REDIRECTS));
    if let Some(jar) = jar {
        builder = builder.cookie_provider(jar);
    }
    builder.build()
}

// ---------------------------------------------------------------------------
// Session
// ---------------------------------------------------------------------------

/// Build the plain Lua table that backs a session. State (the cookie store)
/// lives in the closures; the Lua layer applies a metatable for the method
/// shorthands. Methods are called as `s:method(...)`, so each closure receives
/// the session table as a leading `_this` argument that it ignores.
fn make_session(
    lua: &Lua,
    client: reqwest::Client,
    jar: Arc<CookieStoreMutex>,
) -> LuaResult<LuaTable> {
    let tbl = lua.create_table()?;

    tbl.raw_set(
        "_request",
        lua.create_async_function({
            let client = client.clone();
            let jar = jar.clone();
            move |lua, (_this, method, url, opts): (LuaTable, String, String, Option<LuaTable>)| {
                let client = client.clone();
                let jar = jar.clone();
                async move { execute_request(lua, client, Some(jar), method, url, opts).await }
            }
        })?,
    )?;

    // Cookie-jar files hold live credentials; both directions go through the
    // fs permission system (async only because the permission cycle is).
    tbl.raw_set("save", {
        let jar = jar.clone();
        lua.create_async_function(move |lua, (_this, path): (LuaTable, String)| {
            let jar = jar.clone();
            async move {
                let (dir, target) =
                    super::fs::write_target("session:save", Path::new(&path)).await?;
                super::fs::ensure_access(&lua, "session:save", dir, super::fs::Wants::WRITE)
                    .await?;
                // Session cookies (no Expires/Max-Age — most login tokens) are
                // exactly what the caller wants to persist, so include them.
                let mut buf = Vec::new();
                cookie_store::serde::json::save_incl_expired_and_nonpersistent(
                    &jar.lock().unwrap_or_else(|e| e.into_inner()),
                    &mut buf,
                )
                .map_err(|e| mlua::Error::external(format!("session:save: {e}")))?;
                tokio::fs::write(&target, buf)
                    .await
                    .map_err(|e| mlua::Error::external(format!("session:save: {e}")))
            }
        })?
    })?;

    tbl.raw_set("load", {
        let jar = jar.clone();
        lua.create_async_function(move |lua, (_this, path): (LuaTable, String)| {
            let jar = jar.clone();
            async move {
                let (dir, target) =
                    super::fs::read_target("session:load", Path::new(&path)).await?;
                super::fs::ensure_access(&lua, "session:load", dir, super::fs::Wants::READ)
                    .await?;
                let src = tokio::fs::read_to_string(&target)
                    .await
                    .map_err(|e| mlua::Error::external(format!("session:load: {e}")))?;
                let loaded = load_store("session:load", &src)?;
                *jar.lock().unwrap_or_else(|e| e.into_inner()) = loaded;
                Ok(())
            }
        })?
    })?;

    tbl.raw_set("clearCookies", {
        let jar = jar.clone();
        lua.create_function(move |_, _this: LuaTable| {
            jar.lock().unwrap_or_else(|e| e.into_inner()).clear();
            Ok(())
        })?
    })?;

    tbl.raw_set("cookies", {
        let jar = jar.clone();
        lua.create_function(move |lua, _this: LuaTable| {
            let outer = lua.create_table()?;
            let store = jar.lock().unwrap_or_else(|e| e.into_inner());
            for c in store.iter_unexpired() {
                let Some(domain) = c.domain.as_cow() else { continue };
                let inner = match outer.raw_get::<Option<LuaTable>>(domain.as_ref())? {
                    Some(t) => t,
                    None => {
                        let t = lua.create_table()?;
                        outer.raw_set(domain.as_ref(), &t)?;
                        t
                    }
                };
                inner.raw_set(c.name(), c.value())?;
            }
            Ok(outer)
        })?
    })?;

    Ok(tbl)
}

/// Parse a saved jar (JSONL, one cookie per line, `cookie_store`'s format),
/// re-attaching the public suffix list — a loaded store must reject bad
/// Domain attributes exactly like a fresh one.
fn load_store(fname: &str, src: &str) -> LuaResult<CookieStore> {
    cookie_store::serde::json::load_all(src.as_bytes())
        .map_err(|e| mlua::Error::external(format!("{fname}: malformed cookie jar: {e}")))
        .map(|s| s.with_suffix_list(public_suffix_list().clone()))
}

// ---------------------------------------------------------------------------
// Installation
// ---------------------------------------------------------------------------

pub(super) fn install(lua: &Lua) -> LuaResult<()> {
    // reqwest is built with `rustls-no-provider`, so it has no crypto provider
    // of its own and panics ("No provider set") unless one is installed as the
    // process default before the client is built. `ring` is self-contained
    // (compiled in, no system OpenSSL). Idempotent across Lua states — only the
    // first install in the process wins, and they would all install ring.
    let _ = rustls::crypto::ring::default_provider().install_default();

    let client = build_client(None).map_err(mlua::Error::external)?;

    let http = lua.create_table()?;

    // Stateless http.request — no cookie jar.
    http.raw_set(
        "request",
        lua.create_async_function({
            let client = client.clone();
            move |lua, (method, url, opts): (String, String, Option<LuaTable>)| {
                let client = client.clone();
                async move { execute_request(lua, client, None, method, url, opts).await }
            }
        })?,
    )?;

    // http.session(path?) — constructor returning the raw session table.
    http.raw_set(
        "session",
        lua.create_async_function(move |lua, path: Option<String>| async move {
            let mut store = new_cookie_store();
            if let Some(p) = path {
                let (dir, target) =
                    super::fs::read_target("http.session", Path::new(&p)).await?;
                super::fs::ensure_access(&lua, "http.session", dir, super::fs::Wants::READ)
                    .await?;
                let src = tokio::fs::read_to_string(&target)
                    .await
                    .map_err(|e| mlua::Error::external(format!("http.session: {e}")))?;
                store = load_store("http.session", &src)?;
            }
            let jar = Arc::new(CookieStoreMutex::new(store));
            let client = build_client(Some(jar.clone())).map_err(mlua::Error::external)?;
            make_session(&lua, client, jar)
        })?,
    )?;

    lua.globals().raw_set("http", http)?;

    // Lua side adds: resp.json(), method shorthands, getJSON/postJSON, and the
    // session metatable wrapping http.session().
    lua.load(HTTP_LUA).set_name("@[stdlib/http]").exec()
}

#[cfg(test)]
mod tests {
    //! Unit tests for the cookie-store policy this module relies on: these pin
    //! the security properties (public-suffix, cookie tossing, Secure,
    //! Max-Age) that motivated using `cookie_store` over a hand-rolled jar.
    //! End-to-end behaviour is covered in `crate::lua_tests::http_tests`.

    use super::*;

    fn url(s: &str) -> reqwest::Url {
        reqwest::Url::parse(s).unwrap()
    }

    fn names_for(store: &CookieStore, u: &str) -> Vec<String> {
        let mut v: Vec<String> =
            store.get_request_values(&url(u)).map(|(n, _)| n.to_string()).collect();
        v.sort();
        v
    }

    #[test]
    fn rejects_public_suffix_domain() {
        let mut store = new_cookie_store();
        // A TLD-scoped cookie would be sent to every .com host ("supercookie").
        assert!(store.parse("x=1; Domain=com", &url("http://evil.com/")).is_err());
        assert!(store.iter_any().next().is_none());
    }

    #[test]
    fn rejects_foreign_domain() {
        let mut store = new_cookie_store();
        // Cookie tossing: a response must not plant a cookie for an unrelated
        // domain, including a raw-substring "suffix" that is not a subdomain.
        assert!(store.parse("x=1; Domain=other.example.com", &url("http://evil.com/")).is_err());
        assert!(store.parse("x=1; Domain=example.com", &url("http://notexample.com/")).is_err());
        assert!(store.iter_any().next().is_none());
    }

    #[test]
    fn subdomain_may_scope_to_parent() {
        let mut store = new_cookie_store();
        store.parse("sid=abc; Domain=example.com", &url("http://app.example.com/")).unwrap();
        // Sent to the parent and to sibling subdomains, not to strangers.
        assert_eq!(names_for(&store, "http://example.com/"), ["sid"]);
        assert_eq!(names_for(&store, "http://other.example.com/"), ["sid"]);
        assert!(names_for(&store, "http://example.org/").is_empty());
    }

    #[test]
    fn secure_cookie_not_sent_over_http() {
        let mut store = new_cookie_store();
        store.parse("s=1; Secure", &url("https://example.com/")).unwrap();
        assert_eq!(names_for(&store, "https://example.com/"), ["s"]);
        assert!(names_for(&store, "http://example.com/").is_empty());
    }

    #[test]
    fn max_age_zero_expires_cookie() {
        let u = url("http://example.com/");
        let mut store = new_cookie_store();
        store.parse("sid=abc", &u).unwrap();
        assert_eq!(names_for(&store, "http://example.com/"), ["sid"]);
        // How servers delete a cookie (e.g. logout).
        let _ = store.parse("sid=; Max-Age=0", &u);
        assert!(names_for(&store, "http://example.com/").is_empty());
    }

    #[test]
    fn save_load_round_trips_session_cookies() {
        let mut store = new_cookie_store();
        // No Expires/Max-Age — a session cookie, which plain save would drop.
        store.parse("sid=abc", &url("https://example.com/")).unwrap();
        let mut buf = Vec::new();
        cookie_store::serde::json::save_incl_expired_and_nonpersistent(&store, &mut buf).unwrap();

        let loaded = load_store("test", std::str::from_utf8(&buf).unwrap()).unwrap();
        assert_eq!(names_for(&loaded, "https://example.com/"), ["sid"]);
        // The reloaded store must keep rejecting public-suffix domains.
        let mut loaded = loaded;
        assert!(loaded.parse("x=1; Domain=com", &url("http://evil.com/")).is_err());
    }

    #[test]
    fn load_rejects_malformed_jar() {
        assert!(load_store("test", "not json\n").is_err());
    }
}