From a2f7817d3119f87e0038a5f37259049c5292d5e5 Mon Sep 17 00:00:00 2001 From: Omer Faruk Oruc Date: Mon, 31 Aug 2026 13:49:34 +0300 Subject: [PATCH] Post the login form from the sandbox and replay every collection body On a login page the sensor posts twice: the sensor_data payload, then a follow-up json body. Only the first was replayed, and clicking submit in the sandbox did nothing, so a run never produced the form navigation the edge expects. - HTMLFormElement submits for real. submit() posts the entry list, requestSubmit() fires a cancellable submit event first, and a click on a submit control goes through it, so a page that calls preventDefault and posts by XHR is not double-posted. - solve replays every collection post the script made, keeping each body's own framing, then tops up with fresh payloads to the requested rounds. - A form navigation carries the header block and wire order of the browser family the fingerprint was picked from. Both are chosen in one place so they cannot drift from each other or from the fingerprint. - request gains redirects, omit_referer and omit_origin, and reports the attributes of every set-cookie line the edge sent. - The sandbox html cap no longer cuts before the last script the page loads, which was hiding the sensor on large pages. --- clients/akamai/src/discover.rs | 29 +++- clients/akamai/src/lib.rs | 140 ++++++++++++++---- clients/akamai/src/sensor.rs | 96 ++++++++++++- clients/akamai/src/session.rs | 205 +++++++++++++++++++++------ clients/akamai/tests/session.rs | 56 ++++++++ clients/kasada/src/session.rs | 1 + crates/wre-client/src/context.rs | 4 +- crates/wre-net/src/emulate.rs | 9 ++ crates/wre-net/src/http.rs | 10 ++ crates/wre-net/src/jar.rs | 78 ++++++++++ crates/wre-net/src/lib.rs | 4 +- crates/wre-sandbox/assets/browser.js | 97 ++++++++++++- crates/wre-sandbox/src/page.rs | 76 ++++++++-- crates/wre-sandbox/tests/browser.rs | 113 ++++++++++++++- 14 files changed, 817 insertions(+), 101 deletions(-) diff --git a/clients/akamai/src/discover.rs b/clients/akamai/src/discover.rs index 0e5005d..4a74447 100644 --- a/clients/akamai/src/discover.rs +++ b/clients/akamai/src/discover.rs @@ -21,7 +21,7 @@ static CHALLENGE: LazyLock = LazyLock::new(|| { }); static SEGMENT: LazyLock = - LazyLock::new(|| Regex::new(r"^[A-Za-z0-9_-]{1,24}$").expect("segment pattern")); + LazyLock::new(|| Regex::new(r"^[A-Za-z0-9_-]{1,64}$").expect("segment pattern")); const MARK: &str = "aeiouy13579"; @@ -355,4 +355,31 @@ mod tests { assert!(surface.challenge_page); assert!(!surface.is_protected()); } + + #[test] + fn eventim_obfuscated_path_with_a_27_char_segment_is_the_sensor() { + let html = r#" + + + + + +"#; + let surface = discover(html, "https://www.eventim.de/"); + let sensor = surface.sensor.expect("sensor"); + + assert_eq!(sensor.kind, Kind::Obfuscated); + assert!( + sensor + .url + .ends_with("/3PNuxeTv-7Er-OkJZQeZ2anaH-E/DYf3rVYzubX34L3b/SAsPOw/J0/MAZlIVMUAB") + ); + + let obfuscated: Vec<_> = surface + .scripts + .iter() + .filter(|script| script.kind == Kind::Obfuscated) + .collect(); + assert_eq!(obfuscated.len(), 2, "{obfuscated:?}"); + } } diff --git a/clients/akamai/src/lib.rs b/clients/akamai/src/lib.rs index 27d6ceb..73092de 100644 --- a/clients/akamai/src/lib.rs +++ b/clients/akamai/src/lib.rs @@ -13,7 +13,9 @@ use serde::Deserialize; use serde_json::{Map, Value, json}; use wre_client::client::{Client, Registration}; -use wre_client::context::{Call, Ctx, FetchRequest, HttpOptions, Jar}; +use wre_client::context::{ + Call, Ctx, FetchRequest, HttpOptions, Jar, SetCookie, parse_set_cookie, +}; use wre_client::error::{ClientError, ClientResult}; use wre_client::shape::{Shape, field}; use wre_client::spec::{Capabilities, ClientDescriptor, Concurrency, OpSpec}; @@ -227,6 +229,18 @@ pub fn describe() -> ClientDescriptor { field("kind", Shape::optional(Shape::Str)).summary( "form sends it the way the page submits a form, otherwise it goes as an XHR", ), + field("redirects", Shape::optional(Shape::Int)).summary( + "Follow this many redirects. 0 returns the first hop. Default is the session client policy", + ), + field("omit_referer", Shape::optional(Shape::Bool)).summary( + "Do not attach Referer from the session page url", + ), + field("omit_origin", Shape::optional(Shape::Bool)).summary( + "Do not attach Origin, which the session otherwise derives from the page url", + ), + field("origin", Shape::optional(Shape::Str)).summary( + "Origin header sent as given. Pass the literal string null for an opaque origin", + ), ], ), Shape::object( @@ -236,6 +250,9 @@ pub fn describe() -> ClientDescriptor { field("url", Shape::Str).summary("Url after redirects"), field("headers", Shape::Json) .summary("Response headers as name and value pairs, without set-cookie"), + field("set_cookies", Shape::Json).summary( + "One entry per set-cookie line, with its name, domain, path, same_site, secure and http_only", + ), field("body", Shape::Str), field("cookies", Shape::Json).summary("Jar after the request"), field("refused", Shape::Bool) @@ -298,8 +315,12 @@ pub fn describe() -> ClientDescriptor { "cookies", Shape::object( "CookiesInput", - [field("set", Shape::optional(Shape::Str)) - .summary("A Cookie header to seed the jar with before reading it back")], + [ + field("set", Shape::optional(Shape::Str)) + .summary("A Cookie header to seed the jar with before reading it back"), + field("url", Shape::optional(Shape::Str)) + .summary("Match or seed cookies for this url, defaults to page_url"), + ], ), Shape::object( "Cookies", @@ -478,6 +499,9 @@ fn config_shape() -> Shape { .summary("Text typed into the page's first text field. Empty picks a short word \ from the session seed") .with_default(json!("")), + field("typed_password", Shape::Str) + .summary("Value written into the page's password field before the submit click") + .with_default(json!("")), field("live_xhr", Shape::Bool) .summary("Let the sensor's own requests leave the sandbox, which is what the \ script does in a browser. Turning it off has the host post what the \ @@ -548,6 +572,8 @@ struct Config { warp: f64, #[serde(default = "default_typed")] typed: String, + #[serde(default = "default_typed")] + typed_password: String, #[serde(default)] keep_payloads: bool, #[serde(default = "default_rounds")] @@ -697,6 +723,7 @@ impl Akamai { load_posts_ms: self.config.load_posts_ms, warp: self.config.warp, typed: self.config.typed.clone(), + typed_password: self.config.typed_password.clone(), } } @@ -828,6 +855,34 @@ fn escape(text: &str) -> String { out } +/// A boolean request parameter, off unless the caller asked for it. +fn flag(params: &Value, name: &str) -> bool { + params.get(name).and_then(Value::as_bool).unwrap_or(false) +} + +/// The Origin header to send, or `None` to leave it off. +/// +/// A caller can name one, suppress it, or let it fall out of the page url the +/// way a browser would for a request that carries a body. +fn origin_for(params: &Value, method: &str, page_url: &str) -> Option { + if flag(params, "omit_origin") { + return None; + } + + if let Some(given) = params.get("origin").and_then(Value::as_str) { + return Some(given.to_string()); + } + + if method == "GET" { + return None; + } + + let parsed = url::Url::parse(page_url).ok()?; + let host = parsed.host_str()?; + + Some(format!("{}://{}", parsed.scheme(), host)) +} + fn refused(status: u16, body: &str) -> bool { if status == 403 || status == 429 { return true; @@ -925,19 +980,31 @@ impl Client for Akamai { .collect(); if post { - for round in 0..rounds { + // Replay what the script itself posted first, keeping each body's + // own framing, then top up with fresh payloads until the run has + // made the number of posts it was asked for. + let captured = session.collection_requests(); + let total = rounds.max(1).max(captured.len()); + + for round in 0..total { call.check()?; - let payload = if round == 0 { - first.clone() - } else { - session.nudge(1500.0)?; - session.payload()?.unwrap_or_else(|| first.clone()) + let sent = match captured.get(round) { + Some(request) => session.replay(request)?, + None => { + let payload = if round == 0 { + first.clone() + } else { + session.nudge(1500.0)?; + session.payload()?.unwrap_or_else(|| first.clone()) + }; + + session.post_payload(&payload, endpoint.as_deref())? + } }; - let sent = session.post_payload(&payload, endpoint.as_deref())?; posts.push(serde_json::to_value(&sent).unwrap_or(Value::Null)); - call.progress(round as u64 + 1, rounds as u64, "posted"); + call.progress(round as u64 + 1, total as u64, "posted"); } } @@ -1051,8 +1118,12 @@ impl Client for Akamai { let header = if telemetry { session.telemetry()? } else { None }; let submitted = params.get("kind").and_then(Value::as_str) == Some("form"); + let plan = session::plan(&user_agent, submitted); - let order = if submitted { &session::FORM_ORDER[..] } else { &session::XHR_ORDER[..] }; + let redirects = params + .get("redirects") + .and_then(Value::as_u64) + .map(|value| value as usize); let mut request = FetchRequest { url: url.clone(), @@ -1060,32 +1131,28 @@ impl Client for Akamai { headers: Vec::new(), body, fingerprint: None, - order: order.iter().map(|name| name.to_string()).collect(), + order: plan.order.iter().map(|name| name.to_string()).collect(), + redirects, }; request = request .header("accept-language", languages) .header("user-agent", user_agent); - if submitted { - for (name, value) in session::FORM { - request = request.header(name, value); - } - } else { + for (name, value) in plan.rows { + request = request.header(*name, *value); + } + + if !submitted { request = request.header("accept", "*/*"); } - if !session.page_url().is_empty() { + if !flag(¶ms, "omit_referer") && !session.page_url().is_empty() { request = request.header("referer", session.page_url().to_string()); + } - if request.method != "GET" { - if let Ok(parsed) = url::Url::parse(session.page_url()) { - if let Some(host) = parsed.host_str() { - let origin = format!("{}://{}", parsed.scheme(), host); - request = request.header("origin", origin); - } - } - } + if let Some(origin) = origin_for(¶ms, &request.method, session.page_url()) { + request = request.header("origin", origin); } if !submitted && params.get("form").and_then(Value::as_object).is_some() { @@ -1117,6 +1184,8 @@ impl Client for Akamai { let response = session.fetch(request)?; let text = response.text(); + let set_cookies: Vec = + response.set_cookies().into_iter().filter_map(parse_set_cookie).collect(); Ok(json!({ "status": response.status, @@ -1127,6 +1196,7 @@ impl Client for Akamai { .filter(|(name, _)| !name.eq_ignore_ascii_case("set-cookie")) .map(|(name, value)| json!([name, value])) .collect::>(), + "set_cookies": set_cookies, "body": text, "cookies": session.cookies(), "refused": refused(response.status, &text), @@ -1178,6 +1248,7 @@ impl Client for Akamai { body, fingerprint: None, order, + redirects: None, })?; Ok(json!({ @@ -1206,9 +1277,13 @@ impl Client for Akamai { } "cookies" => { - if let Some(header) = params.get("set").and_then(Value::as_str) { - let url = self.config.page_url.clone().unwrap_or_default(); + let scope = params.get("url").and_then(Value::as_str).map(str::to_string); + let url = scope + .clone() + .or_else(|| self.config.page_url.clone()) + .unwrap_or_default(); + if let Some(header) = params.get("set").and_then(Value::as_str) { for pair in header.split(';') { let trimmed = pair.trim(); if trimmed.is_empty() { @@ -1219,7 +1294,12 @@ impl Client for Akamai { } let session = self.session()?; - let pairs = session.cookie_pairs(); + let pairs = match &scope { + // A named url reads the jar the way that url would see it, with + // no url the session's own page decides what a browser sends. + Some(url) => session.cookie_pairs_at(url), + None => session.cookie_pairs(), + }; Ok(json!({ "header": pairs diff --git a/clients/akamai/src/sensor.rs b/clients/akamai/src/sensor.rs index 7465807..6e4c920 100644 --- a/clients/akamai/src/sensor.rs +++ b/clients/akamai/src/sensor.rs @@ -5,11 +5,57 @@ const OPENER: &str = "{\"sensor_data\":\""; const FIELD: &str = "sensor_data="; const SEPARATOR: &str = "&&&"; +/// How the script framed a body it posted to the collection endpoint. +/// +/// The framing decides the content type, so a replay has to keep it rather than +/// re-wrap everything as sensor data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Payload { + /// A bare `sensor_data` string, wrapped into json before it goes out. + Bare, + /// The json envelope the script built around a `sensor_data` string. + Sensor, + /// A json collection body such as `{"body":"..."}`, posted as it stands. + Collection, +} + +impl Payload { + /// The content type a browser sends this framing with. + pub fn content_type(self) -> &'static str { + match self { + Payload::Collection => "application/json", + Payload::Bare | Payload::Sensor => "text/plain;charset=UTF-8", + } + } + + /// True when the body carries sensor data rather than a collection body. + pub fn is_sensor(self) -> bool { + !matches!(self, Payload::Collection) + } +} + +/// What this request body is, or `None` when it is not one the sensor posts. +pub fn classify(body: &str) -> Option { + let Ok(parsed) = serde_json::from_str::(body) else { + return body.contains("sensor_data").then_some(Payload::Bare); + }; + + if parsed.get("sensor_data").and_then(|value| value.as_str()).is_some() { + return Some(Payload::Sensor); + } + + parsed + .get("body") + .and_then(|value| value.as_str()) + .map(|_| Payload::Collection) +} + pub fn extract(body: &str) -> Option { - if let Ok(parsed) = serde_json::from_str::(body) - && let Some(found) = parsed.get("sensor_data").and_then(|value| value.as_str()) - { - return Some(found.to_string()); + if let Ok(parsed) = serde_json::from_str::(body) { + return parsed + .get("sensor_data") + .and_then(|value| value.as_str()) + .map(str::to_string); } let start = body.find(OPENER)?; @@ -61,8 +107,15 @@ pub fn with_payload(header: &str, payload: &str) -> String { fields.join(SEPARATOR) } -pub fn looks_like_payload(body: &str) -> bool { - body.contains("sensor_data") +/// The bytes and content type to post this payload with, wrapping it first when +/// it is still a bare `sensor_data` string. +pub fn encode(payload: &str) -> (String, &'static str) { + match classify(payload) { + Some(framed @ (Payload::Sensor | Payload::Collection)) => { + (payload.to_string(), framed.content_type()) + } + _ => (wrap(payload), Payload::Bare.content_type()), + } } #[cfg(test)] @@ -73,7 +126,7 @@ mod tests { fn the_payload_comes_out_of_a_post_body() { let body = r#"{"sensor_data":"7a74G7m23Vrp0o5c9XJ~1~abc"}"#; assert_eq!(extract(body).as_deref(), Some("7a74G7m23Vrp0o5c9XJ~1~abc")); - assert!(looks_like_payload(body)); + assert_eq!(classify(body), Some(Payload::Sensor)); } #[test] @@ -88,6 +141,35 @@ mod tests { assert_eq!(extract(&wrap(payload)).as_deref(), Some(payload)); } + #[test] + fn a_body_json_post_is_a_collection_payload() { + let body = r#"{"body":"sbsd-follow"}"#; + assert_eq!(classify(body), Some(Payload::Collection)); + assert!(!classify(body).unwrap().is_sensor()); + assert_eq!(extract(body), None); + assert_eq!(encode(body), (body.to_string(), "application/json")); + } + + #[test] + fn already_wrapped_sensor_json_keeps_text_plain() { + let body = r#"{"sensor_data":"3;abc"}"#; + assert_eq!(classify(body), Some(Payload::Sensor)); + assert_eq!(encode(body), (body.to_string(), "text/plain;charset=UTF-8")); + assert_eq!(extract(body).as_deref(), Some("3;abc")); + } + + #[test] + fn a_bare_payload_is_wrapped_and_anything_else_is_not_a_payload() { + assert_eq!(classify("sensor_data=abc"), Some(Payload::Bare)); + assert_eq!( + encode("sensor_data=abc"), + (wrap("sensor_data=abc"), "text/plain;charset=UTF-8") + ); + + assert_eq!(classify(r#"{"other":1}"#), None); + assert_eq!(classify("hello"), None); + } + #[test] fn the_telemetry_header_carries_the_payload_in_base64() { let header = "a=1&&&b=2&&&sensor_data=N2E3NEc3bTIzVnJw"; diff --git a/clients/akamai/src/session.rs b/clients/akamai/src/session.rs index b36059a..5db8c96 100644 --- a/clients/akamai/src/session.rs +++ b/clients/akamai/src/session.rs @@ -8,7 +8,7 @@ use serde_json::{Value, json}; use url::Url; use wre_behavior::stream::{Point, Shape, Stream}; -use wre_client::context::{Claim, FetchRequest, FetchResponse, Http, Jar}; +use wre_client::context::{Claim, FetchRequest, FetchResponse, Http, Jar, is_firefox}; use wre_client::error::{ClientError, ClientResult}; use wre_live::realm::RealmOptions; use wre_sandbox::machine; @@ -109,6 +109,21 @@ pub const FORM: [(&str, &str); 9] = [ ("upgrade-insecure-requests", "1"), ]; +pub const FIREFOX_FORM: [(&str, &str); 9] = [ + ( + "accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + ), + ("content-type", "application/x-www-form-urlencoded"), + ("priority", "u=0, i"), + ("sec-fetch-dest", "document"), + ("sec-fetch-mode", "navigate"), + ("sec-fetch-site", "same-origin"), + ("sec-fetch-user", "?1"), + ("te", "trailers"), + ("upgrade-insecure-requests", "1"), +]; + pub const FORM_ORDER: [&str; 19] = [ "content-length", "cache-control", @@ -131,6 +146,25 @@ pub const FORM_ORDER: [&str; 19] = [ "priority", ]; +pub const FIREFOX_FORM_ORDER: [&str; 16] = [ + "content-length", + "user-agent", + "accept", + "accept-language", + "accept-encoding", + "content-type", + "origin", + "referer", + "upgrade-insecure-requests", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "cookie", + "te", + "priority", +]; + pub const XHR_ORDER: [&str; 16] = [ "content-length", "sec-ch-ua-platform", @@ -150,6 +184,27 @@ pub const XHR_ORDER: [&str; 16] = [ "priority", ]; +/// The fixed header block a form navigation carries, and the wire order the +/// request is sent in. +/// +/// Everything that builds a request picks its plan here, so the block and the +/// order it is sent in can never drift apart, and neither can drift from the +/// browser family the fingerprint was chosen for. An XHR has no fixed block: +/// each caller adds the headers its own path is responsible for. +#[derive(Debug, Clone, Copy)] +pub struct HeaderPlan { + pub rows: &'static [(&'static str, &'static str)], + pub order: &'static [&'static str], +} + +pub fn plan(user_agent: &str, form: bool) -> HeaderPlan { + match (form, is_firefox(user_agent)) { + (false, _) => HeaderPlan { rows: &[], order: &XHR_ORDER }, + (true, true) => HeaderPlan { rows: &FIREFOX_FORM, order: &FIREFOX_FORM_ORDER }, + (true, false) => HeaderPlan { rows: &FORM, order: &FORM_ORDER }, + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Settings { pub wait_ms: f64, @@ -167,6 +222,7 @@ pub struct Settings { pub load_posts_ms: f64, pub warp: f64, pub typed: String, + pub typed_password: String, } impl Default for Settings { @@ -187,6 +243,7 @@ impl Default for Settings { load_posts_ms: 4_000.0, warp: 16.0, typed: String::new(), + typed_password: String::new(), } } } @@ -510,47 +567,65 @@ impl Wire { return Answer::default(); } + // A form submit leaves the sandbox as a navigation, not an XHR, and the + // page it lands on is never parsed here, so the first hop is the answer. + let document = request.source == "document"; + let plan = plan(&self.user_agent, document); + let mut outgoing = FetchRequest { url: request.url.clone(), method: request.method.to_uppercase(), headers: Vec::new(), body: request.body.as_ref().map(|body| body.as_bytes().to_vec()), fingerprint: None, - order: XHR_ORDER.iter().map(|name| name.to_string()).collect(), + order: plan.order.iter().map(|name| name.to_string()).collect(), + redirects: if document { Some(0) } else { None }, }; let cross = !self.same_origin(&request.url); let mut headers: BTreeMap = BTreeMap::from([ - ("accept".to_string(), "*/*".to_string()), ("accept-encoding".to_string(), "gzip, deflate, br, zstd".to_string()), ("accept-language".to_string(), self.languages.clone()), - ("priority".to_string(), "u=1, i".to_string()), - ("sec-fetch-dest".to_string(), "empty".to_string()), - ("sec-fetch-mode".to_string(), "cors".to_string()), - ( - "sec-fetch-site".to_string(), - if cross { "cross-site".to_string() } else { "same-origin".to_string() }, - ), ("user-agent".to_string(), self.user_agent.clone()), ]); - if cross || outgoing.method != "GET" { - headers.insert("origin".to_string(), self.origin.clone()); + for (name, value) in plan.rows { + headers.insert((*name).to_string(), (*value).to_string()); } - headers.insert( - "referer".to_string(), - if cross { format!("{}/", self.origin) } else { self.referer.clone() }, - ); + if document { + // A sandboxed document has an opaque origin and carries no referrer. + headers.insert("origin".to_string(), "null".to_string()); + } else { + headers.extend([ + ("accept".to_string(), "*/*".to_string()), + ("priority".to_string(), "u=1, i".to_string()), + ("sec-fetch-dest".to_string(), "empty".to_string()), + ("sec-fetch-mode".to_string(), "cors".to_string()), + ( + "sec-fetch-site".to_string(), + if cross { "cross-site".to_string() } else { "same-origin".to_string() }, + ), + ]); + + if cross || outgoing.method != "GET" { + headers.insert("origin".to_string(), self.origin.clone()); + } - headers.extend(self.hints.iter().cloned()); + headers.insert( + "referer".to_string(), + if cross { format!("{}/", self.origin) } else { self.referer.clone() }, + ); + headers.extend(self.hints.iter().cloned()); + } for (name, value) in &request.headers { headers.insert(name.to_lowercase(), value.clone()); } - let names: Vec = XHR_ORDER + let names: Vec = plan + .order .iter() .filter(|name| **name == "content-length" || **name == "cookie" || headers.contains_key(**name)) .map(|name| name.to_string()) @@ -717,8 +792,10 @@ impl Session { self.browser.is_some() } - pub fn cookie_pairs(&self) -> Vec<(String, String)> { - let url = if self.page_url.is_empty() { "https://localhost/" } else { &self.page_url }; + /// The jar as name and value pairs, in the order a browser would send them to + /// `url`. + pub fn cookie_pairs_at(&self, url: &str) -> Vec<(String, String)> { + let url = if url.is_empty() { "https://localhost/" } else { url }; self.jar .matching(url) @@ -727,6 +804,10 @@ impl Session { .collect() } + pub fn cookie_pairs(&self) -> Vec<(String, String)> { + self.cookie_pairs_at(&self.page_url) + } + pub fn cookies(&self) -> Summary { cookies::summarise(&self.cookie_pairs()) } @@ -1373,18 +1454,8 @@ impl Session { self.pump(self.settings.load_posts_ms, |session| session.landed() >= 2)?; self.spent("akamai:load-posts", began); - let browser = self - .browser - .as_mut() - .ok_or_else(|| ClientError::internal("the sandbox is not mounted"))?; - - let failed = |error: wre_core::error::Error| { - ClientError::internal(format!("the sandbox stalled: {error}")) - }; - - browser - .fire("focus", json!({ "target": "window" })) - .map_err(failed)?; + self.focus_window()?; + self.fill_credentials(&typed)?; let start = Point::new((target.x - 128.0).max(12.0), (target.y - 52.0).max(12.0)); let mut stream = Stream::new(seed, start, pointer_shape()); @@ -1406,7 +1477,7 @@ impl Session { browser .play_warped(stream.events(), warp) - .map_err(failed)?; + .map_err(|error| ClientError::internal(format!("the sandbox stalled: {error}")))?; let began = std::time::Instant::now(); self.wait_for_posts(before + 1, self.settings.load_posts_ms)?; @@ -1414,6 +1485,41 @@ impl Session { Ok(()) } + fn focus_window(&mut self) -> ClientResult<()> { + let browser = self + .browser + .as_mut() + .ok_or_else(|| ClientError::internal("the sandbox is not mounted"))?; + + browser + .fire("focus", json!({ "target": "window" })) + .map_err(|error| ClientError::internal(format!("the sandbox stalled: {error}")))?; + + Ok(()) + } + + /// Write the configured identity into the page's login fields before the + /// pointer stream clicks submit, so the form posts something a login page + /// would accept rather than an empty body. + fn fill_credentials(&mut self, email: &str) -> ClientResult<()> { + let email = json!(email); + let secret = json!(self.settings.typed_password); + + self.eval(&format!( + r#"(function () {{ + var email = {email}; + var secret = {secret}; + var mail = document.querySelector('input[name="email"], input[type="email"], input[name="username"]'); + if (mail && email) mail.value = email; + var password = document.querySelector('input[name="password"], input[type="password"]'); + if (password && secret) password.value = secret; + return true; +}})()"# + ))?; + + Ok(()) + } + fn typed_text(&self, seed: u64) -> String { if !self.settings.typed.is_empty() { return self.settings.typed.clone(); @@ -1515,13 +1621,7 @@ impl Session { self.requests() .iter() .filter(|request| !request.url.contains("/akam/")) - .filter(|request| { - request - .body - .as_deref() - .map(sensor::looks_like_payload) - .unwrap_or(false) - }) + .filter(|request| request.body.as_deref().and_then(sensor::classify).is_some()) .count() } @@ -1607,7 +1707,6 @@ impl Session { .iter() .rev() .filter_map(|request| request.body.as_deref()) - .filter(|body| sensor::looks_like_payload(body)) .find_map(sensor::extract); if posted.is_some() { @@ -1634,14 +1733,27 @@ impl Session { request .body .as_deref() - .map(sensor::looks_like_payload) - .unwrap_or(false) + .and_then(sensor::classify) + .is_some_and(sensor::Payload::is_sensor) }) .map(|request| request.url.clone()); posted.or_else(|| Some(self.sensor_url.clone()).filter(|url| !url.is_empty())) } + /// Every collection post the script itself made, sensor data and follow-up + /// bodies alike, so a host-side run can replay them as they were framed. + pub fn collection_requests(&self) -> Vec { + self.requests() + .into_iter() + .filter(|request| { + request.method.eq_ignore_ascii_case("POST") + && !request.url.contains("/akam/") + && request.body.as_deref().and_then(sensor::classify).is_some() + }) + .collect() + } + pub fn post_payload(&mut self, payload: &str, endpoint: Option<&str>) -> ClientResult { let url = match endpoint { Some(found) => found.to_string(), @@ -1655,13 +1767,15 @@ impl Session { .map(|parsed| format!("{}://{}", parsed.scheme(), parsed.host_str().unwrap_or_default())) .unwrap_or_default(); - let mut request = FetchRequest::post(url.clone(), sensor::wrap(payload).into_bytes()) + let (body, content_type) = sensor::encode(payload); + + let mut request = FetchRequest::post(url.clone(), body.into_bytes()) .ordered(&XHR_ORDER) .header("accept", "*/*") .header("accept-encoding", "gzip, deflate, br, zstd") .header("accept-language", self.languages.clone()) .header("priority", "u=1, i") - .header("content-type", "text/plain;charset=UTF-8") + .header("content-type", content_type) .header("origin", origin) .header("referer", self.page_url.clone()) .header("sec-fetch-dest", "empty") @@ -1779,6 +1893,7 @@ impl Session { body: request.body.as_ref().map(|body| body.as_bytes().to_vec()), fingerprint: None, order: XHR_ORDER.iter().map(|name| name.to_string()).collect(), + redirects: None, }; let mut headers: BTreeMap = BTreeMap::from([ diff --git a/clients/akamai/tests/session.rs b/clients/akamai/tests/session.rs index e14b493..689e83f 100644 --- a/clients/akamai/tests/session.rs +++ b/clients/akamai/tests/session.rs @@ -114,11 +114,18 @@ const SENSOR: &str = r##" request.setRequestHeader("Content-Type", "application/json"); request.send(JSON.stringify({ sensor_data: payload() })); globalThis.posted = request.status; + + var follow = new XMLHttpRequest(); + follow.open("POST", location.origin + "/pWSY7c1/2ib/AKfr/hFDsQoTHmWt/YwEfMkyRK8Um/Y3g/cIdnhuansX"); + follow.setRequestHeader("Content-Type", "application/json"); + follow.send(JSON.stringify({ body: "sbsd-follow" })); + globalThis.followed = follow.status; }, 1200); })(); "##; const SENSOR_PATH: &str = "/pWSY7c1/2ib/AKfr/hFDsQoTHmWt/YwEfMkyRK8Um/Y3g/AWJXAjJfBQ"; +const BODY_PATH: &str = "/pWSY7c1/2ib/AKfr/hFDsQoTHmWt/YwEfMkyRK8Um/Y3g/cIdnhuansX"; fn page(origin: &str) -> String { format!( @@ -267,6 +274,11 @@ fn answer( reply(&mut stream, "201 Created", "application/json", &cookies, r#"{"success":true}"#); } + ("POST", path) if path == BODY_PATH => { + posts.lock().unwrap().push(body.clone()); + reply(&mut stream, "200 OK", "application/json", &[], r#"{"success":true}"#); + } + ("GET", "/akam/13/4ebc0144") => { let script = "(function () { var request = new XMLHttpRequest(); \ request.open('POST', location.origin + '/akam/13/pixel_' + (77 ^ Number(bazadebezolkohpepadr)).toString(16)); \ @@ -398,6 +410,10 @@ fn the_client_runs_a_sensor_and_carries_the_session_it_produces() { let posts = edge.posts.lock().unwrap().clone(); assert!(posts.len() >= 2, "{posts:?}"); assert!(posts.iter().any(|body| body.contains("sensor_data"))); + assert!( + posts.iter().any(|body| body.contains(r#""body":"sbsd-follow""#)), + "the second collection post was missing: {posts:?}" + ); assert!(posts.iter().any(|body| body.starts_with("pixel:")), "the pixel client did not post"); let answered = call( @@ -413,6 +429,46 @@ fn the_client_runs_a_sensor_and_carries_the_session_it_produces() { assert_eq!(edge.telemetry.lock().unwrap().len(), 1); } +#[test] +fn host_post_replays_a_body_collection_as_json() { + let edge = serve(); + let url = format!("http://127.0.0.1:{}/login", edge.port); + + let (mut client, descriptor) = client(json!({ + "page_url": url, + "wait_ms": 4000, + "rounds": 2, + "seed": 42, + "live_xhr": false, + "pixel": false, + })); + + let solved = call(&mut client, &descriptor, "solve", json!({})); + assert_eq!(solved["run"]["threw"], Value::Null); + + let posts = edge.posts.lock().unwrap().clone(); + assert!( + posts.iter().any(|body| body.contains("sensor_data")), + "sensor_data replay missing: {posts:?}" + ); + assert!( + posts.iter().any(|body| body.contains("sbsd-follow")), + "body collection replay missing: {posts:?}" + ); + + let replayed: Vec<&Value> = solved["posts"] + .as_array() + .unwrap() + .iter() + .filter(|post| post["source"].as_str().unwrap_or_default().starts_with("replay:")) + .collect(); + assert!( + replayed.iter().any(|post| post["url"].as_str().unwrap_or_default().ends_with(BODY_PATH)), + "{}", + solved["posts"] + ); +} + #[test] fn a_request_without_the_session_is_refused_by_the_edge() { let edge = serve(); diff --git a/clients/kasada/src/session.rs b/clients/kasada/src/session.rs index c7340f7..c31107d 100644 --- a/clients/kasada/src/session.rs +++ b/clients/kasada/src/session.rs @@ -269,6 +269,7 @@ impl Transport for Live { body, fingerprint: None, order: Vec::new(), + redirects: None, }; let answer = match self.http.fetch(outgoing) { diff --git a/crates/wre-client/src/context.rs b/crates/wre-client/src/context.rs index f0b62a3..3670908 100644 --- a/crates/wre-client/src/context.rs +++ b/crates/wre-client/src/context.rs @@ -9,9 +9,9 @@ use serde_json::{Value, json}; use wre_core::store::Store; use wre_net::proxy::ProxySpec; -pub use wre_net::emulate::{Claim, Fingerprint, Platform, Profile}; +pub use wre_net::emulate::{Claim, Fingerprint, Platform, Profile, is_firefox}; pub use wre_net::http::{FetchRequest, FetchResponse}; -pub use wre_net::jar::{Cookie, Jar}; +pub use wre_net::jar::{Cookie, Jar, SetCookie, parse_set_cookie}; pub use wre_net::proxy::ProxyScheme; use wre_net::http::{Client as HttpClient, ClientOptions}; diff --git a/crates/wre-net/src/emulate.rs b/crates/wre-net/src/emulate.rs index 86ea588..e64f909 100644 --- a/crates/wre-net/src/emulate.rs +++ b/crates/wre-net/src/emulate.rs @@ -261,6 +261,15 @@ fn platform_of(agent: &str) -> Platform { } } +/// True when this user agent is a Gecko build. +/// +/// Firefox sends a different header block, in a different order, than a Chromium +/// one, so anything shaping headers has to agree with the family the fingerprint +/// was picked from. +pub fn is_firefox(agent: &str) -> bool { + family_of(agent, platform_of(agent)).is_some_and(|family| family.starts_with("firefox")) +} + fn family_of(agent: &str, platform: Platform) -> Option<&'static str> { if agent.starts_with("okhttp/") { return Some("okhttp"); diff --git a/crates/wre-net/src/http.rs b/crates/wre-net/src/http.rs index e765c1e..7d4db45 100644 --- a/crates/wre-net/src/http.rs +++ b/crates/wre-net/src/http.rs @@ -194,6 +194,14 @@ impl Client { builder = builder.headers(header_map(&request.headers)?); + if let Some(redirects) = request.redirects { + builder = builder.redirect(if redirects == 0 { + wreq::redirect::Policy::none() + } else { + wreq::redirect::Policy::limited(redirects) + }); + } + if !request.order.is_empty() { let mut order = OrigHeaderMap::with_capacity(request.order.len()); for name in &request.order { @@ -266,6 +274,8 @@ pub struct FetchRequest { pub fingerprint: Option, #[serde(default)] pub order: Vec, + #[serde(default)] + pub redirects: Option, } fn get_method() -> String { diff --git a/crates/wre-net/src/jar.rs b/crates/wre-net/src/jar.rs index be75db3..ea1e918 100644 --- a/crates/wre-net/src/jar.rs +++ b/crates/wre-net/src/jar.rs @@ -1,6 +1,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use serde::{Deserialize, Serialize}; use url::Url; use wreq::cookie::{CookieStore, Cookies}; use wreq::header::HeaderValue; @@ -20,6 +21,61 @@ pub struct Cookie { pub http_only: bool, } +/// The attributes one `Set-Cookie` response line declares. +/// +/// [`Cookie`] is what the jar holds after it accepted a line. This is what the +/// edge asked for, which is what a caller needs when a cookie was rejected and +/// never made it into the jar at all. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SetCookie { + pub name: String, + pub domain: String, + pub path: String, + pub same_site: String, + pub secure: bool, + pub http_only: bool, +} + +/// Read one `Set-Cookie` line into the attributes it declares, or `None` when the +/// line carries no `name=value` at all. +pub fn parse_set_cookie(line: &str) -> Option { + let mut parts = line.split(';'); + let (name, _) = parts.next()?.trim().split_once('=')?; + + let mut out = SetCookie { name: name.trim().to_string(), ..SetCookie::default() }; + + for part in parts { + let attribute = part.trim(); + + if attribute.eq_ignore_ascii_case("secure") { + out.secure = true; + continue; + } + + if attribute.eq_ignore_ascii_case("httponly") { + out.http_only = true; + continue; + } + + let Some((key, value)) = attribute.split_once('=') else { + continue; + }; + + let key = key.trim(); + let value = value.trim().to_string(); + + if key.eq_ignore_ascii_case("domain") { + out.domain = value; + } else if key.eq_ignore_ascii_case("path") { + out.path = value; + } else if key.eq_ignore_ascii_case("samesite") { + out.same_site = value; + } + } + + Some(out) +} + #[derive(Clone)] pub struct Jar { id: String, @@ -284,6 +340,28 @@ mod tests { assert!(jar.get("https://other.test/", "_abck").is_none()); } + #[test] + fn a_set_cookie_line_reads_out_the_attributes_it_declares() { + assert_eq!( + parse_set_cookie("_abck=abc~-1~x; Domain=.example.com; Path=/; SameSite=None; Secure; HttpOnly"), + Some(SetCookie { + name: "_abck".to_string(), + domain: ".example.com".to_string(), + path: "/".to_string(), + same_site: "None".to_string(), + secure: true, + http_only: true, + }) + ); + + assert_eq!( + parse_set_cookie("bm_sz=plain"), + Some(SetCookie { name: "bm_sz".to_string(), ..SetCookie::default() }) + ); + + assert_eq!(parse_set_cookie("not-a-cookie"), None); + } + #[test] fn http_only_cookies_stay_out_of_the_script_view() { let jar = Jar::new(); diff --git a/crates/wre-net/src/lib.rs b/crates/wre-net/src/lib.rs index 3bc46d0..2fbea07 100644 --- a/crates/wre-net/src/lib.rs +++ b/crates/wre-net/src/lib.rs @@ -6,8 +6,8 @@ pub mod jar; pub mod proxy; pub mod tls; -pub use emulate::{Fingerprint, Platform, Profile}; -pub use jar::{Cookie, Jar}; +pub use emulate::{Fingerprint, Platform, Profile, is_firefox}; +pub use jar::{Cookie, Jar, SetCookie}; pub use h2::{Frame, FrameKind, H2Fingerprint, fingerprint_bytes}; pub use http::{CHROME_UA, Client, ClientOptions, FetchRequest, FetchResponse}; pub use proxy::{ProxyScheme, ProxySpec, random_session}; diff --git a/crates/wre-sandbox/assets/browser.js b/crates/wre-sandbox/assets/browser.js index 9b6f1da..0a43932 100644 --- a/crates/wre-sandbox/assets/browser.js +++ b/crates/wre-sandbox/assets/browser.js @@ -1445,7 +1445,75 @@ }, "HTMLAudioElement", HTMLMediaElement); } - HTMLFormElement.prototype.submit = function () {}; + var FORM_HEADERS = { + "content-type": "application/x-www-form-urlencoded", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "origin": "null", + "upgrade-insecure-requests": "1", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1" + }; + + // True when clicking this control submits the form it belongs to. + function submits(node) { + var type = String(node.type || "").toLowerCase(); + + return node.localName === "button" + ? type !== "button" && type !== "reset" + : type === "submit" || type === "image"; + } + + // The entry list a form submits: every named control that is not a button, + // plus the button that submitted it, the way a browser builds it. + function formEntries(form, submitter) { + var fields = form.querySelectorAll("input, select, textarea, button"); + var pairs = []; + + for (var index = 0; index < fields.length; index += 1) { + var field = fields[index]; + if (!field.name) continue; + + var type = String(field.type || "").toLowerCase(); + var button = field.localName === "button" || type === "submit" || type === "image" + || type === "button" || type === "reset"; + + if (button && field !== submitter) continue; + if ((type === "checkbox" || type === "radio") && !field.checked) continue; + + pairs.push(encodeURIComponent(field.name) + "=" + encodeURIComponent(field.value || "")); + } + + return pairs.join("&"); + } + + function submitForm(form, submitter) { + var method = String(form.getAttribute("method") || "GET").toUpperCase(); + var body = formEntries(form, submitter); + + credit(hostRequest({ + method: method, + url: resolveUrl(form.getAttribute("action") || location.href, location.href), + headers: FORM_HEADERS, + body: method === "GET" ? null : body, + at: now(), + source: "document" + })); + } + + HTMLFormElement.prototype.submit = function () { + submitForm(this, null); + }; + + HTMLFormElement.prototype.requestSubmit = function (submitter) { + var by = submitter && submits(submitter) ? submitter : null; + var asked = new Event("submit", { bubbles: true, cancelable: true }); + asked.submitter = by; + + if (this.dispatchEvent(asked)) submitForm(this, by); + }; + HTMLFormElement.prototype.reset = function () {}; HTMLInputElement.prototype.select = function () {}; @@ -2119,6 +2187,8 @@ XMLHttpRequest.DONE = 4; XMLHttpRequest.prototype.UNSENT = 0; XMLHttpRequest.prototype.OPENED = 1; + XMLHttpRequest.prototype.HEADERS_RECEIVED = 2; + XMLHttpRequest.prototype.LOADING = 3; XMLHttpRequest.prototype.DONE = 4; XMLHttpRequest.prototype.open = function (method, url, async) { @@ -2163,6 +2233,8 @@ return { body: String(body) }; } + var STATUS_TEXT = { 200: "OK", 201: "Created" }; + XMLHttpRequest.prototype.send = function (body) { requestCount += 1; @@ -2177,12 +2249,18 @@ var deliver = function (answer) { request.__answer = answer; request.status = answer.status; - request.statusText = answer.status === 200 ? "OK" : ""; + request.statusText = STATUS_TEXT[answer.status] || ""; + request.responseURL = request.__url || ""; + + request.readyState = 2; + request.dispatchEvent(new Event("readystatechange")); + request.responseText = answer.body; request.response = request.responseType === "json" ? safeParse(answer.body) : answer.body; - request.responseURL = request.__url || ""; - request.readyState = 4; + request.readyState = 3; + request.dispatchEvent(new Event("readystatechange")); + request.readyState = 4; request.dispatchEvent(new Event("readystatechange")); request.dispatchEvent(new ProgressEvent("load")); request.dispatchEvent(new ProgressEvent("loadend")); @@ -5022,9 +5100,20 @@ globalThis.dispatchEvent(made); if (type === "keypress") applyTyping(made, target); + if (type === "click") applySubmit(made, target); return true; } + // Implicit submission: clicking a submit control asks its form to submit, which + // the page can still cancel from either the click or the submit handler. + function applySubmit(event, target) { + if (event.defaultPrevented || !target || target === globalThis) return; + if (!submits(target)) return; + + var form = target.form || (target.closest && target.closest("form")); + if (form) form.requestSubmit(target); + } + function keyDetail(options) { var key = String(options.key || ""); var code = options.keyCode; diff --git a/crates/wre-sandbox/src/page.rs b/crates/wre-sandbox/src/page.rs index 9603736..f88aedf 100644 --- a/crates/wre-sandbox/src/page.rs +++ b/crates/wre-sandbox/src/page.rs @@ -161,20 +161,30 @@ impl Page { .map(|found| found.as_str().trim().to_string()) .unwrap_or_default(); - self.script_offsets = SCRIPT_TAG + let tags: Vec<(String, usize, usize)> = SCRIPT_TAG .captures_iter(html) .map(|found| { - let at = found.get(0).map_or(0, |part| part.start()); + let tag = found.get(0); let attributes = attributes(found.get(1).map_or("", |part| part.as_str())); let src = match attributes.get("src") { Some(src) => absolute(&base, src), None => "[inline]".to_string(), }; - (src, at) + (src, tag.map_or(0, |part| part.start()), tag.map_or(0, |part| part.end())) }) .collect(); + // The html the sandbox parses is capped, but a script the page loads has to + // survive the cap or the document never declares it. + let sourced_end = tags + .iter() + .filter(|(src, _, _)| src != "[inline]") + .map(|(_, _, end)| *end) + .max() + .unwrap_or(0); + + self.script_offsets = tags.into_iter().map(|(src, at, _)| (src, at)).collect(); self.scripts = self.script_offsets.iter().map(|(src, _)| src.clone()).collect(); self.inline_scripts = INLINE_SCRIPT @@ -216,11 +226,7 @@ impl Page { }) .collect(); - self.html = if html.len() > self.html_limit && self.html_limit > 0 { - html[..self.html_limit].to_string() - } else { - html.to_string() - }; + self.html = cap_html(html, self.html_limit, sourced_end); } pub fn with_referrer(mut self, referrer: impl Into) -> Self { @@ -332,6 +338,27 @@ fn attributes(text: &str) -> BTreeMap { out } +/// Trim `html` to `limit` bytes, never cutting before `keep_to`, and close the +/// document off so what is left still parses. +fn cap_html(html: &str, limit: usize, keep_to: usize) -> String { + if limit == 0 || html.len() <= limit { + return html.to_string(); + } + + let mut at = limit.max(keep_to).min(html.len()); + while at < html.len() && !html.is_char_boundary(at) { + at += 1; + } + + if at == html.len() { + return html.to_string(); + } + + let mut out = html[..at].to_string(); + out.push_str(""); + out +} + fn absolute(base: &str, href: &str) -> String { match Url::parse(base).and_then(|parsed| parsed.join(href)) { Ok(joined) => joined.to_string(), @@ -397,6 +424,39 @@ mod tests { assert_eq!(page.fields_at_current_script(), 1); } + #[test] + fn a_sensor_script_past_the_html_cap_still_lands_in_the_document() { + let sensor = r#""#; + let follow = r#""#; + let html = format!( + "x{}{}{}", + "a".repeat(350_000), + sensor, + follow + ); + + let page = Page::read("https://www.eventim.de/", &html); + + assert!(page.html.contains("")); + assert!( + page.html + .contains("/rWTdayjrz/X5BQn/lEAw/uauNSw3cE33VSV/DS92/Z1ACFiMM/CQ0B"), + "sensor src missing from sandbox html" + ); + assert!( + page.html.contains("LikY?v=8ea18caa-35d4-628a-adef-19aee1560210"), + "second collection src missing from sandbox html" + ); + assert!( + page + .scripts + .iter() + .any(|src| src.ends_with("/rWTdayjrz/X5BQn/lEAw/uauNSw3cE33VSV/DS92/Z1ACFiMM/CQ0B")), + "{:?}", + page.scripts + ); + } + #[test] fn the_location_splits_the_way_a_browser_reports_it() { let page = Page::new("https://www.example.com:8443/deep/page?a=1#top"); diff --git a/crates/wre-sandbox/tests/browser.rs b/crates/wre-sandbox/tests/browser.rs index 74609e6..bb8c030 100644 --- a/crates/wre-sandbox/tests/browser.rs +++ b/crates/wre-sandbox/tests/browser.rs @@ -47,6 +47,113 @@ fn ask(browser: &mut Browser, expression: &str) -> Value { browser.eval(expression).expect(expression) } +#[test] +fn a_click_on_submit_posts_the_form_unless_the_page_cancels_it() { + let html = r#" +
+ + + + + +
"#; + + let page = Page::read("https://login.example.com/identity/user/login", html); + let recorder = Arc::new(Recorder::default()); + let hooks = Hooks { + transport: Arc::clone(&recorder) as Arc, + cookies: Arc::new(Held::default()), + }; + let mut browser = open( + &Profile::desktop_chrome(), + &page, + hooks, + RealmOptions::default(), + ) + .expect("browser"); + + let spot = ask( + &mut browser, + "(function () { \ + var box = document.querySelector('button').getBoundingClientRect(); \ + return [box.left + box.width / 2, box.top + box.height / 2]; \ + })()", + ); + let at = json!({ "clientX": spot[0], "clientY": spot[1] }); + + browser.fire("click", at.clone()).unwrap(); + + let posted = recorder.seen.lock().unwrap().clone(); + assert_eq!(posted.len(), 1, "{posted:?}"); + assert_eq!(posted[0].method, "POST"); + assert_eq!(posted[0].source, "document"); + assert!(posted[0].url.ends_with("/login"), "{}", posted[0].url); + + let body = posted[0].body.clone().unwrap_or_default(); + assert!(body.contains("token=secret"), "{body}"); + assert!(body.contains("Username=a%40b.test"), "{body}"); + assert!(body.contains("commit=Sign%20in"), "the submitter is part of the entry list: {body}"); + assert!(!body.contains("remember"), "an unchecked box is not sent: {body}"); + + // The control arm: a page that cancels the submit event sends nothing. + browser + .run( + "document.querySelector('form') \ + .addEventListener('submit', function (event) { event.preventDefault(); });", + "test:cancel", + ) + .unwrap(); + browser.fire("click", at).unwrap(); + + assert_eq!( + recorder.seen.lock().unwrap().len(), + 1, + "a cancelled submit still reached the transport" + ); +} + +#[test] +fn a_sensor_script_past_the_html_cap_is_on_the_document() { + let html = format!( + "x{}", + "a".repeat(350_000) + ); + let url = "https://www.eventim.de/rWTdayjrz/X5BQn/lEAw/uauNSw3cE33VSV/DS92/Z1ACFiMM/CQ0B"; + let page = Page::read("https://www.eventim.de/", &html).running(url); + let mut browser = open( + &Profile::desktop_chrome(), + &page, + Hooks::default(), + RealmOptions::default(), + ) + .expect("browser"); + + let found = ask( + &mut browser, + "(function () { \ + var nodes = document.getElementsByTagName('script'); \ + for (var i = 0; i < nodes.length; i++) { \ + var src = nodes[i].src || ''; \ + if (src.indexOf('CQ0B') !== -1) return src; \ + } \ + return ''; \ + })()", + ); + assert!( + found.as_str().unwrap_or_default().ends_with("/CQ0B"), + "{found}" + ); + + browser.running_script(url).unwrap(); + assert_eq!( + ask( + &mut browser, + "document.currentScript && String(document.currentScript.src).indexOf('CQ0B') !== -1" + ), + json!(true) + ); +} + #[test] fn the_document_carries_the_page_it_was_opened_with() { let mut browser = mounted(); @@ -297,17 +404,19 @@ fn a_post_from_the_page_reaches_the_transport_and_its_answer_comes_back() { browser .run( "var request = new XMLHttpRequest(); \ + var states = []; \ + request.onreadystatechange = function () { states.push(request.readyState); }; \ request.open('POST', 'https://www.example.com/akam/13/abcdef'); \ request.setRequestHeader('Content-Type', 'application/json'); \ request.send(JSON.stringify({ sensor_data: '7a74G7m23Vrp' })); \ - globalThis.answer = [request.status, request.responseText, request.readyState];", + globalThis.answer = [request.status, request.responseText, request.readyState, request.statusText, states];", "test:xhr", ) .unwrap(); assert_eq!( ask(&mut browser, "answer"), - json!([201, r#"{"success":true}"#, 4]) + json!([201, r#"{"success":true}"#, 4, "Created", [1, 2, 3, 4]]) ); let seen = recorder.seen.lock().unwrap().clone();