From 0f13de0251db80d1120c5b253ffd1eb7ae48edf6 Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Wed, 20 May 2026 03:20:40 +0700 Subject: [PATCH] feat: XHR-hook captures + calibrate command for native synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0024 v1.8.0 P2 + P3. P2 — px-camoufox::capture_sensor: - Spawns a fresh session, navigates to about:blank, injects a small monkey-patch that wraps `JSON.stringify` (logging `[{t,d},…]` event batches) and `XMLHttpRequest.prototype.send` (logging URL + body for /b/s targets). - Navigates to the target URL, waits, extracts captures back to Rust as a `CaptureResult` (plaintext_events, xhr_sends, cookies, ua). - Live test gated by `CAPTURE_PX=1` writes JSON to `px-research/captures/eT15wiaE/.json`. P3 — px-native::events::calibrate + `px-cli calibrate`: - `calibrate(observed, synthesised)` returns a `CalibrationReport`: missing_tags, extra_tags, per_tag observed/synth/missing keys. - New CLI subcommand: `px-cli calibrate [--json]` prints a human-readable diff for operators. - `SensorEvent` + `EventField` derive `Deserialize` so captures round-trip back into the calibrator. Operator workflow: CAPTURE_PX=1 PX_PROXIES=... cargo test -p pxsolver-camoufox \ --test capture_sensor -- --ignored --nocapture px-cli calibrate px-research/captures/eT15wiaE/.json 132 workspace tests pass; clippy clean; all files ≤200 LOC. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 1 + px-camoufox/src/infrastructure/mod.rs | 3 + .../src/infrastructure/sensor_capture.rs | 131 ++++++++++++++++++ .../src/infrastructure/sensor_capture_hook.js | 51 +++++++ px-camoufox/src/lib.rs | 1 + px-camoufox/tests/capture_sensor.rs | 87 ++++++++++++ px-cli/Cargo.toml | 1 + px-cli/src/cli.rs | 11 ++ px-cli/src/commands/calibrate.rs | 63 +++++++++ px-cli/src/commands/mod.rs | 1 + px-cli/src/main.rs | 1 + px-native/src/events/calibrate.rs | 125 +++++++++++++++++ px-native/src/events/mod.rs | 2 + px-native/src/events/model.rs | 6 +- 14 files changed, 481 insertions(+), 3 deletions(-) create mode 100644 px-camoufox/src/infrastructure/sensor_capture.rs create mode 100644 px-camoufox/src/infrastructure/sensor_capture_hook.js create mode 100644 px-camoufox/tests/capture_sensor.rs create mode 100644 px-cli/src/commands/calibrate.rs create mode 100644 px-native/src/events/calibrate.rs diff --git a/Cargo.lock b/Cargo.lock index 6eb1d80..14d5437 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,6 +1471,7 @@ dependencies = [ "clap", "pxsolver-auth", "pxsolver-detector", + "pxsolver-native", "reqwest 0.12.28", "serde", "serde_json", diff --git a/px-camoufox/src/infrastructure/mod.rs b/px-camoufox/src/infrastructure/mod.rs index 3337b03..363c990 100644 --- a/px-camoufox/src/infrastructure/mod.rs +++ b/px-camoufox/src/infrastructure/mod.rs @@ -4,6 +4,9 @@ pub mod caps; pub mod fetch_script; pub mod fetch_strategies; pub mod proxy_pool; +pub mod sensor_capture; pub mod session; pub mod session_pool; pub mod synthetic_user; + +pub use sensor_capture::{CaptureResult, CaptureXhr, capture_sensor}; diff --git a/px-camoufox/src/infrastructure/sensor_capture.rs b/px-camoufox/src/infrastructure/sensor_capture.rs new file mode 100644 index 0000000..3e1ae43 --- /dev/null +++ b/px-camoufox/src/infrastructure/sensor_capture.rs @@ -0,0 +1,131 @@ +//! Ground-truth XHR + plaintext capture for the px-3 sensor. +//! +//! Spawns a fresh Camoufox session, navigates to `about:blank`, injects +//! a small monkey-patch that records: +//! * every `JSON.stringify(arr)` where `arr` is a `[{t, d}]` shape — +//! i.e. the plaintext sensor events before XOR/base64; +//! * every `XMLHttpRequest.send(body)` whose URL contains `/b/s` — +//! the final wire payload. +//! +//! Then navigates to the target URL and waits `wait_ms` for the +//! runtime to fire. Captures are returned as a `CaptureResult` plus the +//! cookies the session collected. + +use crate::domain::config::CamoufoxConfig; +use crate::infrastructure::caps::{build_capabilities, pick_free_port, wait_for_geckodriver}; +use fantoccini::ClientBuilder; +use px_errors::AppError; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use tokio::process::Command; +use tokio::time::sleep; + +const HOOK_SCRIPT: &str = include_str!("sensor_capture_hook.js"); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureXhr { + pub url: String, + pub body: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CaptureResult { + pub plaintext_events: Vec, + pub xhr_sends: Vec, + pub cookies: Vec<(String, String)>, + pub user_agent: String, +} + +pub async fn capture_sensor( + config: &CamoufoxConfig, + proxy: Option<&str>, + target_url: &str, + wait_ms: u64, +) -> Result { + let port = pick_free_port().await?; + let mut child = Command::new(&config.geckodriver_bin) + .arg("--port") + .arg(port.to_string()) + .arg("--binary") + .arg(&config.camoufox_bin) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .map_err(|e| AppError::InternalError(format!("spawn geckodriver: {e}")))?; + wait_for_geckodriver(port, Duration::from_secs(15)).await?; + let caps = build_capabilities(config, proxy); + let endpoint = format!("http://127.0.0.1:{port}"); + let client = ClientBuilder::native() + .capabilities(caps) + .connect(&endpoint) + .await + .map_err(|e| AppError::InternalError(format!("webdriver connect: {e}")))?; + + let outcome = run_capture(&client, target_url, wait_ms).await; + let _ = client.close().await; + let _ = child.kill().await; + outcome +} + +async fn run_capture( + client: &fantoccini::Client, + target_url: &str, + wait_ms: u64, +) -> Result { + if tokio::time::timeout(Duration::from_secs(20), client.goto("about:blank")) + .await + .is_err() + { + return Err(AppError::InternalError("nav about:blank timeout".into())); + } + client + .execute(HOOK_SCRIPT, vec![]) + .await + .map_err(|e| AppError::InternalError(format!("inject hook: {e}")))?; + + if tokio::time::timeout(Duration::from_secs(60), client.goto(target_url)) + .await + .is_err() + { + return Err(AppError::InternalError(format!("nav {target_url} timeout"))); + } + sleep(Duration::from_millis(wait_ms)).await; + + let dump = client + .execute( + "return JSON.stringify({\ + captures: window.__pxCaptures || [],\ + xhr: window.__pxXhr || [],\ + ua: navigator.userAgent\ + });", + vec![], + ) + .await + .map_err(|e| AppError::InternalError(format!("read captures: {e}")))?; + let raw = dump.as_str().unwrap_or(""); + #[derive(Deserialize)] + struct Raw { + captures: Vec, + xhr: Vec, + ua: String, + } + let parsed: Raw = serde_json::from_str(raw) + .map_err(|e| AppError::InternalError(format!("parse captures: {e}")))?; + + let raw_cookies = client + .get_all_cookies() + .await + .map_err(|e| AppError::InternalError(format!("cookies: {e}")))?; + let cookies = raw_cookies + .into_iter() + .map(|c| (c.name().to_string(), c.value().to_string())) + .collect(); + + Ok(CaptureResult { + plaintext_events: parsed.captures, + xhr_sends: parsed.xhr, + cookies, + user_agent: parsed.ua, + }) +} diff --git a/px-camoufox/src/infrastructure/sensor_capture_hook.js b/px-camoufox/src/infrastructure/sensor_capture_hook.js new file mode 100644 index 0000000..4182324 --- /dev/null +++ b/px-camoufox/src/infrastructure/sensor_capture_hook.js @@ -0,0 +1,51 @@ +// Monkey-patch hooks injected into a Camoufox session so we can read +// the plaintext sensor events the PX runtime is about to encrypt, plus +// the final wire payload it POSTs to /b/s. +// +// Stores into: +// window.__pxCaptures — array of decoded `[{t, d}, …]` event batches +// window.__pxXhr — array of { url, body } for XHRs that hit /b/s +// +// The hook is idempotent — re-injecting is a no-op. +(function () { + if (window.__pxHooked) return; + window.__pxHooked = true; + window.__pxCaptures = []; + window.__pxXhr = []; + + var origStringify = JSON.stringify; + JSON.stringify = function (value) { + try { + if ( + Array.isArray(value) && + value.length > 0 && + value[0] && + typeof value[0].t === 'string' && + value[0].d && + typeof value[0].d === 'object' + ) { + // Snapshot a copy so later mutation doesn't tamper with the record. + window.__pxCaptures.push(JSON.parse(origStringify(value))); + } + } catch (_) {} + return origStringify.apply(JSON, arguments); + }; + + var origOpen = XMLHttpRequest.prototype.open; + var origSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.open = function (method, url) { + this.__pxUrl = url; + return origOpen.apply(this, arguments); + }; + XMLHttpRequest.prototype.send = function (body) { + try { + if (this.__pxUrl && String(this.__pxUrl).indexOf('/b/s') !== -1) { + window.__pxXhr.push({ + url: this.__pxUrl, + body: typeof body === 'string' ? body : null, + }); + } + } catch (_) {} + return origSend.apply(this, arguments); + }; +})(); diff --git a/px-camoufox/src/lib.rs b/px-camoufox/src/lib.rs index 6e9c983..0132854 100644 --- a/px-camoufox/src/lib.rs +++ b/px-camoufox/src/lib.rs @@ -3,3 +3,4 @@ pub mod infrastructure; pub use domain::config::{CamoufoxConfig, CamoufoxConfigError}; pub use infrastructure::camoufox_pool::CamoufoxPool; +pub use infrastructure::sensor_capture::{CaptureResult, CaptureXhr, capture_sensor}; diff --git a/px-camoufox/tests/capture_sensor.rs b/px-camoufox/tests/capture_sensor.rs new file mode 100644 index 0000000..e0e16d1 --- /dev/null +++ b/px-camoufox/tests/capture_sensor.rs @@ -0,0 +1,87 @@ +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +//! Live ground-truth capture for the px-3 sensor. +//! +//! Run manually with: +//! CAPTURE_PX=1 \ +//! [CAPTURE_URL=https://www.pedidosya.com.ar/] \ +//! [CAPTURE_WAIT_MS=12000] \ +//! [CAPTURE_OUT=px-research/captures/eT15wiaE/.json] \ +//! [PX_PROXIES=socks5://...] \ +//! cargo test -q -p pxsolver-camoufox --test capture_sensor -- --ignored --nocapture +//! +//! The output JSON contains: +//! * `plaintext_events`: every `[{t, d}, …]` batch the runtime +//! JSON-stringified during the wait window (= the input to `vP`). +//! * `xhr_sends`: URL + body of every XHR that hit `/b/s` (= the +//! wire-format payload after `vP`). +//! * `cookies`: post-solve cookie jar. +//! * `user_agent`: navigator.userAgent observed in the session. +//! +//! Use the captured batches to calibrate `px-native::events::default_batch` +//! (ADR-0024 N3 follow-up). + +use px_camoufox::{CamoufoxConfig, capture_sensor}; +use std::path::PathBuf; +use std::time::Duration; + +#[tokio::test] +#[ignore] +async fn live_capture_for_calibration() { + if std::env::var("CAPTURE_PX").ok().as_deref() != Some("1") { + eprintln!("set CAPTURE_PX=1 to run"); + return; + } + let url = + std::env::var("CAPTURE_URL").unwrap_or_else(|_| "https://www.pedidosya.com.ar/".into()); + let wait_ms: u64 = std::env::var("CAPTURE_WAIT_MS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(12_000); + let out: PathBuf = std::env::var("CAPTURE_OUT") + .map(PathBuf::from) + .unwrap_or_else(|_| { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + PathBuf::from(format!("../px-research/captures/eT15wiaE/{secs}.json")) + }); + + let mut cfg = CamoufoxConfig::from_env(); + cfg.navigate_timeout = Duration::from_secs(60); + eprintln!( + "capturing: url={url} wait_ms={wait_ms} out={}", + out.display() + ); + + let proxy = std::env::var("PX_PROXIES") + .ok() + .and_then(|s| s.split(',').next().map(|p| p.trim().to_string())); + + let result = tokio::time::timeout( + Duration::from_secs(120), + capture_sensor(&cfg, proxy.as_deref(), &url, wait_ms), + ) + .await + .expect("capture timeout") + .expect("capture ok"); + + if let Some(parent) = out.parent() { + std::fs::create_dir_all(parent).expect("create capture dir"); + } + let json = serde_json::to_string_pretty(&result).expect("serialize"); + std::fs::write(&out, json).expect("write capture"); + + eprintln!( + "\n=== CAPTURE_PX ok ===\n plaintext_events: {}\n xhr_sends: {}\n cookies: {}\n user_agent: {}\n wrote: {}", + result.plaintext_events.len(), + result.xhr_sends.len(), + result.cookies.len(), + result.user_agent, + out.display(), + ); + assert!( + !result.plaintext_events.is_empty() || !result.xhr_sends.is_empty(), + "no PX traffic captured — check CAPTURE_WAIT_MS / proxy / target reachability", + ); +} diff --git a/px-cli/Cargo.toml b/px-cli/Cargo.toml index 42e12bd..4118070 100644 --- a/px-cli/Cargo.toml +++ b/px-cli/Cargo.toml @@ -22,6 +22,7 @@ argon2 = { workspace = true } clap = { workspace = true } px-auth = { workspace = true } px-detector = { workspace = true } +px-native = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/px-cli/src/cli.rs b/px-cli/src/cli.rs index 388b182..ea02c32 100644 --- a/px-cli/src/cli.rs +++ b/px-cli/src/cli.rs @@ -26,6 +26,8 @@ pub enum Cmd { Serve, /// Solve a target URL by calling a running px-server's POST /v1/solve. Solve(SolveArgs), + /// Diff a Camoufox sensor capture against px-native's synthetic batch. + Calibrate(CalibrateArgs), } #[derive(Args, Debug)] @@ -91,3 +93,12 @@ pub struct SolveArgs { #[arg(long)] pub proxy: Option, } + +#[derive(Args, Debug)] +pub struct CalibrateArgs { + /// Path to a capture JSON emitted by `pxsolver-camoufox::capture_sensor`. + pub capture: PathBuf, + /// Print the report as JSON (default: human-readable). + #[arg(long)] + pub json: bool, +} diff --git a/px-cli/src/commands/calibrate.rs b/px-cli/src/commands/calibrate.rs new file mode 100644 index 0000000..346c838 --- /dev/null +++ b/px-cli/src/commands/calibrate.rs @@ -0,0 +1,63 @@ +//! `px-cli calibrate` — compare a Camoufox sensor capture against +//! `px-native`'s synthetic baseline batch and print the diff. Drives +//! ADR-0024 v1.8.0 P3 (field grammar calibration). + +use std::fs; + +use anyhow::{Context, Result}; +use px_native::events::{SensorEvent, SyntheticIdentity, calibrate, default_batch}; +use serde::Deserialize; + +use crate::cli::CalibrateArgs; + +#[derive(Deserialize)] +struct Capture { + plaintext_events: Vec>, +} + +pub async fn run(args: CalibrateArgs) -> Result<()> { + let raw = fs::read_to_string(&args.capture) + .with_context(|| format!("read capture {}", args.capture.display()))?; + let capture: Capture = serde_json::from_str(&raw) + .with_context(|| format!("parse capture {}", args.capture.display()))?; + + let observed: Vec = capture.plaintext_events.into_iter().flatten().collect(); + let synth = default_batch(&SyntheticIdentity::test_default(), 1_700_000_000_000); + let report = calibrate(&observed, &synth); + + if args.json { + println!( + "{}", + serde_json::to_string_pretty(&report).context("serialize report")? + ); + return Ok(()); + } + + println!("Observed events: {}", observed.len()); + println!("Synthetic events: {}\n", synth.len()); + + if !report.missing_tags.is_empty() { + println!("Missing tags (runtime emits, default_batch does not):"); + for t in &report.missing_tags { + println!(" - {t}"); + } + println!(); + } + if !report.extra_tags.is_empty() { + println!("Extra tags (default_batch emits, runtime did not):"); + for t in &report.extra_tags { + println!(" - {t}"); + } + println!(); + } + for (tag, diff) in &report.per_tag { + let obs = diff.observed_keys.len(); + let syn = diff.synthesised_keys.len(); + let miss = diff.missing_keys.len(); + println!("[{tag}] observed={obs} synth={syn} missing={miss}"); + for k in &diff.missing_keys { + println!(" + need: {k}"); + } + } + Ok(()) +} diff --git a/px-cli/src/commands/mod.rs b/px-cli/src/commands/mod.rs index d8e0ed2..7fdab9f 100644 --- a/px-cli/src/commands/mod.rs +++ b/px-cli/src/commands/mod.rs @@ -1,4 +1,5 @@ pub mod allowlist; +pub mod calibrate; pub mod detect; pub mod keys; pub mod serve; diff --git a/px-cli/src/main.rs b/px-cli/src/main.rs index 28e529b..2f66811 100644 --- a/px-cli/src/main.rs +++ b/px-cli/src/main.rs @@ -14,5 +14,6 @@ async fn main() -> Result<()> { Cmd::Allowlist { op } => commands::allowlist::run(op).await, Cmd::Serve => commands::serve::run(), Cmd::Solve(args) => commands::solve::run(args).await, + Cmd::Calibrate(args) => commands::calibrate::run(args).await, } } diff --git a/px-native/src/events/calibrate.rs b/px-native/src/events/calibrate.rs new file mode 100644 index 0000000..9d95170 --- /dev/null +++ b/px-native/src/events/calibrate.rs @@ -0,0 +1,125 @@ +//! Field-grammar calibration: compare a captured `[{t,d},…]` batch +//! against what [`super::default_batch`] would synthesise, and emit a +//! diff (missing tags, extra tags, per-tag field overlap). +//! +//! This is the consumer of ground-truth JSON produced by +//! `px-camoufox::capture_sensor` (ADR-0024 v1.8.0 P2). Use it to drive +//! P3 — closing the gap between the synthetic batch and what the +//! runtime actually emits. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::events::model::SensorEvent; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct CalibrationReport { + /// Tags the runtime emitted but `default_batch` does not. + pub missing_tags: Vec, + /// Tags `default_batch` emits but the runtime did not in this capture. + pub extra_tags: Vec, + /// Per-tag breakdown: { tag → { observed_keys, synthesised_keys, missing_keys } }. + pub per_tag: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct TagDiff { + pub observed_keys: Vec, + pub synthesised_keys: Vec, + pub missing_keys: Vec, +} + +pub fn calibrate(observed: &[SensorEvent], synthesised: &[SensorEvent]) -> CalibrationReport { + let observed_by_tag = group_by_tag(observed); + let synth_by_tag = group_by_tag(synthesised); + + let observed_tags: BTreeSet<&String> = observed_by_tag.keys().collect(); + let synth_tags: BTreeSet<&String> = synth_by_tag.keys().collect(); + + let mut report = CalibrationReport::default(); + for t in observed_tags.difference(&synth_tags) { + report.missing_tags.push((*t).clone()); + } + for t in synth_tags.difference(&observed_tags) { + report.extra_tags.push((*t).clone()); + } + for tag in observed_tags.intersection(&synth_tags) { + let obs_keys = union_keys(observed_by_tag.get(*tag).unwrap_or(&Vec::new())); + let syn_keys = union_keys(synth_by_tag.get(*tag).unwrap_or(&Vec::new())); + let mut missing: Vec = obs_keys.difference(&syn_keys).cloned().collect(); + missing.sort(); + report.per_tag.insert( + (*tag).clone(), + TagDiff { + observed_keys: sorted(obs_keys), + synthesised_keys: sorted(syn_keys), + missing_keys: missing, + }, + ); + } + report +} + +fn group_by_tag(events: &[SensorEvent]) -> BTreeMap> { + let mut by_tag: BTreeMap> = BTreeMap::new(); + for ev in events { + by_tag.entry(ev.t.clone()).or_default().push(ev); + } + by_tag +} + +fn union_keys(events: &[&SensorEvent]) -> BTreeSet { + let mut keys: BTreeSet = BTreeSet::new(); + for ev in events { + for k in ev.d.keys() { + keys.insert(k.clone()); + } + } + keys +} + +fn sorted(s: BTreeSet) -> Vec { + s.into_iter().collect() +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + use crate::events::SensorEvent; + + #[test] + fn empty_captures_yield_empty_report() { + let r = calibrate(&[], &[]); + assert!(r.missing_tags.is_empty()); + assert!(r.extra_tags.is_empty()); + } + + #[test] + fn reports_missing_and_extra_tags() { + let obs = vec![SensorEvent::new("PXobs").with("k1", "v")]; + let syn = vec![SensorEvent::new("PXsyn").with("k2", "v")]; + let r = calibrate(&obs, &syn); + assert_eq!(r.missing_tags, vec!["PXobs"]); + assert_eq!(r.extra_tags, vec!["PXsyn"]); + } + + #[test] + fn flags_missing_keys_within_shared_tag() { + let obs = vec![ + SensorEvent::new("PX561") + .with("AzNweUZUfEs=", 1u64) + .with("EwNgCVZlZDw=", "ua") + .with("MISSING_KEY", "x"), + ]; + let syn = vec![ + SensorEvent::new("PX561") + .with("AzNweUZUfEs=", 1u64) + .with("EwNgCVZlZDw=", "ua"), + ]; + let r = calibrate(&obs, &syn); + let diff = r.per_tag.get("PX561").expect("PX561 entry"); + assert_eq!(diff.missing_keys, vec!["MISSING_KEY"]); + } +} diff --git a/px-native/src/events/mod.rs b/px-native/src/events/mod.rs index 8b8da4b..6f4abdc 100644 --- a/px-native/src/events/mod.rs +++ b/px-native/src/events/mod.rs @@ -9,9 +9,11 @@ //! structure; per-tenant population is a profile concern (N5). pub mod batch; +pub mod calibrate; pub mod identity; pub mod model; pub use batch::default_batch; +pub use calibrate::{CalibrationReport, TagDiff, calibrate}; pub use identity::SyntheticIdentity; pub use model::{EventField, SensorEvent}; diff --git a/px-native/src/events/model.rs b/px-native/src/events/model.rs index 8751ea5..6e8b7e1 100644 --- a/px-native/src/events/model.rs +++ b/px-native/src/events/model.rs @@ -8,9 +8,9 @@ use std::collections::BTreeMap; -use serde::Serialize; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum EventField { Null, @@ -50,7 +50,7 @@ impl From for EventField { } } -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SensorEvent { pub t: String, pub d: BTreeMap,