From 9d3cdba0ed948b851a4bcc668615cb91e51ab1a6 Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Fri, 7 Aug 2026 21:56:33 +0700 Subject: [PATCH 1/7] feat(proxy): honour the per-request egress proxy end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three of four proxy surfaces were wired to nothing. `POST /v1/solve` deserialized `"proxy"` into SolveRequestDto and never read it, `px-cli solve --proxy` sent it in the body for the server to drop, and the published `pxsolver_core::SolveRequest::with_proxy()` had zero consumers in the workspace. `HarvestRequest.proxy` was honoured by CamoufoxPool and silently ignored by ChromiumoxidePool, which built its BrowserConfig with no --proxy-server argument at all. The cause was structural: SolveDispatcher::solve took `&str` and ChallengeHandler::solve took `&PageHtml`, so no proxy could travel inward, and both browser handlers built HarvestRequest::new(url), which defaults proxy to None. ChallengeHandler::solve now takes a SolveAction { page, proxy } built at the HTTP edge; SolveDispatcher::solve takes px_core::SolveRequest, making the published builder the type the edge maps into. Both browser handlers forward it, ChromiumoxidePool gained --proxy-server, and the native sensor path posts through a per-proxy reqwest client cached in ProxyClients. Solving deliberately does not fall back to the PX_PROXIES rotation: _px3 is bound to the IP that earned it, so a caller who named no egress could not use a bundle harvested through a rotating one. Rotation stays with /v1/fetch sessions. The proxy is hashed into the cache key so a bundle earned through proxy A is never replayed for proxy B; a direct solve keeps fp_key 0 and existing entries still resolve. Browser engines cannot authenticate to a proxy — geckodriver's W3C proxy capability has no credential field and Chromium ignores userinfo without a CDP Fetch.authRequired handler — so strip_credentials removes `user:pass@` and warns. reqwest does support it, so the native path is exempt. Docs corrected alongside: deployment.md claimed N x len(proxies) parallel egress paths, but SessionPool::acquire only draws a proxy when spawning, so distinct egress IPs per domain is min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES)). The native runbook told operators PX_PROXIES covered the solve path, and its soak built a direct client while claiming to run through their proxy. Breaking: ChallengeHandler and SolveDispatcher signatures change. See ADR-0025. --- README.md | 6 +- .../0025-egress-proxy-propagation-contract.md | 76 ++++++++++ docs/adr/README.md | 2 + docs/deployment.md | 66 ++++++++- docs/runbook-native-bypass.md | 23 +++- .../src/infrastructure/camoufox_pool.rs | 11 +- .../src/infrastructure/session_pool.rs | 10 -- px-captcha/src/lib.rs | 4 +- px-cli/Cargo.toml | 1 + px-cli/src/cli.rs | 5 +- px-cli/src/commands/solve.rs | 15 +- px-cloudflare/Cargo.toml | 2 +- px-cloudflare/src/lib.rs | 60 ++++++-- px-core/src/solve_request.rs | 11 ++ px-datadome/src/lib.rs | 4 +- px-harvester/src/domain/harvester.rs | 8 ++ .../src/infrastructure/chromiumoxide_pool.rs | 17 ++- px-harvester/src/infrastructure/egress.rs | 58 ++++++++ px-harvester/src/infrastructure/mod.rs | 1 + px-harvester/src/lib.rs | 1 + px-native/src/domain/native_solver.rs | 9 ++ px-native/src/infrastructure/handler.rs | 43 ++++-- px-native/src/infrastructure/mod.rs | 1 + px-native/src/infrastructure/native_first.rs | 20 +-- px-native/src/infrastructure/proxy_clients.rs | 83 +++++++++++ px-native/src/infrastructure/sensor_solver.rs | 17 ++- px-native/tests/throughput_soak.rs | 28 +++- px-perimeterx/src/application/solve_px.rs | 5 +- px-perimeterx/src/infrastructure/handler.rs | 9 +- px-pipeline/src/application/run_pipeline.rs | 8 +- px-pipeline/src/domain/challenge_handler.rs | 3 +- px-pipeline/src/domain/mod.rs | 1 + px-pipeline/src/domain/solve_action.rs | 52 +++++++ px-pipeline/src/lib.rs | 1 + px-pipeline/tests/pipeline.rs | 22 ++- px-server/src/application/routing.rs | 102 +------------- px-server/src/application/solve_endpoint.rs | 35 +++-- .../src/infrastructure/http/handlers/solve.rs | 6 +- px-server/tests/cache.rs | 72 ++++++++++ px-server/tests/common/mod.rs | 5 +- px-server/tests/routing.rs | 130 ++++++++++++++++++ px-turnstile/src/lib.rs | 4 +- 42 files changed, 832 insertions(+), 205 deletions(-) create mode 100644 docs/adr/0025-egress-proxy-propagation-contract.md create mode 100644 px-harvester/src/infrastructure/egress.rs create mode 100644 px-native/src/infrastructure/proxy_clients.rs create mode 100644 px-pipeline/src/domain/solve_action.rs create mode 100644 px-server/tests/routing.rs diff --git a/README.md b/README.md index 6a096f0..b21aa77 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A Rust-built solver service for PerimeterX (HUMAN Security) protection. Given a target URL on a per-domain allowlist, returns a valid `_px3` cookie bundle that a downstream authorized client can use to issue requests as if from a real browser. -> **Status:** v1.2.0 published to crates.io as the `pxsolver-*` family of crates. MVP gate hit at v1.0.0; v1.1.0 added a Camoufox-backed Cloudflare bypass path; v1.2.0 renamed the published library crates. See [GitHub Releases](https://github.com/KeyCode17/px-solver/releases) for the per-version notes. +> **Status:** published to crates.io as the `pxsolver-*` family of crates. MVP gate hit at v1.0.0; v1.1.0 added a Camoufox-backed Cloudflare bypass path; v1.2.0 renamed the published library crates; v1.8.0 activated native `_px3` sensor synthesis ([ADR-0024](docs/adr/0024-activate-native-px3-sensor-synthesis.md)); v2.0.0 makes the per-request egress proxy real end to end ([ADR-0025](docs/adr/0025-egress-proxy-propagation-contract.md)) — a breaking `ChallengeHandler` / `SolveDispatcher` signature change. See [GitHub Releases](https://github.com/KeyCode17/px-solver/releases) for the per-version notes. ## What this is @@ -65,6 +65,8 @@ The 16 `pxsolver-*` library crates are also published individually for downstrea -d '{"url":"https://www.pedidosya.com.ar/","proxy":null}' ``` + `"proxy"` is the egress the solve harvests through — `scheme://host:port` for `http`, `https`, `socks5` or `socks5h`, or `null` for the server's own IP. The returned `_px3` bundle is bound to that IP, so send downstream requests through the same proxy. `PX_PROXIES` is a separate, `/v1/fetch`-only rotation, and browser proxies cannot carry credentials — see [Egress proxies](docs/deployment.md#egress-proxies). + Response shape: ```json @@ -90,7 +92,7 @@ For systemd, reverse proxy, and key rotation workflows see [`docs/deployment.md` | [`docs/000-sow-index.md`](docs/000-sow-index.md) | Statement of Work index + deliverable traceability | | [`docs/adr/README.md`](docs/adr/README.md) | Architecture Decision Records (23 ADRs as of 2026-05-17) | | [`docs/phase/README.md`](docs/phase/README.md) | Phase plan (00–04 critical path + R research) | -| [`docs/deployment.md`](docs/deployment.md) | Fresh-Linux install, systemd, reverse proxy, key generation, allowlist editing | +| [`docs/deployment.md`](docs/deployment.md) | Fresh-Linux install, systemd, reverse proxy, key generation, allowlist editing, egress proxies | | [`docs/threat-model.md`](docs/threat-model.md) | Misuse vectors + mitigations | | [`docs/dual-use-policy.md`](docs/dual-use-policy.md) | Operator commitments per [`docs/011-sow-dual-use.md`](docs/011-sow-dual-use.md) | | [`docs/standards/axum-best-practice.md`](docs/standards/axum-best-practice.md) | Coding standard (Clean Architecture, ≤200 LOC/file, no `unwrap`) | diff --git a/docs/adr/0025-egress-proxy-propagation-contract.md b/docs/adr/0025-egress-proxy-propagation-contract.md new file mode 100644 index 0000000..b771213 --- /dev/null +++ b/docs/adr/0025-egress-proxy-propagation-contract.md @@ -0,0 +1,76 @@ +# 0025. Egress proxy propagation: per-request for solve, session rotation for fetch + +- **Date:** 2026-08-07 +- **Status:** Accepted +- **Deciders:** KeyCode17 +- **Related:** ADR-0014 (challenge pipeline), ADR-0020 (Camoufox), ADR-0021 (handler routing), + ADR-0024 (native sensor), [`docs/deployment.md`](../deployment.md) + +## Context + +Four proxy surfaces shipped through v1.8.0 and only one reached a browser: + +| Surface | Status before this ADR | +|---|---| +| `PX_PROXIES` → `ProxyPool` → `SessionPool` | worked, `/v1/fetch` sessions only | +| `POST /v1/solve` body `"proxy"` | deserialized into `SolveRequestDto`, never read | +| `px-cli solve --proxy` | sent in the body, dropped server-side | +| `pxsolver_core::SolveRequest::with_proxy()` | published builder with zero consumers | +| `HarvestRequest.proxy` | honoured by `CamoufoxPool`, ignored by `ChromiumoxidePool` | + +The cause was structural, not an oversight at one call site: `SolveDispatcher::solve(&self, url: &str)` +and `ChallengeHandler::solve(&self, page: &PageHtml)` had no parameter a proxy could travel in, and +both browser handlers built `HarvestRequest::new(url)`, which defaults `proxy` to `None`. Two +implementations of one `Harvester` trait also disagreed on whether the field meant anything. + +Downstream Rust users were the worst affected: `SolveRequest::with_proxy()` is the obvious API to +reach for, is published on crates.io, and did nothing. + +## Decision + +**1. The solve path carries a per-request proxy end to end.** `ChallengeHandler::solve` takes a +`SolveAction { page, proxy }` — a domain action struct built at the infrastructure edge and passed +inward. `SolveDispatcher::solve` takes `px_core::SolveRequest`, so the published builder is the type +the HTTP edge maps into. Both browser handlers forward it via `HarvestRequest::with_proxy`, and +`ChromiumoxidePool` gained the `--proxy-server` argument it never had. + +**2. The native sensor path honours it too.** `SolveContext` carries the proxy and +`SensorNativeSolver` posts through a per-proxy `reqwest::Client`, cached in `ProxyClients` so the +pooled TLS connection survives repeat solves. + +**3. Solving never falls back to the `PX_PROXIES` rotation.** `_px3` is bound to the IP that earned +it. A caller who did not name an egress cannot route downstream traffic through a rotating one, so a +bundle harvested that way would not be usable. Rotation stays where it is useful: long-lived +`/v1/fetch` Camoufox sessions, which own their cookie jar and consume the bundle themselves. + +**4. The egress is part of the cache key.** `sentinel_cache_key(domain, proxy)` hashes the proxy into +`fp_key`; a direct solve keeps `0`, so pre-existing keys still resolve. Without this, a bundle earned +through proxy A would be replayed to a caller who asked for proxy B. + +**5. Proxy credentials are stripped for browser paths, with a warning.** geckodriver's W3C `proxy` +capability has no credential field and Chromium's `--proxy-server` ignores userinfo without a CDP +`Fetch.authRequired` handler. `strip_credentials` (px-harvester) removes `user:pass@` and logs the +sanitized URL. Authenticated upstreams are used through a local unauthenticated relay. The native +path is exempt: `reqwest` implements proxy auth. + +## Consequences + +- **Breaking:** `ChallengeHandler::solve` and `SolveDispatcher::solve` change signature; every + handler crate (`px-perimeterx`, `px-cloudflare`, `px-native`, and the `px-turnstile` / + `px-captcha` / `px-datadome` stubs) is updated in the same change. Per ADR-0017 this is a + post-1.0.0 architectural change → manual `major` bump. +- `docs/deployment.md` gains an "Egress proxies" section correcting the old + `N × len(proxies)` rotation claim: a session takes its proxy at spawn and keeps it until the 300s + TTL, so distinct egress IPs per domain is `min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES))`. +- `/v1/fetch` deliberately gains **no** per-request proxy field: switching a warm session's egress + mid-life means respawning the browser and discarding the cookie jar it exists to hold. + +## Alternatives considered + +- **Delete the dead surfaces.** Smaller diff, but removes a published `pxsolver-core` API and two + documented operator-facing options — a bigger break for downstream users than wiring them up. +- **Rotate on solve as well, and report the chosen egress in the response.** Needs the resolved + proxy threaded back through `HarvestResult` → `HandlerOutcome` → `SolveOutput` → DTO. Deferred: + no operator has asked for solve-side rotation, and per-request assignment already covers it. +- **Per-request proxy on `/v1/fetch`, keying sessions by `(domain, proxy)`.** Rejected for now; it + multiplies live browsers per domain and complicates TTL eviction for a case the env list covers. diff --git a/docs/adr/README.md b/docs/adr/README.md index d988a72..cf08cd8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -37,6 +37,8 @@ Format: [MADR](https://adr.github.io/madr/) lite. One file per decision, never e | [0021](0021-domain-based-handler-routing.md) | Domain-based handler routing in the solve dispatcher (PX_CAMOUFOX_DOMAINS env CSV → CloudflareHandler) | Accepted | 2026-05-17 | | [0022](0022-readmit-pedidosya-to-canary-with-deep-stealth-budget.md) | Re-admit pedidosya to canary with relaxed AC-2 budget (15s median / 20s p95) for CF-fronted targets; amends ADR-0018 | Accepted | 2026-05-17 | | [0023](0023-allowlist-handler-field-supersedes-env-csv.md) | `handler:` field in allowlist.yaml supersedes `PX_CAMOUFOX_DOMAINS` env CSV; env retained as deprecated fallback through v1.x | Accepted | 2026-05-17 | +| [0024](0024-activate-native-px3-sensor-synthesis.md) | Activate native px-3 sensor synthesis; promote ADR-0010 | Proposed | 2026-05-20 | +| [0025](0025-egress-proxy-propagation-contract.md) | Egress proxy propagation: per-request for `/v1/solve`, session rotation for `/v1/fetch`; egress in the cache key; credentials stripped for browser paths | Accepted | 2026-08-07 | ## Template diff --git a/docs/deployment.md b/docs/deployment.md index e5457cf..b9dbc56 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -127,18 +127,74 @@ The optional `handler:` field (added in v1.1.x per [ADR-0023](adr/0023-allowlist Currently the server reads `PX_BIND`, `PX_KEYS`, `PX_ALLOWLIST` env vars. A YAML config file is reserved for future use. -### Egress proxy rotation (optional) +## Egress proxies -For sustained `/v1/fetch` traffic against rate-limiting WAFs (pedidosya's PerimeterX flags a single IP after ~30 fetches/min), set `PX_PROXIES` to a CSV list of proxy URLs. Each persistent Camoufox session for a CF-routed domain is assigned a proxy round-robin from the list: +There are **two** proxy mechanisms and they do different jobs. Pick by endpoint: + +| | `/v1/solve` | `/v1/fetch` | +|---|---|---| +| How you assign it | `"proxy"` in the request body (or `px-cli solve --proxy`) | `PX_PROXIES` env var, operator-side | +| Chosen per | request | Camoufox session | +| Rotation | none — the solve uses exactly the proxy you named | round-robin across the list | +| Omitted → | server's own IP | server's own IP | + +### `/v1/solve` — per-request proxy + +`_px3` is bound to the IP that earned it, so a bundle is only usable from that same egress. Name the proxy you intend to send downstream traffic through: ```bash -PX_PROXIES="http://user:pass@proxy1.example:8080,socks5://user:pass@proxy2.example:1080" \ +curl -X POST http://127.0.0.1:8080/v1/solve \ + -H "Authorization: Bearer ops1:" \ + -H "content-type: application/json" \ + -d '{"url":"https://www.pedidosya.com.ar/","proxy":"socks5://127.0.0.1:9050"}' +``` + +Accepted schemes: `http`, `https`, `socks5`, `socks5h`. `"proxy":null` (or omitting the field) harvests from the server's own address. + +The solve **never** falls back to the `PX_PROXIES` rotation — a bundle earned through an IP the caller cannot name would not be usable. The proxy is part of the cache key, so the same domain solved through two different proxies produces two entries and neither is served to the other. + +Rust callers build the same request through the published crate: + +```rust +use pxsolver_core::SolveRequest; + +let req = SolveRequest::new("https://www.pedidosya.com.ar/") + .with_proxy("socks5://127.0.0.1:9050"); +``` + +### `/v1/fetch` — session rotation via `PX_PROXIES` + +For sustained `/v1/fetch` traffic against rate-limiting WAFs (pedidosya's PerimeterX flags a single IP after ~30 fetches/min), set `PX_PROXIES` to a CSV list. Each persistent Camoufox session for a CF-routed domain is assigned one proxy round-robin at spawn: + +```bash +PX_PROXIES="http://proxy1.example:8080,socks5://proxy2.example:1080" \ ./target/release/px-server ``` -Empty / unset → direct connection (no rotation). Both `http://` and `socks5://` schemes are accepted by the underlying geckodriver capability. With `PX_FETCH_MAX_PER_DOMAIN=N` (default 2), the pool spawns up to N browsers per domain, each binding to the next proxy in the rotation; the operator's effective concurrency is `N × len(proxies)` parallel egress paths before round-robin reuse kicks in. +Empty / unset → direct connection (no rotation). `/v1/fetch` has no per-request proxy field: a session is reused for its warm cookie jar, and switching its egress mid-life would mean respawning the browser. + +**How many IPs you actually get.** A session takes a proxy when it is *spawned*, and keeps it until it ages out (300s TTL). Once a domain holds `PX_FETCH_MAX_PER_DOMAIN=N` sessions (default 2), further requests round-robin those existing sessions and no new proxy is drawn. Distinct egress IPs per domain is therefore: + +``` +min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES)) +``` + +Ten proxies with the default `N=2` gives a domain **two** IPs, not twenty. Raise `PX_FETCH_MAX_PER_DOMAIN` to use more of the list — each extra session is another live browser, so size it against RAM. The cursor is shared across domains, so proxy *k* is not reserved for any one target. + +### Proxy credentials + +**Browsers cannot authenticate to a proxy here.** geckodriver's W3C `proxy` capability has no credential field, and Chromium's `--proxy-server` ignores userinfo without a CDP `Fetch.authRequired` handler. `user:pass@` in a `PX_PROXIES` entry or in a `/v1/solve` proxy is **stripped**, and the server logs a warning naming the sanitized URL. + +To use an authenticated upstream, front it with a local unauthenticated relay and point px-solver at the relay: + +```bash +gost -L=socks5://127.0.0.1:1080 -F='http://user:pass@upstream.example:8080' +PX_PROXIES="socks5://127.0.0.1:1080" ./target/release/px-server +``` + +The one exception is the native sensor path (`PX_NATIVE_PROFILES`), which posts over `reqwest` and does support proxy authentication; a credentialed proxy works there and only there. -Tor as a quick test: install `tor`, let it bind `socks5://127.0.0.1:9050`, set `PX_PROXIES="socks5://127.0.0.1:9050"`. Many sites block Tor exit IPs; treat it as a fingerprint smoke-test rather than a production proxy. +Tor as a quick test: install `tor`, let it bind `socks5://127.0.0.1:9050`, set `PX_PROXIES="socks5://127.0.0.1:9050"` or pass it as a per-request proxy. Many sites block Tor exit IPs; treat it as a fingerprint smoke-test rather than a production proxy. ## Run diff --git a/docs/runbook-native-bypass.md b/docs/runbook-native-bypass.md index ee607ed..2d6263d 100644 --- a/docs/runbook-native-bypass.md +++ b/docs/runbook-native-bypass.md @@ -7,7 +7,12 @@ This is the end-to-end procedure for taking the native PX path from tenant". It assumes: - A working AR (or other tenant-appropriate) residential proxy is - available — set `PX_PROXIES=socks5://…` in your shell. + available — set `PX_PROXIES=socks5://…` in your shell. Steps 1 and 4 + read it directly (first CSV entry). The **server** does not apply it to + `/v1/solve`: there the egress is per request, named in the request body + (see [Egress proxies](deployment.md#egress-proxies)). Unlike the browser + paths, the native sensor POST goes through `reqwest` and does accept + `user:pass@` credentials. - Camoufox + geckodriver are installed and `CamoufoxConfig::from_env()` resolves them. - You have the `eT15wiaE` (pedidosya) profile at @@ -68,6 +73,16 @@ The dispatcher will try the native handler first for any solve targeting `pedidosya.com.ar` and fall back to the existing Camoufox path on error or non-solved status. +Name the egress per request — both the native sensor POST and the +Camoufox fallback use it, and the returned bundle is bound to that IP: + +```bash +curl -X POST http://127.0.0.1:8080/v1/solve \ + -H "Authorization: Bearer ops1:" \ + -H "content-type: application/json" \ + -d "{\"url\":\"https://www.pedidosya.com.ar/\",\"proxy\":\"$PX_PROXIES\"}" +``` + ## Step 4 — Throughput soak ```bash @@ -80,8 +95,10 @@ NATIVE_SOAK=1 \ cargo test -q -p pxsolver-native --test throughput_soak -- --ignored --nocapture ``` -The soak runs `SensorNativeSolver::solve` 80× through your live proxy -and asserts ≥40 req/min sustained throughput. Output: +The soak runs `SensorNativeSolver::solve` 80× and asserts ≥40 req/min +sustained throughput. It leaves through `NATIVE_SOAK_PROXY`, or the first +`PX_PROXIES` entry, or the host's own IP if neither is set — the run +prints which. Output: ``` === NATIVE_SOAK === diff --git a/px-camoufox/src/infrastructure/camoufox_pool.rs b/px-camoufox/src/infrastructure/camoufox_pool.rs index 9f20f5e..af83b6e 100644 --- a/px-camoufox/src/infrastructure/camoufox_pool.rs +++ b/px-camoufox/src/infrastructure/camoufox_pool.rs @@ -3,7 +3,7 @@ use crate::infrastructure::caps::{build_capabilities, pick_free_port, wait_for_g use async_trait::async_trait; use fantoccini::ClientBuilder; use px_errors::AppError; -use px_harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; +use px_harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester, strip_credentials}; use serde_json::{Map, Value}; use std::sync::Arc; use std::time::Duration; @@ -35,6 +35,8 @@ impl CamoufoxPool { if !proxies.is_empty() { tracing::info!( count = proxies.len(), + max_per_domain, + distinct_egress_per_domain = max_per_domain.min(proxies.len()), "proxy rotation enabled for /v1/fetch sessions" ); } @@ -101,7 +103,12 @@ impl CamoufoxPool { impl Harvester for CamoufoxPool { async fn harvest(&self, req: HarvestRequest) -> Result { let navigate_timeout = self.config.navigate_timeout; - let proxy = req.proxy.clone(); + let proxy = req.proxy.clone().map(strip_credentials); + tracing::info!( + url = %req.url, + proxy = proxy.as_deref().unwrap_or("direct"), + "camoufox harvest starting" + ); self.with_session(proxy.as_deref(), async move |endpoint, caps| { harvest_session(&endpoint, caps, &req, navigate_timeout).await }) diff --git a/px-camoufox/src/infrastructure/session_pool.rs b/px-camoufox/src/infrastructure/session_pool.rs index 7a9c314..f1efed4 100644 --- a/px-camoufox/src/infrastructure/session_pool.rs +++ b/px-camoufox/src/infrastructure/session_pool.rs @@ -91,14 +91,4 @@ impl SessionPool { let idx = slot.cursor.fetch_add(1, Ordering::Relaxed) % slot.sessions.len(); Ok(Arc::clone(&slot.sessions[idx])) } - - #[allow(dead_code)] - pub(crate) async fn total_sessions(&self) -> usize { - self.domains - .lock() - .await - .values() - .map(|slot| slot.sessions.len()) - .sum() - } } diff --git a/px-captcha/src/lib.rs b/px-captcha/src/lib.rs index 5292831..d1352b7 100644 --- a/px-captcha/src/lib.rs +++ b/px-captcha/src/lib.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml, SolveAction}; pub struct CaptchaHandler; @@ -30,7 +30,7 @@ impl ChallengeHandler for CaptchaHandler { || h.contains("class=\"g-recaptcha\"")) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented(self.name())) } } diff --git a/px-cli/Cargo.toml b/px-cli/Cargo.toml index 4118070..61e36ef 100644 --- a/px-cli/Cargo.toml +++ b/px-cli/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } argon2 = { workspace = true } clap = { workspace = true } px-auth = { workspace = true } +px-core = { workspace = true } px-detector = { workspace = true } px-native = { workspace = true } reqwest = { workspace = true } diff --git a/px-cli/src/cli.rs b/px-cli/src/cli.rs index ea02c32..652e66e 100644 --- a/px-cli/src/cli.rs +++ b/px-cli/src/cli.rs @@ -89,7 +89,10 @@ pub struct SolveArgs { /// API key as `id:secret`. #[arg(long, env = "PX_API_KEY")] pub api_key: String, - /// Optional upstream proxy passed to the solver. + /// Egress proxy the solver harvests through, as + /// `scheme://host:port` (http, https, socks5, socks5h). The returned + /// bundle is bound to that IP, so send downstream requests through the + /// same proxy. Omit to harvest from the server's own address. #[arg(long)] pub proxy: Option, } diff --git a/px-cli/src/commands/solve.rs b/px-cli/src/commands/solve.rs index cce7588..f626e78 100644 --- a/px-cli/src/commands/solve.rs +++ b/px-cli/src/commands/solve.rs @@ -1,15 +1,9 @@ use anyhow::{Context, Result, bail}; -use serde::{Deserialize, Serialize}; +use px_core::SolveRequest; +use serde::Deserialize; use crate::cli::SolveArgs; -#[derive(Debug, Serialize)] -struct SolveRequest<'a> { - url: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - proxy: Option<&'a str>, -} - #[derive(Debug, Deserialize)] struct SolveEnvelope { data: serde_json::Value, @@ -28,10 +22,7 @@ pub async fn run(args: SolveArgs) -> Result<()> { bail!("--api-key (or PX_API_KEY) must be in the form `id:secret`"); } let endpoint = format!("{}/v1/solve", server.trim_end_matches('/')); - let body = SolveRequest { - url: &url, - proxy: proxy.as_deref(), - }; + let body = SolveRequest::new(&url).with_proxy_opt(proxy); let client = reqwest::Client::builder() .build() .context("build http client")?; diff --git a/px-cloudflare/Cargo.toml b/px-cloudflare/Cargo.toml index b2bcf98..770a6f6 100644 --- a/px-cloudflare/Cargo.toml +++ b/px-cloudflare/Cargo.toml @@ -23,7 +23,7 @@ px-harvester = { workspace = true } px-pipeline = { workspace = true } [dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt", "sync"] } [lib] name = "px_cloudflare" diff --git a/px-cloudflare/src/lib.rs b/px-cloudflare/src/lib.rs index 73c3b71..61f6c31 100644 --- a/px-cloudflare/src/lib.rs +++ b/px-cloudflare/src/lib.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use px_core::{CookieJarDelta, NamedCookie}; use px_errors::AppError; use px_harvester::{HarvestRequest, Harvester}; -use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml, SolveAction}; use std::sync::Arc; use std::time::Instant; @@ -53,12 +53,13 @@ impl ChallengeHandler for CloudflareHandler { || h.contains("cf_clearance")) } - async fn solve(&self, page: &PageHtml) -> Result { + async fn solve(&self, action: &SolveAction) -> Result { let Some(harvester) = self.harvester.as_ref() else { return Ok(HandlerOutcome::not_implemented(self.name())); }; let start = Instant::now(); - let result = harvester.harvest(HarvestRequest::new(&page.url)).await?; + let request = HarvestRequest::new(action.url()).with_proxy(action.proxy.clone()); + let result = harvester.harvest(request).await?; let session_cookies: Vec = extract_session_cookies(&result.cookies) .into_iter() .map(|c| NamedCookie { @@ -92,16 +93,30 @@ impl ChallengeHandler for CloudflareHandler { mod tests { use super::*; use px_harvester::{HarvestResult, HarvestedCookie}; + use tokio::sync::Mutex; struct FakeHarvester { ua: String, cookies: Vec, html: String, + seen_proxy: Mutex>, + } + + impl FakeHarvester { + fn new(ua: &str, cookies: Vec, html: &str) -> Self { + Self { + ua: ua.into(), + cookies, + html: html.into(), + seen_proxy: Mutex::new(None), + } + } } #[async_trait] impl Harvester for FakeHarvester { - async fn harvest(&self, _req: HarvestRequest) -> Result { + async fn harvest(&self, req: HarvestRequest) -> Result { + *self.seen_proxy.lock().await = req.proxy.clone(); Ok(HarvestResult { html: self.html.clone(), user_agent: self.ua.clone(), @@ -122,27 +137,46 @@ mod tests { #[tokio::test] async fn solve_without_harvester_is_not_implemented() { let h = CloudflareHandler::new(); - let page = PageHtml::new("https://x.com", ""); - let oc = h.solve(&page).await.expect("solve"); + let action = SolveAction::new(PageHtml::new("https://x.com", "")); + let oc = h.solve(&action).await.expect("solve"); assert_eq!(oc.status, px_pipeline::HandlerStatus::NotImplemented); } + /// Regression: the request's proxy has to reach the harvester. It used + /// to be parsed at the edge and dropped before any browser saw it. + #[tokio::test] + async fn solve_forwards_the_requested_proxy_to_the_harvester() { + let fake = Arc::new(FakeHarvester::new( + "ua", + vec![cookie("cf_clearance")], + "page", + )); + let h = CloudflareHandler::with_harvester(Arc::clone(&fake) as Arc); + let action = SolveAction::new(PageHtml::new("https://x.com", "")) + .with_proxy(Some("socks5://127.0.0.1:9050".into())); + let _ = h.solve(&action).await.expect("solve"); + assert_eq!( + fake.seen_proxy.lock().await.as_deref(), + Some("socks5://127.0.0.1:9050") + ); + } + #[tokio::test] async fn solve_with_harvester_returns_session_cookies_and_ua() { - let fake = Arc::new(FakeHarvester { - ua: "Mozilla/5.0 Camoufox".into(), - cookies: vec![ + let fake = Arc::new(FakeHarvester::new( + "Mozilla/5.0 Camoufox", + vec![ cookie("cf_clearance"), cookie("__cf_bm"), cookie("_px3"), cookie("_pxhd"), cookie("unrelated_session"), ], - html: "real page".into(), - }); + "real page", + )); let h = CloudflareHandler::with_harvester(fake); - let page = PageHtml::new("https://x.com", ""); - let oc = h.solve(&page).await.expect("solve"); + let action = SolveAction::new(PageHtml::new("https://x.com", "")); + let oc = h.solve(&action).await.expect("solve"); assert_eq!(oc.status, px_pipeline::HandlerStatus::Solved); assert_eq!(oc.user_agent.as_deref(), Some("Mozilla/5.0 Camoufox")); let names: Vec<&str> = oc.cookies.set.iter().map(|c| c.name.as_str()).collect(); diff --git a/px-core/src/solve_request.rs b/px-core/src/solve_request.rs index e910f47..d93f06e 100644 --- a/px-core/src/solve_request.rs +++ b/px-core/src/solve_request.rs @@ -17,11 +17,22 @@ impl SolveRequest { } } + /// Route this solve through one egress proxy, so the returned bundle is + /// bound to an IP the caller can reuse. `scheme://host:port`, where + /// scheme is `http`, `https`, `socks5` or `socks5h`. pub fn with_proxy(mut self, proxy: impl Into) -> Self { self.proxy = Some(proxy.into()); self } + /// [`Self::with_proxy`] for an already-optional value, so an edge that + /// deserializes a nullable `proxy` field forwards it without branching. + #[must_use] + pub fn with_proxy_opt(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } + pub fn with_fingerprint(mut self, fingerprint: Fingerprint) -> Self { self.fingerprint = Some(fingerprint); self diff --git a/px-datadome/src/lib.rs b/px-datadome/src/lib.rs index e14c435..82e0f2e 100644 --- a/px-datadome/src/lib.rs +++ b/px-datadome/src/lib.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml, SolveAction}; pub struct DataDomeHandler; @@ -29,7 +29,7 @@ impl ChallengeHandler for DataDomeHandler { || h.contains("ddg_datadome")) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented(self.name())) } } diff --git a/px-harvester/src/domain/harvester.rs b/px-harvester/src/domain/harvester.rs index bff202a..c791243 100644 --- a/px-harvester/src/domain/harvester.rs +++ b/px-harvester/src/domain/harvester.rs @@ -19,6 +19,14 @@ impl HarvestRequest { wait_ms: 2_500, } } + + /// Pin this harvest to one egress proxy. `None` leaves the choice to + /// the [`crate::ProxyPool`] the implementation was built with. + #[must_use] + pub fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } } #[derive(Debug, Clone)] diff --git a/px-harvester/src/infrastructure/chromiumoxide_pool.rs b/px-harvester/src/infrastructure/chromiumoxide_pool.rs index 7a19e68..22b1f35 100644 --- a/px-harvester/src/infrastructure/chromiumoxide_pool.rs +++ b/px-harvester/src/infrastructure/chromiumoxide_pool.rs @@ -1,5 +1,6 @@ use crate::domain::harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; use crate::domain::stealth::{StealthBundle, default_stealth_bundle}; +use crate::infrastructure::egress::strip_credentials; use async_trait::async_trait; use chromiumoxide::browser::{Browser, BrowserConfig}; use chromiumoxide::page::Page; @@ -47,11 +48,17 @@ impl ChromiumoxidePool { self } - async fn launch_browser(&self) -> Result<(Browser, tokio::task::JoinHandle<()>), AppError> { + async fn launch_browser( + &self, + proxy: Option<&str>, + ) -> Result<(Browser, tokio::task::JoinHandle<()>), AppError> { let mut cfg = BrowserConfig::builder(); if !self.config.headless { cfg = cfg.with_head(); } + if let Some(proxy_url) = proxy { + cfg = cfg.arg(("proxy-server", proxy_url)); + } let cfg = cfg .build() .map_err(|e| AppError::InternalError(format!("BrowserConfig build: {e}")))?; @@ -93,7 +100,13 @@ impl Harvester for ChromiumoxidePool { .acquire() .await .map_err(|e| AppError::InternalError(format!("semaphore: {e}")))?; - let (mut browser, _handle) = self.launch_browser().await?; + let proxy = req.proxy.clone().map(strip_credentials); + tracing::info!( + url = %req.url, + proxy = proxy.as_deref().unwrap_or("direct"), + "chromium harvest starting" + ); + let (mut browser, _handle) = self.launch_browser(proxy.as_deref()).await?; let page = browser .new_page("about:blank") .await diff --git a/px-harvester/src/infrastructure/egress.rs b/px-harvester/src/infrastructure/egress.rs new file mode 100644 index 0000000..525d146 --- /dev/null +++ b/px-harvester/src/infrastructure/egress.rs @@ -0,0 +1,58 @@ +//! Egress-proxy handling shared by every [`crate::Harvester`] implementation. +//! +//! A harvest leaves through the proxy named on its [`crate::HarvestRequest`] +//! and no other. Rotation across a pool belongs to long-lived browser +//! sessions, not to solving: a `_px3` bundle is bound to the IP that earned +//! it, so a caller who did not name an egress could not use a bundle +//! harvested through a rotating one. + +/// Drop `user:pass@` from a proxy URL, warning that the credentials are +/// being discarded. +/// +/// Neither browser engine can answer a proxy `407` from userinfo in the +/// URL: geckodriver's W3C `proxy` capability has no credential field, and +/// Chromium's `--proxy-server` ignores them without a CDP +/// `Fetch.authRequired` handler. Passing them through anyway fails the +/// whole harvest with no visible cause, so an authenticated upstream has +/// to be fronted by a local unauthenticated relay (gost, 3proxy). +pub fn strip_credentials(proxy: String) -> String { + let Some((scheme, rest)) = proxy.split_once("://") else { + return proxy; + }; + let Some((_userinfo, host_port)) = rest.rsplit_once('@') else { + return proxy; + }; + let sanitized = format!("{scheme}://{host_port}"); + tracing::warn!( + proxy = %sanitized, + "proxy credentials discarded: no browser engine can authenticate them; front the upstream with a local unauthenticated relay" + ); + sanitized +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn drops_userinfo_from_http_and_socks_urls() { + assert_eq!( + strip_credentials("http://u:p@rotating.example:8080".into()), + "http://rotating.example:8080" + ); + assert_eq!( + strip_credentials("socks5://a:b@x.example:1080".into()), + "socks5://x.example:1080" + ); + } + + #[test] + fn leaves_unauthenticated_urls_alone() { + assert_eq!( + strip_credentials("http://plain.example:8080".into()), + "http://plain.example:8080" + ); + assert_eq!(strip_credentials("host:8080".into()), "host:8080"); + } +} diff --git a/px-harvester/src/infrastructure/mod.rs b/px-harvester/src/infrastructure/mod.rs index 7809962..3b744ea 100644 --- a/px-harvester/src/infrastructure/mod.rs +++ b/px-harvester/src/infrastructure/mod.rs @@ -1,2 +1,3 @@ pub mod chromiumoxide_pool; +pub mod egress; pub mod stealth_bundle; diff --git a/px-harvester/src/lib.rs b/px-harvester/src/lib.rs index 06b2722..cb8e843 100644 --- a/px-harvester/src/lib.rs +++ b/px-harvester/src/lib.rs @@ -6,3 +6,4 @@ pub use application::harvest_page::HarvestPage; pub use domain::harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; pub use domain::stealth::{StealthBundle, default_stealth_bundle}; pub use infrastructure::chromiumoxide_pool::{ChromiumoxidePool, PoolConfig}; +pub use infrastructure::egress::strip_credentials; diff --git a/px-native/src/domain/native_solver.rs b/px-native/src/domain/native_solver.rs index 0e8917e..6a5ea94 100644 --- a/px-native/src/domain/native_solver.rs +++ b/px-native/src/domain/native_solver.rs @@ -7,6 +7,7 @@ pub struct SolveContext { pub url: String, pub app_id: PxAppId, pub fingerprint: Fingerprint, + pub proxy: Option, } impl SolveContext { @@ -15,8 +16,16 @@ impl SolveContext { url: url.into(), app_id, fingerprint, + proxy: None, } } + + /// Route this solve's sensor POST through one egress proxy. + #[must_use] + pub fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } } #[async_trait] diff --git a/px-native/src/infrastructure/handler.rs b/px-native/src/infrastructure/handler.rs index ccb9fb1..21f8653 100644 --- a/px-native/src/infrastructure/handler.rs +++ b/px-native/src/infrastructure/handler.rs @@ -9,7 +9,9 @@ use std::time::Instant; use async_trait::async_trait; use px_core::{CookieJarDelta, Fingerprint, PxAppId}; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerName, HandlerOutcome, PageHtml}; +use px_pipeline::{ + ChallengeHandler, HandlerMetrics, HandlerName, HandlerOutcome, PageHtml, SolveAction, +}; use crate::domain::native_solver::{NativeSolver, SolveContext}; @@ -39,9 +41,10 @@ impl ChallengeHandler for NativePxHandler { Ok(true) } - async fn solve(&self, page: &PageHtml) -> Result { + async fn solve(&self, action: &SolveAction) -> Result { let started = Instant::now(); - let ctx = SolveContext::new(page.url.clone(), self.app_id.clone(), default_fingerprint()); + let ctx = SolveContext::new(action.url(), self.app_id.clone(), default_fingerprint()) + .with_proxy(action.proxy.clone()); let bundle = self.solver.solve(&ctx).await?; let metrics = HandlerMetrics { solve_ms: started.elapsed().as_millis() as u64, @@ -81,12 +84,17 @@ mod tests { use px_core::{NamedCookie, PxCookieBundle}; use px_pipeline::HandlerStatus; use std::time::{Duration, SystemTime}; + use tokio::sync::Mutex; - struct AlwaysOkSolver; + #[derive(Default)] + struct AlwaysOkSolver { + seen_proxy: Mutex>, + } #[async_trait] impl NativeSolver for AlwaysOkSolver { - async fn solve(&self, _ctx: &SolveContext) -> Result { + async fn solve(&self, ctx: &SolveContext) -> Result { + *self.seen_proxy.lock().await = ctx.proxy.clone(); Ok(PxCookieBundle::new( vec![NamedCookie { name: "_px3".into(), @@ -107,12 +115,29 @@ mod tests { #[tokio::test] async fn handler_reports_solved_status() { - let handler = - NativePxHandler::new(Arc::new(AlwaysOkSolver) as Arc, app_id()); - let page = PageHtml::new("https://www.pedidosya.com.ar/", ""); - let out = handler.solve(&page).await.expect("solve"); + let handler = NativePxHandler::new( + Arc::new(AlwaysOkSolver::default()) as Arc, + app_id(), + ); + let action = SolveAction::new(PageHtml::new("https://www.pedidosya.com.ar/", "")); + let out = handler.solve(&action).await.expect("solve"); assert_eq!(out.status, HandlerStatus::Solved); assert_eq!(out.cookies.set.len(), 1); assert_eq!(out.user_agent.as_deref(), Some("ua")); } + + /// The native path must carry the request's proxy into the sensor POST; + /// dropping it here would send the payload from the server's own IP. + #[tokio::test] + async fn handler_forwards_the_requested_proxy_to_the_solver() { + let solver = Arc::new(AlwaysOkSolver::default()); + let handler = NativePxHandler::new(Arc::clone(&solver) as Arc, app_id()); + let action = SolveAction::new(PageHtml::new("https://www.pedidosya.com.ar/", "")) + .with_proxy(Some("http://egress:8080".into())); + let _ = handler.solve(&action).await.expect("solve"); + assert_eq!( + solver.seen_proxy.lock().await.as_deref(), + Some("http://egress:8080") + ); + } } diff --git a/px-native/src/infrastructure/mod.rs b/px-native/src/infrastructure/mod.rs index e03dea1..0306bc9 100644 --- a/px-native/src/infrastructure/mod.rs +++ b/px-native/src/infrastructure/mod.rs @@ -2,6 +2,7 @@ pub mod cookies; pub mod handler; pub mod native_first; pub mod not_implemented; +pub mod proxy_clients; pub mod sensor_solver; pub use handler::NativePxHandler; diff --git a/px-native/src/infrastructure/native_first.rs b/px-native/src/infrastructure/native_first.rs index 3280019..a978219 100644 --- a/px-native/src/infrastructure/native_first.rs +++ b/px-native/src/infrastructure/native_first.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerName, HandlerOutcome, HandlerStatus, PageHtml}; +use px_pipeline::{ + ChallengeHandler, HandlerName, HandlerOutcome, HandlerStatus, PageHtml, SolveAction, +}; pub struct NativeFirstHandler { native: Arc, @@ -33,8 +35,8 @@ impl ChallengeHandler for NativeFirstHandler { self.fallback.detects(page).await } - async fn solve(&self, page: &PageHtml) -> Result { - match self.native.solve(page).await { + async fn solve(&self, action: &SolveAction) -> Result { + match self.native.solve(action).await { Ok(out) if matches!(out.status, HandlerStatus::Solved) => Ok(out), Ok(out) => { tracing::info!( @@ -42,7 +44,7 @@ impl ChallengeHandler for NativeFirstHandler { status = ?out.status, "native handler not solved, falling back" ); - self.fallback.solve(page).await + self.fallback.solve(action).await } Err(e) => { tracing::warn!( @@ -50,7 +52,7 @@ impl ChallengeHandler for NativeFirstHandler { error = %e, "native handler error, falling back" ); - self.fallback.solve(page).await + self.fallback.solve(action).await } } } @@ -74,7 +76,7 @@ mod tests { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::solved_with_ua( self.0, CookieJarDelta::default(), @@ -93,7 +95,7 @@ mod tests { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Err(AppError::InternalError("synthetic".into())) } } @@ -105,7 +107,7 @@ mod tests { Arc::new(SolvedHandler("fallback")), ); let out = h - .solve(&PageHtml::new("https://x/", "")) + .solve(&SolveAction::new(PageHtml::new("https://x/", ""))) .await .expect("solve"); assert_eq!(out.handler, "native"); @@ -115,7 +117,7 @@ mod tests { async fn falls_back_on_error() { let h = NativeFirstHandler::new(Arc::new(FailingHandler), Arc::new(SolvedHandler("fb"))); let out = h - .solve(&PageHtml::new("https://x/", "")) + .solve(&SolveAction::new(PageHtml::new("https://x/", ""))) .await .expect("solve"); assert_eq!(out.handler, "fb"); diff --git a/px-native/src/infrastructure/proxy_clients.rs b/px-native/src/infrastructure/proxy_clients.rs new file mode 100644 index 0000000..dd6e3cd --- /dev/null +++ b/px-native/src/infrastructure/proxy_clients.rs @@ -0,0 +1,83 @@ +//! Per-proxy `reqwest` clients for the native sensor path. +//! +//! A `Client` owns its connection pool, and the proxy is fixed at build +//! time, so honouring a per-request egress means one client per proxy. +//! They are cached because rebuilding one per solve would throw away the +//! pooled TLS connection to the sensor endpoint. +//! +//! Unlike the browser paths, `reqwest` does implement proxy +//! authentication, so `user:pass@` in the URL is honoured here. + +use std::collections::HashMap; +use std::sync::Mutex; + +use px_errors::AppError; +use reqwest::{Client, Proxy}; + +pub struct ProxyClients { + direct: Client, + proxied: Mutex>, +} + +impl ProxyClients { + pub fn new(direct: Client) -> Self { + Self { + direct, + proxied: Mutex::new(HashMap::new()), + } + } + + /// Client whose egress is `proxy`, or the direct one when `None`. + pub fn for_proxy(&self, proxy: Option<&str>) -> Result { + let Some(proxy) = proxy else { + return Ok(self.direct.clone()); + }; + let mut cache = self + .proxied + .lock() + .map_err(|e| AppError::InternalError(format!("proxy client cache poisoned: {e}")))?; + if let Some(client) = cache.get(proxy) { + return Ok(client.clone()); + } + let built = Client::builder() + .proxy( + Proxy::all(proxy) + .map_err(|e| AppError::BadRequest(format!("invalid proxy url: {e}")))?, + ) + .build() + .map_err(|e| AppError::InternalError(format!("build proxied client: {e}")))?; + cache.insert(proxy.to_string(), built.clone()); + Ok(built) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + fn clients() -> ProxyClients { + ProxyClients::new(Client::builder().build().expect("build direct client")) + } + + #[test] + fn no_proxy_yields_the_direct_client() { + assert!(clients().for_proxy(None).is_ok()); + } + + #[test] + fn a_proxy_url_builds_and_caches_one_client() { + let clients = clients(); + assert!(clients.for_proxy(Some("http://127.0.0.1:8080")).is_ok()); + assert!(clients.for_proxy(Some("http://127.0.0.1:8080")).is_ok()); + assert_eq!(clients.proxied.lock().expect("cache not poisoned").len(), 1); + } + + #[test] + fn a_malformed_proxy_url_is_a_bad_request() { + let err = clients() + .for_proxy(Some("not a url")) + .expect_err("malformed proxy must fail"); + assert!(matches!(err, AppError::BadRequest(_))); + } +} diff --git a/px-native/src/infrastructure/sensor_solver.rs b/px-native/src/infrastructure/sensor_solver.rs index 73fb596..098bd74 100644 --- a/px-native/src/infrastructure/sensor_solver.rs +++ b/px-native/src/infrastructure/sensor_solver.rs @@ -18,11 +18,12 @@ use crate::cipher::encrypt_sensor; use crate::domain::native_solver::{NativeSolver, SolveContext}; use crate::events::{SyntheticIdentity, default_batch}; use crate::infrastructure::cookies::parse_set_cookies; +use crate::infrastructure::proxy_clients::ProxyClients; use crate::profile::TenantProfile; /// HTTP-backed native solver bound to a single tenant profile. pub struct SensorNativeSolver { - client: Client, + clients: ProxyClients, profile: Arc, cookie_ttl: Duration, } @@ -30,7 +31,7 @@ pub struct SensorNativeSolver { impl SensorNativeSolver { pub fn new(client: Client, profile: Arc) -> Self { Self { - client, + clients: ProxyClients::new(client), profile, cookie_ttl: Duration::from_secs(300), } @@ -71,9 +72,9 @@ impl NativeSolver for SensorNativeSolver { ); let sensor_url = self.profile.sensor_url(&origin); let payload = self.build_payload(ctx)?; + let client = self.clients.for_proxy(ctx.proxy.as_deref())?; - let resp = self - .client + let resp = client .post(&sensor_url) .header(USER_AGENT, header(&ctx.fingerprint.user_agent)?) .header(ACCEPT, HeaderValue::from_static("*/*")) @@ -108,7 +109,13 @@ impl NativeSolver for SensorNativeSolver { "sensor POST returned no Set-Cookie headers".into(), )); } - tracing::info!(target: "px_native", url = %sensor_url, count = cookies.len(), "native sensor solved"); + tracing::info!( + target: "px_native", + url = %sensor_url, + count = cookies.len(), + proxy = ctx.proxy.as_deref().unwrap_or("direct"), + "native sensor solved" + ); Ok(PxCookieBundle::new( cookies, ctx.fingerprint.user_agent.clone(), diff --git a/px-native/tests/throughput_soak.rs b/px-native/tests/throughput_soak.rs index 6bb2e00..1242e67 100644 --- a/px-native/tests/throughput_soak.rs +++ b/px-native/tests/throughput_soak.rs @@ -19,7 +19,11 @@ //! [NATIVE_SOAK_CONCURRENCY=8] \ //! [NATIVE_SOAK_TARGET_RPM=40] \ //! [NATIVE_SOAK_PROFILE=px-native/profiles/eT15wiaE.toml] \ +//! [NATIVE_SOAK_PROXY=socks5://... | PX_PROXIES=socks5://...] \ //! cargo test -p pxsolver-native --test throughput_soak -- --ignored --nocapture +//! +//! Without a proxy the soak leaves through the host's own IP, which a +//! rate-limiting tenant will flag long before the target rpm is reached. use std::path::PathBuf; use std::sync::Arc; @@ -61,9 +65,14 @@ async fn native_throughput_soak() { let client = Client::builder().build().expect("client"); let solver: Arc = Arc::new(SensorNativeSolver::new(client, Arc::new(profile))); - let ctx_template = SolveContext::new(url.clone(), app_id.clone(), soak_fingerprint()); + let proxy = soak_proxy(); + let ctx_template = SolveContext::new(url.clone(), app_id.clone(), soak_fingerprint()) + .with_proxy(proxy.clone()); - eprintln!("soak: n={n} concurrency={concurrency} target_rpm={target_rpm} url={url}"); + eprintln!( + "soak: n={n} concurrency={concurrency} target_rpm={target_rpm} url={url} proxy={}", + proxy.as_deref().unwrap_or("direct") + ); let mut latencies_ms: Vec = Vec::with_capacity(n); let mut ok_count: usize = 0; @@ -113,6 +122,21 @@ async fn native_throughput_soak() { ); } +/// Egress for the soak: `NATIVE_SOAK_PROXY`, else the first `PX_PROXIES` +/// entry so an operator who already exported the list for the capture +/// step does not have to restate it. +fn soak_proxy() -> Option { + std::env::var("NATIVE_SOAK_PROXY") + .ok() + .or_else(|| std::env::var("PX_PROXIES").ok()) + .and_then(|raw| { + raw.split(',') + .map(str::trim) + .find(|entry| !entry.is_empty()) + .map(str::to_string) + }) +} + fn soak_fingerprint() -> Fingerprint { Fingerprint { user_agent: "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0".into(), diff --git a/px-perimeterx/src/application/solve_px.rs b/px-perimeterx/src/application/solve_px.rs index f0eeef6..efeb053 100644 --- a/px-perimeterx/src/application/solve_px.rs +++ b/px-perimeterx/src/application/solve_px.rs @@ -20,9 +20,10 @@ impl SolvePx { Self { harvester } } - pub async fn execute(&self, url: &str) -> Result { + pub async fn execute(&self, url: &str, proxy: Option) -> Result { let start = Instant::now(); - let result = self.harvester.harvest(HarvestRequest::new(url)).await?; + let request = HarvestRequest::new(url).with_proxy(proxy); + let result = self.harvester.harvest(request).await?; let px_cookies: Vec = extract_px_cookies(&result.cookies) .into_iter() .map(|c| NamedCookie { diff --git a/px-perimeterx/src/infrastructure/handler.rs b/px-perimeterx/src/infrastructure/handler.rs index e534da6..749dced 100644 --- a/px-perimeterx/src/infrastructure/handler.rs +++ b/px-perimeterx/src/infrastructure/handler.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use px_detector::{Detected, Detector, RegexDetector}; use px_errors::AppError; use px_harvester::Harvester; -use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml, SolveAction}; use std::sync::Arc; pub struct PerimeterxHandler { @@ -30,8 +30,11 @@ impl ChallengeHandler for PerimeterxHandler { Ok(matches!(self.detector.detect(&page.html), Detected::Yes(_))) } - async fn solve(&self, page: &PageHtml) -> Result { - let out = self.solver.execute(&page.url).await?; + async fn solve(&self, action: &SolveAction) -> Result { + let out = self + .solver + .execute(action.url(), action.proxy.clone()) + .await?; let metrics = HandlerMetrics { detect_us: 0, solve_ms: out.solve_ms, diff --git a/px-pipeline/src/application/run_pipeline.rs b/px-pipeline/src/application/run_pipeline.rs index 34a4d74..48217d6 100644 --- a/px-pipeline/src/application/run_pipeline.rs +++ b/px-pipeline/src/application/run_pipeline.rs @@ -1,6 +1,6 @@ use crate::domain::challenge_handler::ChallengeHandler; use crate::domain::handler_outcome::HandlerOutcome; -use crate::domain::page_html::PageHtml; +use crate::domain::solve_action::SolveAction; use px_errors::AppError; use std::sync::Arc; @@ -27,11 +27,11 @@ impl Pipeline { self.handlers.len() } - pub async fn run(&self, page: &PageHtml) -> Result, AppError> { + pub async fn run(&self, action: &SolveAction) -> Result, AppError> { let mut outcomes = Vec::with_capacity(self.handlers.len()); for handler in &self.handlers { - if handler.detects(page).await? { - let outcome = handler.solve(page).await?; + if handler.detects(&action.page).await? { + let outcome = handler.solve(action).await?; let solved = matches!( outcome.status, crate::domain::handler_outcome::HandlerStatus::Solved diff --git a/px-pipeline/src/domain/challenge_handler.rs b/px-pipeline/src/domain/challenge_handler.rs index 22f7e3e..8e80aae 100644 --- a/px-pipeline/src/domain/challenge_handler.rs +++ b/px-pipeline/src/domain/challenge_handler.rs @@ -1,5 +1,6 @@ use crate::domain::handler_outcome::HandlerOutcome; use crate::domain::page_html::PageHtml; +use crate::domain::solve_action::SolveAction; use async_trait::async_trait; use px_errors::AppError; @@ -9,5 +10,5 @@ pub use crate::domain::handler_outcome::HandlerName; pub trait ChallengeHandler: Send + Sync { fn name(&self) -> HandlerName; async fn detects(&self, page: &PageHtml) -> Result; - async fn solve(&self, page: &PageHtml) -> Result; + async fn solve(&self, action: &SolveAction) -> Result; } diff --git a/px-pipeline/src/domain/mod.rs b/px-pipeline/src/domain/mod.rs index 956938c..ac3817f 100644 --- a/px-pipeline/src/domain/mod.rs +++ b/px-pipeline/src/domain/mod.rs @@ -2,3 +2,4 @@ pub mod challenge_handler; pub mod fetcher; pub mod handler_outcome; pub mod page_html; +pub mod solve_action; diff --git a/px-pipeline/src/domain/solve_action.rs b/px-pipeline/src/domain/solve_action.rs new file mode 100644 index 0000000..124046f --- /dev/null +++ b/px-pipeline/src/domain/solve_action.rs @@ -0,0 +1,52 @@ +//! What a [`crate::ChallengeHandler`] is asked to solve. +//! +//! Detection only ever needs the page, but solving also needs the egress +//! the operator picked for this request, so the action carries both. It is +//! built at the infrastructure edge (the HTTP handler) and passed inward; +//! handlers never read transport types. + +use crate::domain::page_html::PageHtml; + +#[derive(Debug, Clone)] +pub struct SolveAction { + pub page: PageHtml, + pub proxy: Option, +} + +impl SolveAction { + pub fn new(page: PageHtml) -> Self { + Self { page, proxy: None } + } + + /// Pin this solve to one egress proxy. `None` leaves the choice to the + /// harvester's rotation. + #[must_use] + pub fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } + + pub fn url(&self) -> &str { + &self.page.url + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn defaults_to_no_proxy() { + let action = SolveAction::new(PageHtml::new("https://example.com", "")); + assert_eq!(action.url(), "https://example.com"); + assert!(action.proxy.is_none()); + } + + #[test] + fn carries_the_requested_proxy() { + let action = SolveAction::new(PageHtml::new("https://example.com", "")) + .with_proxy(Some("socks5://127.0.0.1:9050".into())); + assert_eq!(action.proxy.as_deref(), Some("socks5://127.0.0.1:9050")); + } +} diff --git a/px-pipeline/src/lib.rs b/px-pipeline/src/lib.rs index 4d1f8e6..c7ecdb9 100644 --- a/px-pipeline/src/lib.rs +++ b/px-pipeline/src/lib.rs @@ -6,3 +6,4 @@ pub use domain::challenge_handler::{ChallengeHandler, HandlerName}; pub use domain::fetcher::{FetchRequest, FetchResponse, Fetcher}; pub use domain::handler_outcome::{HandlerMetrics, HandlerOutcome, HandlerStatus}; pub use domain::page_html::PageHtml; +pub use domain::solve_action::SolveAction; diff --git a/px-pipeline/tests/pipeline.rs b/px-pipeline/tests/pipeline.rs index 3ae9e95..8bffabf 100644 --- a/px-pipeline/tests/pipeline.rs +++ b/px-pipeline/tests/pipeline.rs @@ -5,6 +5,7 @@ use px_core::{CookieJarDelta, NamedToken}; use px_errors::AppError; use px_pipeline::{ ChallengeHandler, HandlerMetrics, HandlerOutcome, HandlerStatus, PageHtml, Pipeline, + SolveAction, }; use std::sync::Arc; @@ -18,7 +19,7 @@ impl ChallengeHandler for SkippingHandler { async fn detects(&self, _page: &PageHtml) -> Result { Ok(false) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Err(AppError::InternalError( "should not be called when detects=false".into(), )) @@ -35,7 +36,7 @@ impl ChallengeHandler for SolvingHandler { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::solved( "solve", CookieJarDelta::default(), @@ -55,7 +56,7 @@ impl ChallengeHandler for StubHandler { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented("stub")) } } @@ -64,7 +65,10 @@ impl ChallengeHandler for StubHandler { async fn pipeline_skips_then_solves_and_stops() { let pipeline = Pipeline::new(vec![Arc::new(SkippingHandler), Arc::new(SolvingHandler)]); let page = PageHtml::new("https://x", ""); - let outcomes = pipeline.run(&page).await.expect("run ok"); + let outcomes = pipeline + .run(&SolveAction::new(page.clone())) + .await + .expect("run ok"); assert_eq!(outcomes.len(), 2); assert_eq!(outcomes[0].status, HandlerStatus::Skipped); assert_eq!(outcomes[1].status, HandlerStatus::Solved); @@ -75,7 +79,10 @@ async fn stop_on_solve_false_continues_past_solver() { let pipeline = Pipeline::new(vec![Arc::new(SolvingHandler), Arc::new(StubHandler)]) .with_stop_on_solve(false); let page = PageHtml::new("https://x", ""); - let outcomes = pipeline.run(&page).await.expect("run ok"); + let outcomes = pipeline + .run(&SolveAction::new(page.clone())) + .await + .expect("run ok"); assert_eq!(outcomes.len(), 2); assert_eq!(outcomes[0].status, HandlerStatus::Solved); assert_eq!(outcomes[1].status, HandlerStatus::NotImplemented); @@ -85,7 +92,10 @@ async fn stop_on_solve_false_continues_past_solver() { async fn first_solver_stops_pipeline() { let pipeline = Pipeline::new(vec![Arc::new(SolvingHandler), Arc::new(StubHandler)]); let page = PageHtml::new("https://x", ""); - let outcomes = pipeline.run(&page).await.expect("run ok"); + let outcomes = pipeline + .run(&SolveAction::new(page.clone())) + .await + .expect("run ok"); assert_eq!(outcomes.len(), 1); assert_eq!(outcomes[0].status, HandlerStatus::Solved); } diff --git a/px-server/src/application/routing.rs b/px-server/src/application/routing.rs index e75e605..d8d8026 100644 --- a/px-server/src/application/routing.rs +++ b/px-server/src/application/routing.rs @@ -11,9 +11,9 @@ use crate::application::solve_endpoint::{SolveDispatcher, SolveOutput, domain_from_url}; use async_trait::async_trait; -use px_core::PxCookieBundle; +use px_core::{PxCookieBundle, SolveRequest}; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml, SolveAction}; use std::collections::BTreeMap; use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -68,11 +68,11 @@ impl RoutingDispatcher { #[async_trait] impl SolveDispatcher for RoutingDispatcher { - async fn solve(&self, url: &str) -> Result { - let host = domain_from_url(url)?; + async fn solve(&self, req: SolveRequest) -> Result { + let host = domain_from_url(&req.url)?; let handler = self.resolve(&host); - let page = PageHtml::new(url, ""); - let outcome = handler.solve(&page).await?; + let action = SolveAction::new(PageHtml::new(&req.url, "")).with_proxy(req.proxy); + let outcome = handler.solve(&action).await?; if !matches!(outcome.status, HandlerStatus::Solved) { return Err(AppError::Conflict(format!( "{} returned status {:?}", @@ -106,93 +106,3 @@ pub fn parse_camoufox_domains(raw: Option<&str>) -> Vec { .filter(|s| !s.is_empty()) .collect() } - -#[cfg(test)] -#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] -mod tests { - use super::*; - use px_core::CookieJarDelta; - use px_pipeline::{HandlerMetrics, HandlerOutcome}; - - struct StaticHandler { - name: &'static str, - } - - #[async_trait] - impl ChallengeHandler for StaticHandler { - fn name(&self) -> &'static str { - self.name - } - async fn detects(&self, _page: &PageHtml) -> Result { - Ok(true) - } - async fn solve(&self, _page: &PageHtml) -> Result { - Ok(HandlerOutcome::solved_with_ua( - self.name, - CookieJarDelta::default(), - Vec::new(), - HandlerMetrics::default(), - "ua", - )) - } - } - - #[tokio::test] - async fn default_handler_used_when_no_match() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })).with_route( - "pedidosya.com.ar", - Arc::new(StaticHandler { name: "cloudflare" }), - ); - let out = d - .solve("https://www.havenwellwithin.com/") - .await - .expect("solve"); - assert_eq!(out.handler, "perimeterx"); - } - - #[tokio::test] - async fn exact_host_routes_to_match() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })).with_route( - "pedidosya.com.ar", - Arc::new(StaticHandler { name: "cloudflare" }), - ); - let out = d.solve("https://pedidosya.com.ar/").await.expect("solve"); - assert_eq!(out.handler, "cloudflare"); - } - - #[tokio::test] - async fn subdomain_routes_to_match() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })).with_route( - "pedidosya.com.ar", - Arc::new(StaticHandler { name: "cloudflare" }), - ); - let out = d - .solve("https://www.pedidosya.com.ar/x") - .await - .expect("solve"); - assert_eq!(out.handler, "cloudflare"); - } - - /// Regression: the cookie bundle's `user_agent` must carry the real - /// harvester UA (so cache-hit replies preserve it), not a literal - /// placeholder string. - #[tokio::test] - async fn bundle_user_agent_matches_harvester() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })); - let out = d.solve("https://example.com/").await.expect("solve"); - assert_eq!(out.user_agent, "ua"); - assert_eq!(out.bundle.user_agent, "ua"); - } - - #[test] - fn parse_csv_trims_and_lowercases() { - let r = parse_camoufox_domains(Some(" Pedidosya.com.AR , ,foo.com ")); - assert_eq!(r, vec!["pedidosya.com.ar", "foo.com"]); - } - - #[test] - fn parse_csv_empty_unset() { - assert!(parse_camoufox_domains(None).is_empty()); - assert!(parse_camoufox_domains(Some("")).is_empty()); - } -} diff --git a/px-server/src/application/solve_endpoint.rs b/px-server/src/application/solve_endpoint.rs index c12906a..adcb34b 100644 --- a/px-server/src/application/solve_endpoint.rs +++ b/px-server/src/application/solve_endpoint.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; -use px_core::{CacheKey, PxAppId, PxCookieBundle}; +use px_core::{CacheKey, PxAppId, PxCookieBundle, SolveRequest}; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml, SolveAction}; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::sync::Arc; use std::time::{Duration, SystemTime}; use url::Url; #[async_trait] pub trait SolveDispatcher: Send + Sync { - async fn solve(&self, url: &str) -> Result; + async fn solve(&self, req: SolveRequest) -> Result; } #[derive(Debug, Clone)] @@ -37,9 +39,9 @@ impl PxSolveDispatcher { #[async_trait] impl SolveDispatcher for PxSolveDispatcher { - async fn solve(&self, url: &str) -> Result { - let page = PageHtml::new(url, ""); - let outcome = self.handler.solve(&page).await?; + async fn solve(&self, req: SolveRequest) -> Result { + let action = SolveAction::new(PageHtml::new(&req.url, "")).with_proxy(req.proxy); + let outcome = self.handler.solve(&action).await?; if !matches!(outcome.status, HandlerStatus::Solved) { return Err(AppError::Conflict(format!( "{} returned status {:?}", @@ -75,8 +77,25 @@ pub fn cache_key_for(domain: &str, app_id: PxAppId, fp_key: u64) -> CacheKey { CacheKey::new(domain, app_id, fp_key) } -pub fn sentinel_cache_key(domain: &str) -> Result { +/// Cache key for one solve. The egress is part of the identity: PX binds a +/// `_px3` bundle to the IP that earned it, so replaying a proxy-A bundle +/// for a request that asked for proxy B (or for a direct route) hands the +/// caller cookies their traffic will not match. +pub fn sentinel_cache_key(domain: &str, proxy: Option<&str>) -> Result { let app_id = PxAppId::new("Unknown000") .map_err(|e| AppError::InternalError(format!("sentinel app_id invalid: {e}")))?; - Ok(CacheKey::new(domain, app_id, 0)) + Ok(CacheKey::new(domain, app_id, egress_key(proxy))) +} + +/// `0` for a direct route, so keys minted before proxies were honoured +/// keep resolving to the same entry. +fn egress_key(proxy: Option<&str>) -> u64 { + match proxy { + None => 0, + Some(proxy) => { + let mut hasher = DefaultHasher::new(); + proxy.hash(&mut hasher); + hasher.finish() + } + } } diff --git a/px-server/src/infrastructure/http/handlers/solve.rs b/px-server/src/infrastructure/http/handlers/solve.rs index 5c7387e..139b6b7 100644 --- a/px-server/src/infrastructure/http/handlers/solve.rs +++ b/px-server/src/infrastructure/http/handlers/solve.rs @@ -5,6 +5,7 @@ use axum::Json; use axum::extract::State; use axum::http::HeaderMap; use px_auth::{AuditEvent, AuditOutcome}; +use px_core::SolveRequest; use px_errors::AppError; use px_types::SingleResponse; use std::sync::atomic::Ordering; @@ -33,7 +34,8 @@ pub async fn handle( .fetch_add(1, Ordering::Relaxed); })?; state.metrics.solves_total.fetch_add(1, Ordering::Relaxed); - let cache_key = sentinel_cache_key(&domain)?; + let request = SolveRequest::new(&payload.url).with_proxy_opt(payload.proxy); + let cache_key = sentinel_cache_key(&domain, request.proxy.as_deref())?; let cached = state.cache.get(&cache_key).await?; let (out, cache_hit) = if let Some(bundle) = cached { ( @@ -47,7 +49,7 @@ pub async fn handle( true, ) } else { - let result = state.dispatcher.solve(&payload.url).await; + let result = state.dispatcher.solve(request).await; let solved = match result { Ok(v) => v, Err(e) => { diff --git a/px-server/tests/cache.rs b/px-server/tests/cache.rs index 6bd3b91..ad5e43a 100644 --- a/px-server/tests/cache.rs +++ b/px-server/tests/cache.rs @@ -39,3 +39,75 @@ async fn second_solve_for_same_domain_hits_cache() { assert_eq!(dispatcher.calls.load(Ordering::Relaxed), 1); assert_eq!(audit.count.load(Ordering::Relaxed), 2); } + +/// The egress is part of a bundle's identity: PX binds `_px3` to the IP +/// that earned it, so two solves of one domain through different proxies +/// must not share a cache entry. +#[tokio::test] +async fn solves_through_different_proxies_do_not_share_a_cache_entry() { + let audit = Arc::new(CountingAuditSink::default()); + let dispatcher = Arc::new(FakeDispatcher::default()); + let state = build_state_with_dispatcher(audit.clone(), dispatcher.clone()); + let app = build_router(state); + + let first = app + .clone() + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://a.example:8080"}"#, + )) + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::OK); + assert!(body_string(first).await.contains("\"cache_hit\":false")); + assert_eq!( + dispatcher.last_proxy.lock().await.as_deref(), + Some("http://a.example:8080") + ); + + let second = app + .clone() + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://b.example:8080"}"#, + )) + .await + .unwrap(); + assert!(body_string(second).await.contains("\"cache_hit\":false")); + + let repeat = app + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://a.example:8080"}"#, + )) + .await + .unwrap(); + assert!(body_string(repeat).await.contains("\"cache_hit\":true")); + + assert_eq!(dispatcher.calls.load(Ordering::Relaxed), 2); +} + +/// A direct solve must not be served a bundle harvested through a proxy. +#[tokio::test] +async fn a_proxied_bundle_is_not_replayed_for_a_direct_solve() { + let audit = Arc::new(CountingAuditSink::default()); + let dispatcher = Arc::new(FakeDispatcher::default()); + let state = build_state_with_dispatcher(audit.clone(), dispatcher.clone()); + let app = build_router(state); + + let proxied = app + .clone() + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://a.example:8080"}"#, + )) + .await + .unwrap(); + assert!(body_string(proxied).await.contains("\"cache_hit\":false")); + + let direct = app + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":null}"#, + )) + .await + .unwrap(); + assert!(body_string(direct).await.contains("\"cache_hit\":false")); + assert!(dispatcher.last_proxy.lock().await.is_none()); + assert_eq!(dispatcher.calls.load(Ordering::Relaxed), 2); +} diff --git a/px-server/tests/common/mod.rs b/px-server/tests/common/mod.rs index 2d56842..887b0c1 100644 --- a/px-server/tests/common/mod.rs +++ b/px-server/tests/common/mod.rs @@ -19,6 +19,7 @@ use px_server::{AppState, AppStateConfig}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, SystemTime}; +use tokio::sync::Mutex; pub fn hash(secret: &str) -> String { let salt = SaltString::encode_b64(b"px-solver-test-salt-fixed").expect("salt"); @@ -31,12 +32,14 @@ pub fn hash(secret: &str) -> String { #[derive(Default)] pub struct FakeDispatcher { pub calls: AtomicUsize, + pub last_proxy: Mutex>, } #[async_trait] impl SolveDispatcher for FakeDispatcher { - async fn solve(&self, _url: &str) -> Result { + async fn solve(&self, req: px_core::SolveRequest) -> Result { self.calls.fetch_add(1, Ordering::Relaxed); + *self.last_proxy.lock().await = req.proxy; Ok(SolveOutput { bundle: px_core::PxCookieBundle::new( vec![NamedCookie { diff --git a/px-server/tests/routing.rs b/px-server/tests/routing.rs new file mode 100644 index 0000000..5f38ff7 --- /dev/null +++ b/px-server/tests/routing.rs @@ -0,0 +1,130 @@ +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use async_trait::async_trait; +use px_core::{CookieJarDelta, SolveRequest}; +use px_errors::AppError; +use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml, SolveAction}; +use px_server::application::routing::{RoutingDispatcher, parse_camoufox_domains}; +use px_server::application::solve_endpoint::SolveDispatcher; +use std::sync::Arc; +use tokio::sync::Mutex; + +#[derive(Default)] +struct StaticHandler { + name: &'static str, + seen_proxy: Mutex>, +} + +impl StaticHandler { + fn new(name: &'static str) -> Arc { + Arc::new(Self { + name, + seen_proxy: Mutex::new(None), + }) + } +} + +#[async_trait] +impl ChallengeHandler for StaticHandler { + fn name(&self) -> &'static str { + self.name + } + + async fn detects(&self, _page: &PageHtml) -> Result { + Ok(true) + } + + async fn solve(&self, action: &SolveAction) -> Result { + *self.seen_proxy.lock().await = action.proxy.clone(); + Ok(HandlerOutcome::solved_with_ua( + self.name, + CookieJarDelta::default(), + Vec::new(), + HandlerMetrics::default(), + "ua", + )) + } +} + +fn dispatcher_with_cf_route() -> RoutingDispatcher { + RoutingDispatcher::new(StaticHandler::new("perimeterx")) + .with_route("pedidosya.com.ar", StaticHandler::new("cloudflare")) +} + +#[tokio::test] +async fn test_solve_unrouted_host_uses_default_handler() { + let out = dispatcher_with_cf_route() + .solve(SolveRequest::new("https://www.havenwellwithin.com/")) + .await + .expect("solve"); + assert_eq!(out.handler, "perimeterx"); +} + +#[tokio::test] +async fn test_solve_exact_host_uses_routed_handler() { + let out = dispatcher_with_cf_route() + .solve(SolveRequest::new("https://pedidosya.com.ar/")) + .await + .expect("solve"); + assert_eq!(out.handler, "cloudflare"); +} + +#[tokio::test] +async fn test_solve_subdomain_uses_routed_handler() { + let out = dispatcher_with_cf_route() + .solve(SolveRequest::new("https://www.pedidosya.com.ar/x")) + .await + .expect("solve"); + assert_eq!(out.handler, "cloudflare"); +} + +/// Regression: the cookie bundle's `user_agent` must carry the real +/// harvester UA (so cache-hit replies preserve it), not a placeholder. +#[tokio::test] +async fn test_solve_bundle_user_agent_matches_harvester() { + let out = RoutingDispatcher::new(StaticHandler::new("perimeterx")) + .solve(SolveRequest::new("https://example.com/")) + .await + .expect("solve"); + assert_eq!(out.user_agent, "ua"); + assert_eq!(out.bundle.user_agent, "ua"); +} + +/// Regression: the request's proxy has to reach the handler. It used to be +/// deserialized at the edge and dropped — no browser ever saw it. +#[tokio::test] +async fn test_solve_forwards_requested_proxy_to_handler() { + let handler = StaticHandler::new("perimeterx"); + let dispatcher = RoutingDispatcher::new(Arc::clone(&handler) as Arc); + dispatcher + .solve(SolveRequest::new("https://example.com/").with_proxy("socks5://127.0.0.1:9050")) + .await + .expect("solve"); + assert_eq!( + handler.seen_proxy.lock().await.as_deref(), + Some("socks5://127.0.0.1:9050") + ); +} + +#[tokio::test] +async fn test_solve_without_proxy_forwards_none() { + let handler = StaticHandler::new("perimeterx"); + let dispatcher = RoutingDispatcher::new(Arc::clone(&handler) as Arc); + dispatcher + .solve(SolveRequest::new("https://example.com/")) + .await + .expect("solve"); + assert!(handler.seen_proxy.lock().await.is_none()); +} + +#[test] +fn test_parse_camoufox_domains_trims_and_lowercases() { + let parsed = parse_camoufox_domains(Some(" Pedidosya.com.AR , ,foo.com ")); + assert_eq!(parsed, vec!["pedidosya.com.ar", "foo.com"]); +} + +#[test] +fn test_parse_camoufox_domains_unset_is_empty() { + assert!(parse_camoufox_domains(None).is_empty()); + assert!(parse_camoufox_domains(Some("")).is_empty()); +} diff --git a/px-turnstile/src/lib.rs b/px-turnstile/src/lib.rs index 8051e3c..d3f8e94 100644 --- a/px-turnstile/src/lib.rs +++ b/px-turnstile/src/lib.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml, SolveAction}; pub struct TurnstileHandler; @@ -27,7 +27,7 @@ impl ChallengeHandler for TurnstileHandler { Ok(h.contains("challenges.cloudflare.com/turnstile") || h.contains("cf-turnstile")) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented(self.name())) } } From ed4bf6740666b0b874e8fe14c64c5251b4ed482c Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Fri, 7 Aug 2026 21:56:43 +0700 Subject: [PATCH 2/7] fix(xtask): re-pin internal deps when bumping the workspace version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bump` rewrote only `[workspace.package] version`, leaving the `px-* = { path, version, package = "pxsolver-*" }` entries at whatever they were last set to by hand — 1.4.0 since that release, while crates were publishing at 1.8.0. Minor bumps hid it: 1.8.0 still satisfies `^1.4.0`. A major bump does not, so `cargo` fails to resolve the workspace the moment the version crosses 2.0.0, with no obvious link back to the bump that caused it. Every crate carries `version.workspace = true`, so the pins are meant to move in lockstep; bump now rewrites them with the workspace version. --- xtask/src/main.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ebc2071..a2dfb65 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -124,6 +124,7 @@ fn bump( if new == manifest { bail!("could not find `{from}` in Cargo.toml"); } + let new = repin_workspace_deps(&new, &next); fs::write(manifest_path, new).context("write root Cargo.toml")?; if !skip_gate { @@ -356,6 +357,30 @@ impl std::fmt::Display for Version { } } +/// Re-pin the internal `px-* = { path, version, package = "pxsolver-*" }` +/// entries to the new workspace version. +/// +/// Every crate carries `version.workspace = true`, so the published +/// artifacts all bump together; a stale pin only survives while the new +/// version still satisfies the old caret requirement. A major bump does +/// not, and `cargo` then fails to resolve the workspace at all. +fn repin_workspace_deps(manifest: &str, next: &Version) -> String { + manifest + .lines() + .map(|line| match line.split_once("version = \"") { + Some((head, rest)) if line.contains("package = \"pxsolver-") => { + match rest.split_once('"') { + Some((_old, tail)) => format!("{head}version = \"{next}\"{tail}"), + None => line.to_string(), + } + } + _ => line.to_string(), + }) + .collect::>() + .join("\n") + + "\n" +} + fn parse_workspace_version(manifest: &str) -> Result { let line = manifest .lines() From 27f43b8f5f8bdf83628312b3dc23e6e5b0f708ea Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Fri, 7 Aug 2026 21:56:43 +0700 Subject: [PATCH 3/7] chore: bump to 2.0.0 Post-1.0.0 architectural change per ADR-0017: ChallengeHandler::solve and SolveDispatcher::solve change signature so the per-request egress proxy reaches the browser and native paths (ADR-0025). Breaking for downstream users of pxsolver-pipeline, -harvester, -native and -core. Internal dependency pins move to 2.0.0 with the workspace version; they had been stale at 1.4.0 since that release. --- Cargo.lock | 37 +++++++++++++++++++------------------ Cargo.toml | 34 +++++++++++++++++----------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d4acbf..7a95cfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1464,12 +1464,13 @@ dependencies = [ [[package]] name = "px-cli" -version = "1.8.0" +version = "2.0.0" dependencies = [ "anyhow", "argon2", "clap", "pxsolver-auth", + "pxsolver-core", "pxsolver-detector", "pxsolver-native", "reqwest 0.12.28", @@ -1482,7 +1483,7 @@ dependencies = [ [[package]] name = "px-server" -version = "1.8.0" +version = "2.0.0" dependencies = [ "anyhow", "argon2", @@ -1515,7 +1516,7 @@ dependencies = [ [[package]] name = "pxsolver-auth" -version = "1.8.0" +version = "2.0.0" dependencies = [ "argon2", "async-trait", @@ -1532,7 +1533,7 @@ dependencies = [ [[package]] name = "pxsolver-cache" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "dashmap", @@ -1544,7 +1545,7 @@ dependencies = [ [[package]] name = "pxsolver-camoufox" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "fantoccini", @@ -1562,7 +1563,7 @@ dependencies = [ [[package]] name = "pxsolver-captcha" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1571,7 +1572,7 @@ dependencies = [ [[package]] name = "pxsolver-cloudflare" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1583,7 +1584,7 @@ dependencies = [ [[package]] name = "pxsolver-core" -version = "1.8.0" +version = "2.0.0" dependencies = [ "serde", "uuid", @@ -1591,7 +1592,7 @@ dependencies = [ [[package]] name = "pxsolver-datadome" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1600,7 +1601,7 @@ dependencies = [ [[package]] name = "pxsolver-detector" -version = "1.8.0" +version = "2.0.0" dependencies = [ "pxsolver-core", "regex", @@ -1608,7 +1609,7 @@ dependencies = [ [[package]] name = "pxsolver-errors" -version = "1.8.0" +version = "2.0.0" dependencies = [ "axum", "serde", @@ -1617,7 +1618,7 @@ dependencies = [ [[package]] name = "pxsolver-harvester" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "chromiumoxide", @@ -1630,7 +1631,7 @@ dependencies = [ [[package]] name = "pxsolver-native" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "futures", @@ -1649,7 +1650,7 @@ dependencies = [ [[package]] name = "pxsolver-perimeterx" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1663,7 +1664,7 @@ dependencies = [ [[package]] name = "pxsolver-pipeline" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1675,7 +1676,7 @@ dependencies = [ [[package]] name = "pxsolver-turnstile" -version = "1.8.0" +version = "2.0.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1684,7 +1685,7 @@ dependencies = [ [[package]] name = "pxsolver-types" -version = "1.8.0" +version = "2.0.0" dependencies = [ "serde", "serde_json", @@ -1692,7 +1693,7 @@ dependencies = [ [[package]] name = "pxsolver-validation" -version = "1.8.0" +version = "2.0.0" dependencies = [ "axum", "pxsolver-errors", diff --git a/Cargo.toml b/Cargo.toml index acbf11a..05a8a76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "1.8.0" +version = "2.0.0" edition = "2024" rust-version = "1.95" license = "AGPL-3.0-or-later" @@ -38,22 +38,22 @@ unsafe_code = "forbid" # so `cargo publish` can resolve to the registry copy. Each entry's package # = pxsolver-* (the crates.io name); the local alias `px-*` is kept so # source files can continue to `use px_core::…` unchanged. -px-core = { path = "px-core", version = "1.4.0", package = "pxsolver-core" } -px-types = { path = "px-types", version = "1.4.0", package = "pxsolver-types" } -px-errors = { path = "px-errors", version = "1.4.0", package = "pxsolver-errors" } -px-validation = { path = "px-validation", version = "1.4.0", package = "pxsolver-validation" } -px-cache = { path = "px-cache", version = "1.4.0", package = "pxsolver-cache" } -px-detector = { path = "px-detector", version = "1.4.0", package = "pxsolver-detector" } -px-harvester = { path = "px-harvester", version = "1.4.0", package = "pxsolver-harvester" } -px-pipeline = { path = "px-pipeline", version = "1.4.0", package = "pxsolver-pipeline" } -px-perimeterx = { path = "px-perimeterx", version = "1.4.0", package = "pxsolver-perimeterx" } -px-cloudflare = { path = "px-cloudflare", version = "1.4.0", package = "pxsolver-cloudflare" } -px-turnstile = { path = "px-turnstile", version = "1.4.0", package = "pxsolver-turnstile" } -px-captcha = { path = "px-captcha", version = "1.4.0", package = "pxsolver-captcha" } -px-datadome = { path = "px-datadome", version = "1.4.0", package = "pxsolver-datadome" } -px-native = { path = "px-native", version = "1.4.0", package = "pxsolver-native" } -px-auth = { path = "px-auth", version = "1.4.0", package = "pxsolver-auth" } -px-camoufox = { path = "px-camoufox", version = "1.4.0", package = "pxsolver-camoufox" } +px-core = { path = "px-core", version = "2.0.0", package = "pxsolver-core" } +px-types = { path = "px-types", version = "2.0.0", package = "pxsolver-types" } +px-errors = { path = "px-errors", version = "2.0.0", package = "pxsolver-errors" } +px-validation = { path = "px-validation", version = "2.0.0", package = "pxsolver-validation" } +px-cache = { path = "px-cache", version = "2.0.0", package = "pxsolver-cache" } +px-detector = { path = "px-detector", version = "2.0.0", package = "pxsolver-detector" } +px-harvester = { path = "px-harvester", version = "2.0.0", package = "pxsolver-harvester" } +px-pipeline = { path = "px-pipeline", version = "2.0.0", package = "pxsolver-pipeline" } +px-perimeterx = { path = "px-perimeterx", version = "2.0.0", package = "pxsolver-perimeterx" } +px-cloudflare = { path = "px-cloudflare", version = "2.0.0", package = "pxsolver-cloudflare" } +px-turnstile = { path = "px-turnstile", version = "2.0.0", package = "pxsolver-turnstile" } +px-captcha = { path = "px-captcha", version = "2.0.0", package = "pxsolver-captcha" } +px-datadome = { path = "px-datadome", version = "2.0.0", package = "pxsolver-datadome" } +px-native = { path = "px-native", version = "2.0.0", package = "pxsolver-native" } +px-auth = { path = "px-auth", version = "2.0.0", package = "pxsolver-auth" } +px-camoufox = { path = "px-camoufox", version = "2.0.0", package = "pxsolver-camoufox" } anyhow = "1.0" argon2 = "0.5" From f534b63150ce82d24904049a71f893fc93d6c19a Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Fri, 7 Aug 2026 21:57:50 +0700 Subject: [PATCH 4/7] style(errors): capitalize error messages in the proxy-touched harvester files AppError::message() returns the payload verbatim as the whole user-facing message, so each one starts a sentence. --- .../src/infrastructure/camoufox_pool.rs | 16 +++++++-------- .../src/infrastructure/chromiumoxide_pool.rs | 20 +++++++++---------- px-native/src/infrastructure/proxy_clients.rs | 6 +++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/px-camoufox/src/infrastructure/camoufox_pool.rs b/px-camoufox/src/infrastructure/camoufox_pool.rs index af83b6e..094aeb9 100644 --- a/px-camoufox/src/infrastructure/camoufox_pool.rs +++ b/px-camoufox/src/infrastructure/camoufox_pool.rs @@ -25,7 +25,7 @@ impl CamoufoxPool { pub fn new(config: CamoufoxConfig) -> Result { config .validate() - .map_err(|e| AppError::InternalError(format!("camoufox config: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Camoufox config: {e}")))?; let permits = Arc::new(Semaphore::new(config.max_concurrent)); let max_per_domain = std::env::var("PX_FETCH_MAX_PER_DOMAIN") .ok() @@ -78,7 +78,7 @@ impl CamoufoxPool { .permits .acquire() .await - .map_err(|e| AppError::InternalError(format!("semaphore: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Semaphore: {e}")))?; let port = pick_free_port().await?; let mut child = Command::new(&self.config.geckodriver_bin) .arg("--port") @@ -89,7 +89,7 @@ impl CamoufoxPool { .stderr(std::process::Stdio::null()) .kill_on_drop(true) .spawn() - .map_err(|e| AppError::InternalError(format!("spawn geckodriver: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Spawn geckodriver: {e}")))?; wait_for_geckodriver(port, Duration::from_secs(15)).await?; let caps = build_capabilities(&self.config, proxy); let endpoint = format!("http://127.0.0.1:{port}"); @@ -126,26 +126,26 @@ async fn harvest_session( .capabilities(caps) .connect(endpoint) .await - .map_err(|e| AppError::InternalError(format!("webdriver connect: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Webdriver connect: {e}")))?; let nav = client.goto(&req.url); if tokio::time::timeout(navigate_timeout, nav).await.is_err() { let _ = client.close().await; - return Err(AppError::InternalError("navigate timeout".into())); + return Err(AppError::InternalError("Navigate timeout".into())); } sleep(Duration::from_millis(req.wait_ms)).await; let html = client .source() .await - .map_err(|e| AppError::InternalError(format!("source: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Source: {e}")))?; let ua_val = client .execute("return navigator.userAgent;", vec![]) .await - .map_err(|e| AppError::InternalError(format!("ua eval: {e}")))?; + .map_err(|e| AppError::InternalError(format!("User agent eval: {e}")))?; let user_agent = ua_val.as_str().unwrap_or("").to_string(); let raw_cookies = client .get_all_cookies() .await - .map_err(|e| AppError::InternalError(format!("cookies: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Cookies: {e}")))?; let cookies = raw_cookies .into_iter() .map(|c| HarvestedCookie { diff --git a/px-harvester/src/infrastructure/chromiumoxide_pool.rs b/px-harvester/src/infrastructure/chromiumoxide_pool.rs index 22b1f35..fe1a258 100644 --- a/px-harvester/src/infrastructure/chromiumoxide_pool.rs +++ b/px-harvester/src/infrastructure/chromiumoxide_pool.rs @@ -64,7 +64,7 @@ impl ChromiumoxidePool { .map_err(|e| AppError::InternalError(format!("BrowserConfig build: {e}")))?; let (browser, mut handler) = Browser::launch(cfg) .await - .map_err(|e| AppError::InternalError(format!("browser launch: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Browser launch: {e}")))?; let handle = tokio::spawn(async move { while let Some(event) = handler.next().await { if event.is_err() { @@ -79,7 +79,7 @@ impl ChromiumoxidePool { let cookies = page .get_cookies() .await - .map_err(|e| AppError::InternalError(format!("get_cookies: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Get cookies: {e}")))?; Ok(cookies .into_iter() .map(|c| HarvestedCookie { @@ -99,7 +99,7 @@ impl Harvester for ChromiumoxidePool { .permits .acquire() .await - .map_err(|e| AppError::InternalError(format!("semaphore: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Semaphore: {e}")))?; let proxy = req.proxy.clone().map(strip_credentials); tracing::info!( url = %req.url, @@ -110,29 +110,29 @@ impl Harvester for ChromiumoxidePool { let page = browser .new_page("about:blank") .await - .map_err(|e| AppError::InternalError(format!("new_page: {e}")))?; + .map_err(|e| AppError::InternalError(format!("New page: {e}")))?; let script = self.stealth.combined(); if !script.is_empty() { page.evaluate_on_new_document(script.as_str()) .await - .map_err(|e| AppError::InternalError(format!("inject stealth: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Inject stealth: {e}")))?; } let navigate = page.goto(&req.url); tokio::time::timeout(self.config.navigate_timeout, navigate) .await - .map_err(|_| AppError::InternalError("navigate timeout".into()))? - .map_err(|e| AppError::InternalError(format!("goto: {e}")))?; + .map_err(|_| AppError::InternalError("Navigate timeout".into()))? + .map_err(|e| AppError::InternalError(format!("Goto: {e}")))?; tokio::time::sleep(Duration::from_millis(req.wait_ms)).await; let html = page .content() .await - .map_err(|e| AppError::InternalError(format!("content: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Content: {e}")))?; let ua = page .evaluate("navigator.userAgent") .await - .map_err(|e| AppError::InternalError(format!("eval ua: {e}")))? + .map_err(|e| AppError::InternalError(format!("Eval user agent: {e}")))? .into_value::() - .map_err(|e| AppError::InternalError(format!("ua parse: {e}")))?; + .map_err(|e| AppError::InternalError(format!("User agent parse: {e}")))?; let cookies = Self::extract_cookies(&page).await?; let _ = browser.close().await; Ok(HarvestResult { diff --git a/px-native/src/infrastructure/proxy_clients.rs b/px-native/src/infrastructure/proxy_clients.rs index dd6e3cd..e42fbab 100644 --- a/px-native/src/infrastructure/proxy_clients.rs +++ b/px-native/src/infrastructure/proxy_clients.rs @@ -35,17 +35,17 @@ impl ProxyClients { let mut cache = self .proxied .lock() - .map_err(|e| AppError::InternalError(format!("proxy client cache poisoned: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Proxy client cache poisoned: {e}")))?; if let Some(client) = cache.get(proxy) { return Ok(client.clone()); } let built = Client::builder() .proxy( Proxy::all(proxy) - .map_err(|e| AppError::BadRequest(format!("invalid proxy url: {e}")))?, + .map_err(|e| AppError::BadRequest(format!("Invalid proxy url: {e}")))?, ) .build() - .map_err(|e| AppError::InternalError(format!("build proxied client: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Failed to build proxied client: {e}")))?; cache.insert(proxy.to_string(), built.clone()); Ok(built) } From 24be22e0bffff12f1edd7416089ce3d46672925c Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Fri, 7 Aug 2026 22:00:57 +0700 Subject: [PATCH 5/7] fix(proxy): stop two silent drops on the wired-up egress path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chromium has no `socks5h` scheme and silently ignores a `--proxy-server` spec it cannot parse, so a `socks5h://` proxy — a scheme this API documents and geckodriver accepts — would have gone direct with no error. chromium_proxy_spec rewrites it to `socks5://`, and proxy_arg is now covered: the Chromium leg was the one part of the wiring with no test, and the part that had no proxy argument at all before. SolveRequest also serialized its absent optionals as `null`. The CLI's old private body struct skipped them, so switching it to the published type put `"fingerprint":null` on the wire — a field the server has no place for and silently discards, which is the same class of unkept promise this change set out to remove. --- Cargo.lock | 1 + docs/deployment.md | 2 +- px-core/Cargo.toml | 3 ++ px-core/src/solve_request.rs | 24 +++++++++++++ .../src/infrastructure/chromiumoxide_pool.rs | 32 +++++++++++++++-- px-harvester/src/infrastructure/egress.rs | 35 +++++++++++++++++++ 6 files changed, 94 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a95cfe..618d4fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1587,6 +1587,7 @@ name = "pxsolver-core" version = "2.0.0" dependencies = [ "serde", + "serde_json", "uuid", ] diff --git a/docs/deployment.md b/docs/deployment.md index b9dbc56..3e2b601 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -149,7 +149,7 @@ curl -X POST http://127.0.0.1:8080/v1/solve \ -d '{"url":"https://www.pedidosya.com.ar/","proxy":"socks5://127.0.0.1:9050"}' ``` -Accepted schemes: `http`, `https`, `socks5`, `socks5h`. `"proxy":null` (or omitting the field) harvests from the server's own address. +Accepted schemes: `http`, `https`, `socks5`, `socks5h`. Chromium has no `socks5h` scheme and silently ignores a spec it cannot parse, so the Chromium path rewrites it to `socks5://` rather than going direct without saying so. `"proxy":null` (or omitting the field) harvests from the server's own address. The solve **never** falls back to the `PX_PROXIES` rotation — a bundle earned through an IP the caller cannot name would not be usable. The proxy is part of the cache key, so the same domain solved through two different proxies produces two entries and neither is served to the other. diff --git a/px-core/Cargo.toml b/px-core/Cargo.toml index be8221c..fdead31 100644 --- a/px-core/Cargo.toml +++ b/px-core/Cargo.toml @@ -15,5 +15,8 @@ workspace = true serde = { workspace = true } uuid = { workspace = true } +[dev-dependencies] +serde_json = { workspace = true } + [lib] name = "px_core" diff --git a/px-core/src/solve_request.rs b/px-core/src/solve_request.rs index d93f06e..e1c9873 100644 --- a/px-core/src/solve_request.rs +++ b/px-core/src/solve_request.rs @@ -1,10 +1,15 @@ use crate::fingerprint::Fingerprint; use serde::{Deserialize, Serialize}; +/// Absent optionals are omitted from the wire, not sent as `null`: this +/// type is what a client serializes, and a `"fingerprint": null` the +/// server has no field for reads as a promise the API does not keep. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SolveRequest { pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] pub proxy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub fingerprint: Option, } @@ -44,6 +49,25 @@ impl SolveRequest { mod tests { use super::*; + #[test] + fn absent_optionals_stay_off_the_wire() { + let json = serde_json::to_string(&SolveRequest::new("https://example.com")) + .expect("serialize request"); + assert_eq!(json, r#"{"url":"https://example.com"}"#); + } + + #[test] + fn a_named_proxy_is_serialized() { + let json = serde_json::to_string( + &SolveRequest::new("https://example.com").with_proxy("socks5://127.0.0.1:9050"), + ) + .expect("serialize request"); + assert!( + json.contains(r#""proxy":"socks5://127.0.0.1:9050""#), + "{json}" + ); + } + #[test] fn builder_assembles_request() { let r = SolveRequest::new("https://example.com").with_proxy("http://1.2.3.4:8080"); diff --git a/px-harvester/src/infrastructure/chromiumoxide_pool.rs b/px-harvester/src/infrastructure/chromiumoxide_pool.rs index fe1a258..cec84d3 100644 --- a/px-harvester/src/infrastructure/chromiumoxide_pool.rs +++ b/px-harvester/src/infrastructure/chromiumoxide_pool.rs @@ -1,6 +1,6 @@ use crate::domain::harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; use crate::domain::stealth::{StealthBundle, default_stealth_bundle}; -use crate::infrastructure::egress::strip_credentials; +use crate::infrastructure::egress::{chromium_proxy_spec, strip_credentials}; use async_trait::async_trait; use chromiumoxide::browser::{Browser, BrowserConfig}; use chromiumoxide::page::Page; @@ -27,6 +27,12 @@ impl Default for PoolConfig { } } +/// The launch flag carrying the egress, rendered by chromiumoxide as +/// `--proxy-server=`. +fn proxy_arg(proxy: &str) -> (&'static str, String) { + ("proxy-server", chromium_proxy_spec(proxy)) +} + pub struct ChromiumoxidePool { config: PoolConfig, stealth: StealthBundle, @@ -57,7 +63,8 @@ impl ChromiumoxidePool { cfg = cfg.with_head(); } if let Some(proxy_url) = proxy { - cfg = cfg.arg(("proxy-server", proxy_url)); + let (key, value) = proxy_arg(proxy_url); + cfg = cfg.arg((key, value.as_str())); } let cfg = cfg .build() @@ -142,3 +149,24 @@ impl Harvester for ChromiumoxidePool { }) } } + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + /// Regression: the Chromium leg had no proxy argument at all, so a + /// requested egress was dropped without a trace. + #[test] + fn proxy_arg_carries_the_requested_egress() { + let (key, value) = proxy_arg("http://egress.example:8080"); + assert_eq!(key, "proxy-server"); + assert_eq!(value, "http://egress.example:8080"); + } + + #[test] + fn proxy_arg_normalizes_a_scheme_chromium_would_ignore() { + let (_, value) = proxy_arg("socks5h://egress.example:1080"); + assert_eq!(value, "socks5://egress.example:1080"); + } +} diff --git a/px-harvester/src/infrastructure/egress.rs b/px-harvester/src/infrastructure/egress.rs index 525d146..8fcc672 100644 --- a/px-harvester/src/infrastructure/egress.rs +++ b/px-harvester/src/infrastructure/egress.rs @@ -30,6 +30,20 @@ pub fn strip_credentials(proxy: String) -> String { sanitized } +/// Normalize a proxy URL into a Chromium `--proxy-server` spec. +/// +/// Chromium understands `http`, `https`, `socks4` and `socks5` — but not +/// `socks5h`, which is a curl convention that geckodriver's capability +/// layer accepts. Chromium *silently ignores* a spec it cannot parse and +/// goes direct, which is the failure this whole path exists to remove, so +/// the unknown scheme is rewritten rather than passed through. +pub fn chromium_proxy_spec(proxy: &str) -> String { + match proxy.split_once("://") { + Some(("socks5h", host_port)) => format!("socks5://{host_port}"), + _ => proxy.to_string(), + } +} + #[cfg(test)] #[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] mod tests { @@ -55,4 +69,25 @@ mod tests { ); assert_eq!(strip_credentials("host:8080".into()), "host:8080"); } + + #[test] + fn chromium_spec_rewrites_socks5h_which_chromium_cannot_parse() { + assert_eq!( + chromium_proxy_spec("socks5h://x.example:1080"), + "socks5://x.example:1080" + ); + } + + #[test] + fn chromium_spec_passes_supported_schemes_through() { + for proxy in [ + "http://x.example:8080", + "https://x.example:8443", + "socks4://x.example:1080", + "socks5://x.example:1080", + "x.example:8080", + ] { + assert_eq!(chromium_proxy_spec(proxy), proxy); + } + } } From d6e263ab69a0548092dffe69aa4c8c524e5bce26 Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Sat, 8 Aug 2026 00:20:40 +0700 Subject: [PATCH 6/7] chore: bump to 1.9.0 Supersedes the 2.0.0 bump earlier in this branch. Maintainer's call to ship the proxy contract as a minor, amending the ADR-0017 line that a post-1.0.0 architectural change takes a manual major. The break is real and stated rather than hidden: implementors of ChallengeHandler outside this workspace must add the SolveAction parameter, so a `pxsolver-* = "1"` pin fails to compile on cargo update. README carries the warning; ADR-0025 records the trade. --- Cargo.lock | 36 +++++++++---------- Cargo.toml | 34 +++++++++--------- README.md | 35 ++++++++++++++++-- .../0025-egress-proxy-propagation-contract.md | 10 ++++-- docs/adr/README.md | 2 +- 5 files changed, 76 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 618d4fe..f7987e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1464,7 +1464,7 @@ dependencies = [ [[package]] name = "px-cli" -version = "2.0.0" +version = "1.9.0" dependencies = [ "anyhow", "argon2", @@ -1483,7 +1483,7 @@ dependencies = [ [[package]] name = "px-server" -version = "2.0.0" +version = "1.9.0" dependencies = [ "anyhow", "argon2", @@ -1516,7 +1516,7 @@ dependencies = [ [[package]] name = "pxsolver-auth" -version = "2.0.0" +version = "1.9.0" dependencies = [ "argon2", "async-trait", @@ -1533,7 +1533,7 @@ dependencies = [ [[package]] name = "pxsolver-cache" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "dashmap", @@ -1545,7 +1545,7 @@ dependencies = [ [[package]] name = "pxsolver-camoufox" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "fantoccini", @@ -1563,7 +1563,7 @@ dependencies = [ [[package]] name = "pxsolver-captcha" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1572,7 +1572,7 @@ dependencies = [ [[package]] name = "pxsolver-cloudflare" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1584,7 +1584,7 @@ dependencies = [ [[package]] name = "pxsolver-core" -version = "2.0.0" +version = "1.9.0" dependencies = [ "serde", "serde_json", @@ -1593,7 +1593,7 @@ dependencies = [ [[package]] name = "pxsolver-datadome" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "pxsolver-detector" -version = "2.0.0" +version = "1.9.0" dependencies = [ "pxsolver-core", "regex", @@ -1610,7 +1610,7 @@ dependencies = [ [[package]] name = "pxsolver-errors" -version = "2.0.0" +version = "1.9.0" dependencies = [ "axum", "serde", @@ -1619,7 +1619,7 @@ dependencies = [ [[package]] name = "pxsolver-harvester" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "chromiumoxide", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "pxsolver-native" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "futures", @@ -1651,7 +1651,7 @@ dependencies = [ [[package]] name = "pxsolver-perimeterx" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1665,7 +1665,7 @@ dependencies = [ [[package]] name = "pxsolver-pipeline" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1677,7 +1677,7 @@ dependencies = [ [[package]] name = "pxsolver-turnstile" -version = "2.0.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "pxsolver-types" -version = "2.0.0" +version = "1.9.0" dependencies = [ "serde", "serde_json", @@ -1694,7 +1694,7 @@ dependencies = [ [[package]] name = "pxsolver-validation" -version = "2.0.0" +version = "1.9.0" dependencies = [ "axum", "pxsolver-errors", diff --git a/Cargo.toml b/Cargo.toml index 05a8a76..584641e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "2.0.0" +version = "1.9.0" edition = "2024" rust-version = "1.95" license = "AGPL-3.0-or-later" @@ -38,22 +38,22 @@ unsafe_code = "forbid" # so `cargo publish` can resolve to the registry copy. Each entry's package # = pxsolver-* (the crates.io name); the local alias `px-*` is kept so # source files can continue to `use px_core::…` unchanged. -px-core = { path = "px-core", version = "2.0.0", package = "pxsolver-core" } -px-types = { path = "px-types", version = "2.0.0", package = "pxsolver-types" } -px-errors = { path = "px-errors", version = "2.0.0", package = "pxsolver-errors" } -px-validation = { path = "px-validation", version = "2.0.0", package = "pxsolver-validation" } -px-cache = { path = "px-cache", version = "2.0.0", package = "pxsolver-cache" } -px-detector = { path = "px-detector", version = "2.0.0", package = "pxsolver-detector" } -px-harvester = { path = "px-harvester", version = "2.0.0", package = "pxsolver-harvester" } -px-pipeline = { path = "px-pipeline", version = "2.0.0", package = "pxsolver-pipeline" } -px-perimeterx = { path = "px-perimeterx", version = "2.0.0", package = "pxsolver-perimeterx" } -px-cloudflare = { path = "px-cloudflare", version = "2.0.0", package = "pxsolver-cloudflare" } -px-turnstile = { path = "px-turnstile", version = "2.0.0", package = "pxsolver-turnstile" } -px-captcha = { path = "px-captcha", version = "2.0.0", package = "pxsolver-captcha" } -px-datadome = { path = "px-datadome", version = "2.0.0", package = "pxsolver-datadome" } -px-native = { path = "px-native", version = "2.0.0", package = "pxsolver-native" } -px-auth = { path = "px-auth", version = "2.0.0", package = "pxsolver-auth" } -px-camoufox = { path = "px-camoufox", version = "2.0.0", package = "pxsolver-camoufox" } +px-core = { path = "px-core", version = "1.9.0", package = "pxsolver-core" } +px-types = { path = "px-types", version = "1.9.0", package = "pxsolver-types" } +px-errors = { path = "px-errors", version = "1.9.0", package = "pxsolver-errors" } +px-validation = { path = "px-validation", version = "1.9.0", package = "pxsolver-validation" } +px-cache = { path = "px-cache", version = "1.9.0", package = "pxsolver-cache" } +px-detector = { path = "px-detector", version = "1.9.0", package = "pxsolver-detector" } +px-harvester = { path = "px-harvester", version = "1.9.0", package = "pxsolver-harvester" } +px-pipeline = { path = "px-pipeline", version = "1.9.0", package = "pxsolver-pipeline" } +px-perimeterx = { path = "px-perimeterx", version = "1.9.0", package = "pxsolver-perimeterx" } +px-cloudflare = { path = "px-cloudflare", version = "1.9.0", package = "pxsolver-cloudflare" } +px-turnstile = { path = "px-turnstile", version = "1.9.0", package = "pxsolver-turnstile" } +px-captcha = { path = "px-captcha", version = "1.9.0", package = "pxsolver-captcha" } +px-datadome = { path = "px-datadome", version = "1.9.0", package = "pxsolver-datadome" } +px-native = { path = "px-native", version = "1.9.0", package = "pxsolver-native" } +px-auth = { path = "px-auth", version = "1.9.0", package = "pxsolver-auth" } +px-camoufox = { path = "px-camoufox", version = "1.9.0", package = "pxsolver-camoufox" } anyhow = "1.0" argon2 = "0.5" diff --git a/README.md b/README.md index b21aa77..69d095d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ A Rust-built solver service for PerimeterX (HUMAN Security) protection. Given a target URL on a per-domain allowlist, returns a valid `_px3` cookie bundle that a downstream authorized client can use to issue requests as if from a real browser. -> **Status:** published to crates.io as the `pxsolver-*` family of crates. MVP gate hit at v1.0.0; v1.1.0 added a Camoufox-backed Cloudflare bypass path; v1.2.0 renamed the published library crates; v1.8.0 activated native `_px3` sensor synthesis ([ADR-0024](docs/adr/0024-activate-native-px3-sensor-synthesis.md)); v2.0.0 makes the per-request egress proxy real end to end ([ADR-0025](docs/adr/0025-egress-proxy-propagation-contract.md)) — a breaking `ChallengeHandler` / `SolveDispatcher` signature change. See [GitHub Releases](https://github.com/KeyCode17/px-solver/releases) for the per-version notes. +> **Status:** v1.9.0, published to crates.io as the `pxsolver-*` family of crates. MVP gate hit at v1.0.0; v1.1.0 added a Camoufox-backed Cloudflare bypass path; v1.2.0 renamed the published library crates; v1.8.0 activated native `_px3` sensor synthesis ([ADR-0024](docs/adr/0024-activate-native-px3-sensor-synthesis.md)); v1.9.0 makes the per-request egress proxy real end to end ([ADR-0025](docs/adr/0025-egress-proxy-propagation-contract.md)). See [GitHub Releases](https://github.com/KeyCode17/px-solver/releases) for the per-version notes. +> +> ⚠️ **v1.9.0 is source-breaking despite the minor bump.** `ChallengeHandler::solve` and `SolveDispatcher::solve` take a `SolveAction` / `SolveRequest` instead of `&PageHtml` / `&str`. If you implement `ChallengeHandler` outside this workspace, add the parameter when upgrading; the rest of the published surface is unchanged. ## What this is @@ -65,8 +67,6 @@ The 16 `pxsolver-*` library crates are also published individually for downstrea -d '{"url":"https://www.pedidosya.com.ar/","proxy":null}' ``` - `"proxy"` is the egress the solve harvests through — `scheme://host:port` for `http`, `https`, `socks5` or `socks5h`, or `null` for the server's own IP. The returned `_px3` bundle is bound to that IP, so send downstream requests through the same proxy. `PX_PROXIES` is a separate, `/v1/fetch`-only rotation, and browser proxies cannot carry credentials — see [Egress proxies](docs/deployment.md#egress-proxies). - Response shape: ```json @@ -85,6 +85,35 @@ The 16 `pxsolver-*` library crates are also published individually for downstrea For systemd, reverse proxy, and key rotation workflows see [`docs/deployment.md`](docs/deployment.md). +## Proxies + +Two mechanisms, one per endpoint. They are **not** interchangeable, and neither one falls back to the other. + +| | `/v1/solve` | `/v1/fetch` | +|---|---|---| +| Assigned by | `"proxy"` in the request body, or `px-cli solve --proxy` | `PX_PROXIES` env var (CSV), operator-side | +| Chosen per | request | Camoufox session, at spawn | +| Rotation | none — the solve uses exactly the proxy you named | round-robin across the list | +| Omitted | server's own IP | server's own IP | + +```bash +# /v1/solve — name the egress you will send downstream traffic through +-d '{"url":"https://www.pedidosya.com.ar/","proxy":"socks5://127.0.0.1:9050"}' + +# /v1/fetch — operator-side rotation across warm sessions +PX_PROXIES="http://p1.example:8080,socks5://p2.example:1080" ./target/release/px-server +``` + +A `_px3` bundle is bound to the IP that earned it, so **use the same proxy downstream that you named on the solve**. The proxy is part of the cache key: the same domain solved through two proxies yields two entries and neither is served to the other. + +Three things that surprise people: + +- **`PX_PROXIES` does nothing for `/v1/solve`.** Rotation there would hand you a bundle bound to an IP you cannot know. +- **Browser proxies cannot authenticate.** geckodriver's W3C `proxy` capability has no credential field and Chromium ignores userinfo without a CDP `Fetch.authRequired` handler, so `user:pass@` is stripped with a warning. Front an authenticated upstream with a local relay (gost, 3proxy). The native sensor path goes over `reqwest` and *does* accept credentials. +- **Distinct egress IPs per domain on `/v1/fetch` is `min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES))`**, not the product — a session takes its proxy at spawn and keeps it for the 300s TTL. + +Full reference: [Egress proxies](docs/deployment.md#egress-proxies) · rationale: [ADR-0025](docs/adr/0025-egress-proxy-propagation-contract.md). + ## Documentation | Doc | Purpose | diff --git a/docs/adr/0025-egress-proxy-propagation-contract.md b/docs/adr/0025-egress-proxy-propagation-contract.md index b771213..207b347 100644 --- a/docs/adr/0025-egress-proxy-propagation-contract.md +++ b/docs/adr/0025-egress-proxy-propagation-contract.md @@ -57,8 +57,14 @@ path is exempt: `reqwest` implements proxy auth. - **Breaking:** `ChallengeHandler::solve` and `SolveDispatcher::solve` change signature; every handler crate (`px-perimeterx`, `px-cloudflare`, `px-native`, and the `px-turnstile` / - `px-captcha` / `px-datadome` stubs) is updated in the same change. Per ADR-0017 this is a - post-1.0.0 architectural change → manual `major` bump. + `px-captcha` / `px-datadome` stubs) is updated in the same change. +- **Shipped as `1.9.0`, not `2.0.0`** — maintainer's call, amending the ADR-0017 line that a + post-1.0.0 architectural change takes a manual `major`. The trade is accepted knowingly: + downstream users pinning `pxsolver-* = "1"` get a compile error on `cargo update` rather than an + opt-in major. Anyone implementing `ChallengeHandler` outside this workspace has to add the + `SolveAction` parameter; nothing else in the published surface moves. +- Internal `[workspace.dependencies]` pins had been stale at `1.4.0` since that release and now + track the workspace version; `xtask bump` re-pins them from here on. - `docs/deployment.md` gains an "Egress proxies" section correcting the old `N × len(proxies)` rotation claim: a session takes its proxy at spawn and keeps it until the 300s TTL, so distinct egress IPs per domain is `min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES))`. diff --git a/docs/adr/README.md b/docs/adr/README.md index cf08cd8..fe73594 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -38,7 +38,7 @@ Format: [MADR](https://adr.github.io/madr/) lite. One file per decision, never e | [0022](0022-readmit-pedidosya-to-canary-with-deep-stealth-budget.md) | Re-admit pedidosya to canary with relaxed AC-2 budget (15s median / 20s p95) for CF-fronted targets; amends ADR-0018 | Accepted | 2026-05-17 | | [0023](0023-allowlist-handler-field-supersedes-env-csv.md) | `handler:` field in allowlist.yaml supersedes `PX_CAMOUFOX_DOMAINS` env CSV; env retained as deprecated fallback through v1.x | Accepted | 2026-05-17 | | [0024](0024-activate-native-px3-sensor-synthesis.md) | Activate native px-3 sensor synthesis; promote ADR-0010 | Proposed | 2026-05-20 | -| [0025](0025-egress-proxy-propagation-contract.md) | Egress proxy propagation: per-request for `/v1/solve`, session rotation for `/v1/fetch`; egress in the cache key; credentials stripped for browser paths | Accepted | 2026-08-07 | +| [0025](0025-egress-proxy-propagation-contract.md) | Egress proxy propagation: per-request for `/v1/solve`, session rotation for `/v1/fetch`; egress in the cache key; credentials stripped for browser paths; shipped as 1.9.0 | Accepted | 2026-08-07 | ## Template From 20944315098dce0ffbd99e0a784db9e91b9c1722 Mon Sep 17 00:00:00 2001 From: KeyCode17 Date: Sat, 8 Aug 2026 00:26:33 +0700 Subject: [PATCH 7/7] chore(deps): clear RUSTSEC-2026-0185 and RUSTSEC-2026-0190 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit anyhow 1.0.102 → 1.0.104 (unsound Error::downcast_mut) and quinn-proto 0.11.14 → 0.11.16 (remote memory exhaustion from unbounded out-of-order stream reassembly). Both are transitive and predate this branch — main carries the same lock entries and fails the same audit, since the advisory database is fetched fresh on every run. Lock-only; no manifest requirement moves. --- Cargo.lock | 70 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7987e0..9d97658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,9 +69,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "argon2" @@ -81,7 +81,7 @@ checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures", + "cpufeatures 0.2.17", "password-hash", ] @@ -283,6 +283,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chromiumoxide" version = "0.9.1" @@ -466,6 +477,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -775,11 +795,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -789,10 +807,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1724,14 +1745,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -1788,6 +1810,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -1813,6 +1846,21 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2146,7 +2194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2611,7 +2659,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand", + "rand 0.9.4", "sha1", "thiserror 2.0.18", "utf-8",