diff --git a/Cargo.lock b/Cargo.lock index 5d4acbf..9d97658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,9 +69,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "argon2" @@ -81,7 +81,7 @@ checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures", + "cpufeatures 0.2.17", "password-hash", ] @@ -283,6 +283,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chromiumoxide" version = "0.9.1" @@ -466,6 +477,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -775,11 +795,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -789,10 +807,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1464,12 +1485,13 @@ dependencies = [ [[package]] name = "px-cli" -version = "1.8.0" +version = "1.9.0" dependencies = [ "anyhow", "argon2", "clap", "pxsolver-auth", + "pxsolver-core", "pxsolver-detector", "pxsolver-native", "reqwest 0.12.28", @@ -1482,7 +1504,7 @@ dependencies = [ [[package]] name = "px-server" -version = "1.8.0" +version = "1.9.0" dependencies = [ "anyhow", "argon2", @@ -1515,7 +1537,7 @@ dependencies = [ [[package]] name = "pxsolver-auth" -version = "1.8.0" +version = "1.9.0" dependencies = [ "argon2", "async-trait", @@ -1532,7 +1554,7 @@ dependencies = [ [[package]] name = "pxsolver-cache" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "dashmap", @@ -1544,7 +1566,7 @@ dependencies = [ [[package]] name = "pxsolver-camoufox" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "fantoccini", @@ -1562,7 +1584,7 @@ dependencies = [ [[package]] name = "pxsolver-captcha" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1571,7 +1593,7 @@ dependencies = [ [[package]] name = "pxsolver-cloudflare" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1583,15 +1605,16 @@ dependencies = [ [[package]] name = "pxsolver-core" -version = "1.8.0" +version = "1.9.0" dependencies = [ "serde", + "serde_json", "uuid", ] [[package]] name = "pxsolver-datadome" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1600,7 +1623,7 @@ dependencies = [ [[package]] name = "pxsolver-detector" -version = "1.8.0" +version = "1.9.0" dependencies = [ "pxsolver-core", "regex", @@ -1608,7 +1631,7 @@ dependencies = [ [[package]] name = "pxsolver-errors" -version = "1.8.0" +version = "1.9.0" dependencies = [ "axum", "serde", @@ -1617,7 +1640,7 @@ dependencies = [ [[package]] name = "pxsolver-harvester" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "chromiumoxide", @@ -1630,7 +1653,7 @@ dependencies = [ [[package]] name = "pxsolver-native" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "futures", @@ -1649,7 +1672,7 @@ dependencies = [ [[package]] name = "pxsolver-perimeterx" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1663,7 +1686,7 @@ dependencies = [ [[package]] name = "pxsolver-pipeline" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-core", @@ -1675,7 +1698,7 @@ dependencies = [ [[package]] name = "pxsolver-turnstile" -version = "1.8.0" +version = "1.9.0" dependencies = [ "async-trait", "pxsolver-errors", @@ -1684,7 +1707,7 @@ dependencies = [ [[package]] name = "pxsolver-types" -version = "1.8.0" +version = "1.9.0" dependencies = [ "serde", "serde_json", @@ -1692,7 +1715,7 @@ dependencies = [ [[package]] name = "pxsolver-validation" -version = "1.8.0" +version = "1.9.0" dependencies = [ "axum", "pxsolver-errors", @@ -1722,14 +1745,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -1786,6 +1810,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -1811,6 +1846,21 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2144,7 +2194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2609,7 +2659,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand", + "rand 0.9.4", "sha1", "thiserror 2.0.18", "utf-8", diff --git a/Cargo.toml b/Cargo.toml index acbf11a..584641e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "1.8.0" +version = "1.9.0" edition = "2024" rust-version = "1.95" license = "AGPL-3.0-or-later" @@ -38,22 +38,22 @@ unsafe_code = "forbid" # so `cargo publish` can resolve to the registry copy. Each entry's package # = pxsolver-* (the crates.io name); the local alias `px-*` is kept so # source files can continue to `use px_core::…` unchanged. -px-core = { path = "px-core", version = "1.4.0", package = "pxsolver-core" } -px-types = { path = "px-types", version = "1.4.0", package = "pxsolver-types" } -px-errors = { path = "px-errors", version = "1.4.0", package = "pxsolver-errors" } -px-validation = { path = "px-validation", version = "1.4.0", package = "pxsolver-validation" } -px-cache = { path = "px-cache", version = "1.4.0", package = "pxsolver-cache" } -px-detector = { path = "px-detector", version = "1.4.0", package = "pxsolver-detector" } -px-harvester = { path = "px-harvester", version = "1.4.0", package = "pxsolver-harvester" } -px-pipeline = { path = "px-pipeline", version = "1.4.0", package = "pxsolver-pipeline" } -px-perimeterx = { path = "px-perimeterx", version = "1.4.0", package = "pxsolver-perimeterx" } -px-cloudflare = { path = "px-cloudflare", version = "1.4.0", package = "pxsolver-cloudflare" } -px-turnstile = { path = "px-turnstile", version = "1.4.0", package = "pxsolver-turnstile" } -px-captcha = { path = "px-captcha", version = "1.4.0", package = "pxsolver-captcha" } -px-datadome = { path = "px-datadome", version = "1.4.0", package = "pxsolver-datadome" } -px-native = { path = "px-native", version = "1.4.0", package = "pxsolver-native" } -px-auth = { path = "px-auth", version = "1.4.0", package = "pxsolver-auth" } -px-camoufox = { path = "px-camoufox", version = "1.4.0", package = "pxsolver-camoufox" } +px-core = { path = "px-core", version = "1.9.0", package = "pxsolver-core" } +px-types = { path = "px-types", version = "1.9.0", package = "pxsolver-types" } +px-errors = { path = "px-errors", version = "1.9.0", package = "pxsolver-errors" } +px-validation = { path = "px-validation", version = "1.9.0", package = "pxsolver-validation" } +px-cache = { path = "px-cache", version = "1.9.0", package = "pxsolver-cache" } +px-detector = { path = "px-detector", version = "1.9.0", package = "pxsolver-detector" } +px-harvester = { path = "px-harvester", version = "1.9.0", package = "pxsolver-harvester" } +px-pipeline = { path = "px-pipeline", version = "1.9.0", package = "pxsolver-pipeline" } +px-perimeterx = { path = "px-perimeterx", version = "1.9.0", package = "pxsolver-perimeterx" } +px-cloudflare = { path = "px-cloudflare", version = "1.9.0", package = "pxsolver-cloudflare" } +px-turnstile = { path = "px-turnstile", version = "1.9.0", package = "pxsolver-turnstile" } +px-captcha = { path = "px-captcha", version = "1.9.0", package = "pxsolver-captcha" } +px-datadome = { path = "px-datadome", version = "1.9.0", package = "pxsolver-datadome" } +px-native = { path = "px-native", version = "1.9.0", package = "pxsolver-native" } +px-auth = { path = "px-auth", version = "1.9.0", package = "pxsolver-auth" } +px-camoufox = { path = "px-camoufox", version = "1.9.0", package = "pxsolver-camoufox" } anyhow = "1.0" argon2 = "0.5" diff --git a/README.md b/README.md index 6a096f0..69d095d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ A Rust-built solver service for PerimeterX (HUMAN Security) protection. Given a target URL on a per-domain allowlist, returns a valid `_px3` cookie bundle that a downstream authorized client can use to issue requests as if from a real browser. -> **Status:** v1.2.0 published to crates.io as the `pxsolver-*` family of crates. MVP gate hit at v1.0.0; v1.1.0 added a Camoufox-backed Cloudflare bypass path; v1.2.0 renamed the published library crates. See [GitHub Releases](https://github.com/KeyCode17/px-solver/releases) for the per-version notes. +> **Status:** v1.9.0, published to crates.io as the `pxsolver-*` family of crates. MVP gate hit at v1.0.0; v1.1.0 added a Camoufox-backed Cloudflare bypass path; v1.2.0 renamed the published library crates; v1.8.0 activated native `_px3` sensor synthesis ([ADR-0024](docs/adr/0024-activate-native-px3-sensor-synthesis.md)); v1.9.0 makes the per-request egress proxy real end to end ([ADR-0025](docs/adr/0025-egress-proxy-propagation-contract.md)). See [GitHub Releases](https://github.com/KeyCode17/px-solver/releases) for the per-version notes. +> +> ⚠️ **v1.9.0 is source-breaking despite the minor bump.** `ChallengeHandler::solve` and `SolveDispatcher::solve` take a `SolveAction` / `SolveRequest` instead of `&PageHtml` / `&str`. If you implement `ChallengeHandler` outside this workspace, add the parameter when upgrading; the rest of the published surface is unchanged. ## What this is @@ -83,6 +85,35 @@ The 16 `pxsolver-*` library crates are also published individually for downstrea For systemd, reverse proxy, and key rotation workflows see [`docs/deployment.md`](docs/deployment.md). +## Proxies + +Two mechanisms, one per endpoint. They are **not** interchangeable, and neither one falls back to the other. + +| | `/v1/solve` | `/v1/fetch` | +|---|---|---| +| Assigned by | `"proxy"` in the request body, or `px-cli solve --proxy` | `PX_PROXIES` env var (CSV), operator-side | +| Chosen per | request | Camoufox session, at spawn | +| Rotation | none — the solve uses exactly the proxy you named | round-robin across the list | +| Omitted | server's own IP | server's own IP | + +```bash +# /v1/solve — name the egress you will send downstream traffic through +-d '{"url":"https://www.pedidosya.com.ar/","proxy":"socks5://127.0.0.1:9050"}' + +# /v1/fetch — operator-side rotation across warm sessions +PX_PROXIES="http://p1.example:8080,socks5://p2.example:1080" ./target/release/px-server +``` + +A `_px3` bundle is bound to the IP that earned it, so **use the same proxy downstream that you named on the solve**. The proxy is part of the cache key: the same domain solved through two proxies yields two entries and neither is served to the other. + +Three things that surprise people: + +- **`PX_PROXIES` does nothing for `/v1/solve`.** Rotation there would hand you a bundle bound to an IP you cannot know. +- **Browser proxies cannot authenticate.** geckodriver's W3C `proxy` capability has no credential field and Chromium ignores userinfo without a CDP `Fetch.authRequired` handler, so `user:pass@` is stripped with a warning. Front an authenticated upstream with a local relay (gost, 3proxy). The native sensor path goes over `reqwest` and *does* accept credentials. +- **Distinct egress IPs per domain on `/v1/fetch` is `min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES))`**, not the product — a session takes its proxy at spawn and keeps it for the 300s TTL. + +Full reference: [Egress proxies](docs/deployment.md#egress-proxies) · rationale: [ADR-0025](docs/adr/0025-egress-proxy-propagation-contract.md). + ## Documentation | Doc | Purpose | @@ -90,7 +121,7 @@ For systemd, reverse proxy, and key rotation workflows see [`docs/deployment.md` | [`docs/000-sow-index.md`](docs/000-sow-index.md) | Statement of Work index + deliverable traceability | | [`docs/adr/README.md`](docs/adr/README.md) | Architecture Decision Records (23 ADRs as of 2026-05-17) | | [`docs/phase/README.md`](docs/phase/README.md) | Phase plan (00–04 critical path + R research) | -| [`docs/deployment.md`](docs/deployment.md) | Fresh-Linux install, systemd, reverse proxy, key generation, allowlist editing | +| [`docs/deployment.md`](docs/deployment.md) | Fresh-Linux install, systemd, reverse proxy, key generation, allowlist editing, egress proxies | | [`docs/threat-model.md`](docs/threat-model.md) | Misuse vectors + mitigations | | [`docs/dual-use-policy.md`](docs/dual-use-policy.md) | Operator commitments per [`docs/011-sow-dual-use.md`](docs/011-sow-dual-use.md) | | [`docs/standards/axum-best-practice.md`](docs/standards/axum-best-practice.md) | Coding standard (Clean Architecture, ≤200 LOC/file, no `unwrap`) | diff --git a/docs/adr/0025-egress-proxy-propagation-contract.md b/docs/adr/0025-egress-proxy-propagation-contract.md new file mode 100644 index 0000000..207b347 --- /dev/null +++ b/docs/adr/0025-egress-proxy-propagation-contract.md @@ -0,0 +1,82 @@ +# 0025. Egress proxy propagation: per-request for solve, session rotation for fetch + +- **Date:** 2026-08-07 +- **Status:** Accepted +- **Deciders:** KeyCode17 +- **Related:** ADR-0014 (challenge pipeline), ADR-0020 (Camoufox), ADR-0021 (handler routing), + ADR-0024 (native sensor), [`docs/deployment.md`](../deployment.md) + +## Context + +Four proxy surfaces shipped through v1.8.0 and only one reached a browser: + +| Surface | Status before this ADR | +|---|---| +| `PX_PROXIES` → `ProxyPool` → `SessionPool` | worked, `/v1/fetch` sessions only | +| `POST /v1/solve` body `"proxy"` | deserialized into `SolveRequestDto`, never read | +| `px-cli solve --proxy` | sent in the body, dropped server-side | +| `pxsolver_core::SolveRequest::with_proxy()` | published builder with zero consumers | +| `HarvestRequest.proxy` | honoured by `CamoufoxPool`, ignored by `ChromiumoxidePool` | + +The cause was structural, not an oversight at one call site: `SolveDispatcher::solve(&self, url: &str)` +and `ChallengeHandler::solve(&self, page: &PageHtml)` had no parameter a proxy could travel in, and +both browser handlers built `HarvestRequest::new(url)`, which defaults `proxy` to `None`. Two +implementations of one `Harvester` trait also disagreed on whether the field meant anything. + +Downstream Rust users were the worst affected: `SolveRequest::with_proxy()` is the obvious API to +reach for, is published on crates.io, and did nothing. + +## Decision + +**1. The solve path carries a per-request proxy end to end.** `ChallengeHandler::solve` takes a +`SolveAction { page, proxy }` — a domain action struct built at the infrastructure edge and passed +inward. `SolveDispatcher::solve` takes `px_core::SolveRequest`, so the published builder is the type +the HTTP edge maps into. Both browser handlers forward it via `HarvestRequest::with_proxy`, and +`ChromiumoxidePool` gained the `--proxy-server` argument it never had. + +**2. The native sensor path honours it too.** `SolveContext` carries the proxy and +`SensorNativeSolver` posts through a per-proxy `reqwest::Client`, cached in `ProxyClients` so the +pooled TLS connection survives repeat solves. + +**3. Solving never falls back to the `PX_PROXIES` rotation.** `_px3` is bound to the IP that earned +it. A caller who did not name an egress cannot route downstream traffic through a rotating one, so a +bundle harvested that way would not be usable. Rotation stays where it is useful: long-lived +`/v1/fetch` Camoufox sessions, which own their cookie jar and consume the bundle themselves. + +**4. The egress is part of the cache key.** `sentinel_cache_key(domain, proxy)` hashes the proxy into +`fp_key`; a direct solve keeps `0`, so pre-existing keys still resolve. Without this, a bundle earned +through proxy A would be replayed to a caller who asked for proxy B. + +**5. Proxy credentials are stripped for browser paths, with a warning.** geckodriver's W3C `proxy` +capability has no credential field and Chromium's `--proxy-server` ignores userinfo without a CDP +`Fetch.authRequired` handler. `strip_credentials` (px-harvester) removes `user:pass@` and logs the +sanitized URL. Authenticated upstreams are used through a local unauthenticated relay. The native +path is exempt: `reqwest` implements proxy auth. + +## Consequences + +- **Breaking:** `ChallengeHandler::solve` and `SolveDispatcher::solve` change signature; every + handler crate (`px-perimeterx`, `px-cloudflare`, `px-native`, and the `px-turnstile` / + `px-captcha` / `px-datadome` stubs) is updated in the same change. +- **Shipped as `1.9.0`, not `2.0.0`** — maintainer's call, amending the ADR-0017 line that a + post-1.0.0 architectural change takes a manual `major`. The trade is accepted knowingly: + downstream users pinning `pxsolver-* = "1"` get a compile error on `cargo update` rather than an + opt-in major. Anyone implementing `ChallengeHandler` outside this workspace has to add the + `SolveAction` parameter; nothing else in the published surface moves. +- Internal `[workspace.dependencies]` pins had been stale at `1.4.0` since that release and now + track the workspace version; `xtask bump` re-pins them from here on. +- `docs/deployment.md` gains an "Egress proxies" section correcting the old + `N × len(proxies)` rotation claim: a session takes its proxy at spawn and keeps it until the 300s + TTL, so distinct egress IPs per domain is `min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES))`. +- `/v1/fetch` deliberately gains **no** per-request proxy field: switching a warm session's egress + mid-life means respawning the browser and discarding the cookie jar it exists to hold. + +## Alternatives considered + +- **Delete the dead surfaces.** Smaller diff, but removes a published `pxsolver-core` API and two + documented operator-facing options — a bigger break for downstream users than wiring them up. +- **Rotate on solve as well, and report the chosen egress in the response.** Needs the resolved + proxy threaded back through `HarvestResult` → `HandlerOutcome` → `SolveOutput` → DTO. Deferred: + no operator has asked for solve-side rotation, and per-request assignment already covers it. +- **Per-request proxy on `/v1/fetch`, keying sessions by `(domain, proxy)`.** Rejected for now; it + multiplies live browsers per domain and complicates TTL eviction for a case the env list covers. diff --git a/docs/adr/README.md b/docs/adr/README.md index d988a72..fe73594 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -37,6 +37,8 @@ Format: [MADR](https://adr.github.io/madr/) lite. One file per decision, never e | [0021](0021-domain-based-handler-routing.md) | Domain-based handler routing in the solve dispatcher (PX_CAMOUFOX_DOMAINS env CSV → CloudflareHandler) | Accepted | 2026-05-17 | | [0022](0022-readmit-pedidosya-to-canary-with-deep-stealth-budget.md) | Re-admit pedidosya to canary with relaxed AC-2 budget (15s median / 20s p95) for CF-fronted targets; amends ADR-0018 | Accepted | 2026-05-17 | | [0023](0023-allowlist-handler-field-supersedes-env-csv.md) | `handler:` field in allowlist.yaml supersedes `PX_CAMOUFOX_DOMAINS` env CSV; env retained as deprecated fallback through v1.x | Accepted | 2026-05-17 | +| [0024](0024-activate-native-px3-sensor-synthesis.md) | Activate native px-3 sensor synthesis; promote ADR-0010 | Proposed | 2026-05-20 | +| [0025](0025-egress-proxy-propagation-contract.md) | Egress proxy propagation: per-request for `/v1/solve`, session rotation for `/v1/fetch`; egress in the cache key; credentials stripped for browser paths; shipped as 1.9.0 | Accepted | 2026-08-07 | ## Template diff --git a/docs/deployment.md b/docs/deployment.md index e5457cf..3e2b601 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -127,18 +127,74 @@ The optional `handler:` field (added in v1.1.x per [ADR-0023](adr/0023-allowlist Currently the server reads `PX_BIND`, `PX_KEYS`, `PX_ALLOWLIST` env vars. A YAML config file is reserved for future use. -### Egress proxy rotation (optional) +## Egress proxies -For sustained `/v1/fetch` traffic against rate-limiting WAFs (pedidosya's PerimeterX flags a single IP after ~30 fetches/min), set `PX_PROXIES` to a CSV list of proxy URLs. Each persistent Camoufox session for a CF-routed domain is assigned a proxy round-robin from the list: +There are **two** proxy mechanisms and they do different jobs. Pick by endpoint: + +| | `/v1/solve` | `/v1/fetch` | +|---|---|---| +| How you assign it | `"proxy"` in the request body (or `px-cli solve --proxy`) | `PX_PROXIES` env var, operator-side | +| Chosen per | request | Camoufox session | +| Rotation | none — the solve uses exactly the proxy you named | round-robin across the list | +| Omitted → | server's own IP | server's own IP | + +### `/v1/solve` — per-request proxy + +`_px3` is bound to the IP that earned it, so a bundle is only usable from that same egress. Name the proxy you intend to send downstream traffic through: ```bash -PX_PROXIES="http://user:pass@proxy1.example:8080,socks5://user:pass@proxy2.example:1080" \ +curl -X POST http://127.0.0.1:8080/v1/solve \ + -H "Authorization: Bearer ops1:" \ + -H "content-type: application/json" \ + -d '{"url":"https://www.pedidosya.com.ar/","proxy":"socks5://127.0.0.1:9050"}' +``` + +Accepted schemes: `http`, `https`, `socks5`, `socks5h`. Chromium has no `socks5h` scheme and silently ignores a spec it cannot parse, so the Chromium path rewrites it to `socks5://` rather than going direct without saying so. `"proxy":null` (or omitting the field) harvests from the server's own address. + +The solve **never** falls back to the `PX_PROXIES` rotation — a bundle earned through an IP the caller cannot name would not be usable. The proxy is part of the cache key, so the same domain solved through two different proxies produces two entries and neither is served to the other. + +Rust callers build the same request through the published crate: + +```rust +use pxsolver_core::SolveRequest; + +let req = SolveRequest::new("https://www.pedidosya.com.ar/") + .with_proxy("socks5://127.0.0.1:9050"); +``` + +### `/v1/fetch` — session rotation via `PX_PROXIES` + +For sustained `/v1/fetch` traffic against rate-limiting WAFs (pedidosya's PerimeterX flags a single IP after ~30 fetches/min), set `PX_PROXIES` to a CSV list. Each persistent Camoufox session for a CF-routed domain is assigned one proxy round-robin at spawn: + +```bash +PX_PROXIES="http://proxy1.example:8080,socks5://proxy2.example:1080" \ ./target/release/px-server ``` -Empty / unset → direct connection (no rotation). Both `http://` and `socks5://` schemes are accepted by the underlying geckodriver capability. With `PX_FETCH_MAX_PER_DOMAIN=N` (default 2), the pool spawns up to N browsers per domain, each binding to the next proxy in the rotation; the operator's effective concurrency is `N × len(proxies)` parallel egress paths before round-robin reuse kicks in. +Empty / unset → direct connection (no rotation). `/v1/fetch` has no per-request proxy field: a session is reused for its warm cookie jar, and switching its egress mid-life would mean respawning the browser. + +**How many IPs you actually get.** A session takes a proxy when it is *spawned*, and keeps it until it ages out (300s TTL). Once a domain holds `PX_FETCH_MAX_PER_DOMAIN=N` sessions (default 2), further requests round-robin those existing sessions and no new proxy is drawn. Distinct egress IPs per domain is therefore: + +``` +min(PX_FETCH_MAX_PER_DOMAIN, len(PX_PROXIES)) +``` + +Ten proxies with the default `N=2` gives a domain **two** IPs, not twenty. Raise `PX_FETCH_MAX_PER_DOMAIN` to use more of the list — each extra session is another live browser, so size it against RAM. The cursor is shared across domains, so proxy *k* is not reserved for any one target. + +### Proxy credentials + +**Browsers cannot authenticate to a proxy here.** geckodriver's W3C `proxy` capability has no credential field, and Chromium's `--proxy-server` ignores userinfo without a CDP `Fetch.authRequired` handler. `user:pass@` in a `PX_PROXIES` entry or in a `/v1/solve` proxy is **stripped**, and the server logs a warning naming the sanitized URL. + +To use an authenticated upstream, front it with a local unauthenticated relay and point px-solver at the relay: + +```bash +gost -L=socks5://127.0.0.1:1080 -F='http://user:pass@upstream.example:8080' +PX_PROXIES="socks5://127.0.0.1:1080" ./target/release/px-server +``` + +The one exception is the native sensor path (`PX_NATIVE_PROFILES`), which posts over `reqwest` and does support proxy authentication; a credentialed proxy works there and only there. -Tor as a quick test: install `tor`, let it bind `socks5://127.0.0.1:9050`, set `PX_PROXIES="socks5://127.0.0.1:9050"`. Many sites block Tor exit IPs; treat it as a fingerprint smoke-test rather than a production proxy. +Tor as a quick test: install `tor`, let it bind `socks5://127.0.0.1:9050`, set `PX_PROXIES="socks5://127.0.0.1:9050"` or pass it as a per-request proxy. Many sites block Tor exit IPs; treat it as a fingerprint smoke-test rather than a production proxy. ## Run diff --git a/docs/runbook-native-bypass.md b/docs/runbook-native-bypass.md index ee607ed..2d6263d 100644 --- a/docs/runbook-native-bypass.md +++ b/docs/runbook-native-bypass.md @@ -7,7 +7,12 @@ This is the end-to-end procedure for taking the native PX path from tenant". It assumes: - A working AR (or other tenant-appropriate) residential proxy is - available — set `PX_PROXIES=socks5://…` in your shell. + available — set `PX_PROXIES=socks5://…` in your shell. Steps 1 and 4 + read it directly (first CSV entry). The **server** does not apply it to + `/v1/solve`: there the egress is per request, named in the request body + (see [Egress proxies](deployment.md#egress-proxies)). Unlike the browser + paths, the native sensor POST goes through `reqwest` and does accept + `user:pass@` credentials. - Camoufox + geckodriver are installed and `CamoufoxConfig::from_env()` resolves them. - You have the `eT15wiaE` (pedidosya) profile at @@ -68,6 +73,16 @@ The dispatcher will try the native handler first for any solve targeting `pedidosya.com.ar` and fall back to the existing Camoufox path on error or non-solved status. +Name the egress per request — both the native sensor POST and the +Camoufox fallback use it, and the returned bundle is bound to that IP: + +```bash +curl -X POST http://127.0.0.1:8080/v1/solve \ + -H "Authorization: Bearer ops1:" \ + -H "content-type: application/json" \ + -d "{\"url\":\"https://www.pedidosya.com.ar/\",\"proxy\":\"$PX_PROXIES\"}" +``` + ## Step 4 — Throughput soak ```bash @@ -80,8 +95,10 @@ NATIVE_SOAK=1 \ cargo test -q -p pxsolver-native --test throughput_soak -- --ignored --nocapture ``` -The soak runs `SensorNativeSolver::solve` 80× through your live proxy -and asserts ≥40 req/min sustained throughput. Output: +The soak runs `SensorNativeSolver::solve` 80× and asserts ≥40 req/min +sustained throughput. It leaves through `NATIVE_SOAK_PROXY`, or the first +`PX_PROXIES` entry, or the host's own IP if neither is set — the run +prints which. Output: ``` === NATIVE_SOAK === diff --git a/px-camoufox/src/infrastructure/camoufox_pool.rs b/px-camoufox/src/infrastructure/camoufox_pool.rs index 9f20f5e..094aeb9 100644 --- a/px-camoufox/src/infrastructure/camoufox_pool.rs +++ b/px-camoufox/src/infrastructure/camoufox_pool.rs @@ -3,7 +3,7 @@ use crate::infrastructure::caps::{build_capabilities, pick_free_port, wait_for_g use async_trait::async_trait; use fantoccini::ClientBuilder; use px_errors::AppError; -use px_harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; +use px_harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester, strip_credentials}; use serde_json::{Map, Value}; use std::sync::Arc; use std::time::Duration; @@ -25,7 +25,7 @@ impl CamoufoxPool { pub fn new(config: CamoufoxConfig) -> Result { config .validate() - .map_err(|e| AppError::InternalError(format!("camoufox config: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Camoufox config: {e}")))?; let permits = Arc::new(Semaphore::new(config.max_concurrent)); let max_per_domain = std::env::var("PX_FETCH_MAX_PER_DOMAIN") .ok() @@ -35,6 +35,8 @@ impl CamoufoxPool { if !proxies.is_empty() { tracing::info!( count = proxies.len(), + max_per_domain, + distinct_egress_per_domain = max_per_domain.min(proxies.len()), "proxy rotation enabled for /v1/fetch sessions" ); } @@ -76,7 +78,7 @@ impl CamoufoxPool { .permits .acquire() .await - .map_err(|e| AppError::InternalError(format!("semaphore: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Semaphore: {e}")))?; let port = pick_free_port().await?; let mut child = Command::new(&self.config.geckodriver_bin) .arg("--port") @@ -87,7 +89,7 @@ impl CamoufoxPool { .stderr(std::process::Stdio::null()) .kill_on_drop(true) .spawn() - .map_err(|e| AppError::InternalError(format!("spawn geckodriver: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Spawn geckodriver: {e}")))?; wait_for_geckodriver(port, Duration::from_secs(15)).await?; let caps = build_capabilities(&self.config, proxy); let endpoint = format!("http://127.0.0.1:{port}"); @@ -101,7 +103,12 @@ impl CamoufoxPool { impl Harvester for CamoufoxPool { async fn harvest(&self, req: HarvestRequest) -> Result { let navigate_timeout = self.config.navigate_timeout; - let proxy = req.proxy.clone(); + let proxy = req.proxy.clone().map(strip_credentials); + tracing::info!( + url = %req.url, + proxy = proxy.as_deref().unwrap_or("direct"), + "camoufox harvest starting" + ); self.with_session(proxy.as_deref(), async move |endpoint, caps| { harvest_session(&endpoint, caps, &req, navigate_timeout).await }) @@ -119,26 +126,26 @@ async fn harvest_session( .capabilities(caps) .connect(endpoint) .await - .map_err(|e| AppError::InternalError(format!("webdriver connect: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Webdriver connect: {e}")))?; let nav = client.goto(&req.url); if tokio::time::timeout(navigate_timeout, nav).await.is_err() { let _ = client.close().await; - return Err(AppError::InternalError("navigate timeout".into())); + return Err(AppError::InternalError("Navigate timeout".into())); } sleep(Duration::from_millis(req.wait_ms)).await; let html = client .source() .await - .map_err(|e| AppError::InternalError(format!("source: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Source: {e}")))?; let ua_val = client .execute("return navigator.userAgent;", vec![]) .await - .map_err(|e| AppError::InternalError(format!("ua eval: {e}")))?; + .map_err(|e| AppError::InternalError(format!("User agent eval: {e}")))?; let user_agent = ua_val.as_str().unwrap_or("").to_string(); let raw_cookies = client .get_all_cookies() .await - .map_err(|e| AppError::InternalError(format!("cookies: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Cookies: {e}")))?; let cookies = raw_cookies .into_iter() .map(|c| HarvestedCookie { diff --git a/px-camoufox/src/infrastructure/session_pool.rs b/px-camoufox/src/infrastructure/session_pool.rs index 7a9c314..f1efed4 100644 --- a/px-camoufox/src/infrastructure/session_pool.rs +++ b/px-camoufox/src/infrastructure/session_pool.rs @@ -91,14 +91,4 @@ impl SessionPool { let idx = slot.cursor.fetch_add(1, Ordering::Relaxed) % slot.sessions.len(); Ok(Arc::clone(&slot.sessions[idx])) } - - #[allow(dead_code)] - pub(crate) async fn total_sessions(&self) -> usize { - self.domains - .lock() - .await - .values() - .map(|slot| slot.sessions.len()) - .sum() - } } diff --git a/px-captcha/src/lib.rs b/px-captcha/src/lib.rs index 5292831..d1352b7 100644 --- a/px-captcha/src/lib.rs +++ b/px-captcha/src/lib.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml, SolveAction}; pub struct CaptchaHandler; @@ -30,7 +30,7 @@ impl ChallengeHandler for CaptchaHandler { || h.contains("class=\"g-recaptcha\"")) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented(self.name())) } } diff --git a/px-cli/Cargo.toml b/px-cli/Cargo.toml index 4118070..61e36ef 100644 --- a/px-cli/Cargo.toml +++ b/px-cli/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } argon2 = { workspace = true } clap = { workspace = true } px-auth = { workspace = true } +px-core = { workspace = true } px-detector = { workspace = true } px-native = { workspace = true } reqwest = { workspace = true } diff --git a/px-cli/src/cli.rs b/px-cli/src/cli.rs index ea02c32..652e66e 100644 --- a/px-cli/src/cli.rs +++ b/px-cli/src/cli.rs @@ -89,7 +89,10 @@ pub struct SolveArgs { /// API key as `id:secret`. #[arg(long, env = "PX_API_KEY")] pub api_key: String, - /// Optional upstream proxy passed to the solver. + /// Egress proxy the solver harvests through, as + /// `scheme://host:port` (http, https, socks5, socks5h). The returned + /// bundle is bound to that IP, so send downstream requests through the + /// same proxy. Omit to harvest from the server's own address. #[arg(long)] pub proxy: Option, } diff --git a/px-cli/src/commands/solve.rs b/px-cli/src/commands/solve.rs index cce7588..f626e78 100644 --- a/px-cli/src/commands/solve.rs +++ b/px-cli/src/commands/solve.rs @@ -1,15 +1,9 @@ use anyhow::{Context, Result, bail}; -use serde::{Deserialize, Serialize}; +use px_core::SolveRequest; +use serde::Deserialize; use crate::cli::SolveArgs; -#[derive(Debug, Serialize)] -struct SolveRequest<'a> { - url: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - proxy: Option<&'a str>, -} - #[derive(Debug, Deserialize)] struct SolveEnvelope { data: serde_json::Value, @@ -28,10 +22,7 @@ pub async fn run(args: SolveArgs) -> Result<()> { bail!("--api-key (or PX_API_KEY) must be in the form `id:secret`"); } let endpoint = format!("{}/v1/solve", server.trim_end_matches('/')); - let body = SolveRequest { - url: &url, - proxy: proxy.as_deref(), - }; + let body = SolveRequest::new(&url).with_proxy_opt(proxy); let client = reqwest::Client::builder() .build() .context("build http client")?; diff --git a/px-cloudflare/Cargo.toml b/px-cloudflare/Cargo.toml index b2bcf98..770a6f6 100644 --- a/px-cloudflare/Cargo.toml +++ b/px-cloudflare/Cargo.toml @@ -23,7 +23,7 @@ px-harvester = { workspace = true } px-pipeline = { workspace = true } [dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt", "sync"] } [lib] name = "px_cloudflare" diff --git a/px-cloudflare/src/lib.rs b/px-cloudflare/src/lib.rs index 73c3b71..61f6c31 100644 --- a/px-cloudflare/src/lib.rs +++ b/px-cloudflare/src/lib.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use px_core::{CookieJarDelta, NamedCookie}; use px_errors::AppError; use px_harvester::{HarvestRequest, Harvester}; -use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml, SolveAction}; use std::sync::Arc; use std::time::Instant; @@ -53,12 +53,13 @@ impl ChallengeHandler for CloudflareHandler { || h.contains("cf_clearance")) } - async fn solve(&self, page: &PageHtml) -> Result { + async fn solve(&self, action: &SolveAction) -> Result { let Some(harvester) = self.harvester.as_ref() else { return Ok(HandlerOutcome::not_implemented(self.name())); }; let start = Instant::now(); - let result = harvester.harvest(HarvestRequest::new(&page.url)).await?; + let request = HarvestRequest::new(action.url()).with_proxy(action.proxy.clone()); + let result = harvester.harvest(request).await?; let session_cookies: Vec = extract_session_cookies(&result.cookies) .into_iter() .map(|c| NamedCookie { @@ -92,16 +93,30 @@ impl ChallengeHandler for CloudflareHandler { mod tests { use super::*; use px_harvester::{HarvestResult, HarvestedCookie}; + use tokio::sync::Mutex; struct FakeHarvester { ua: String, cookies: Vec, html: String, + seen_proxy: Mutex>, + } + + impl FakeHarvester { + fn new(ua: &str, cookies: Vec, html: &str) -> Self { + Self { + ua: ua.into(), + cookies, + html: html.into(), + seen_proxy: Mutex::new(None), + } + } } #[async_trait] impl Harvester for FakeHarvester { - async fn harvest(&self, _req: HarvestRequest) -> Result { + async fn harvest(&self, req: HarvestRequest) -> Result { + *self.seen_proxy.lock().await = req.proxy.clone(); Ok(HarvestResult { html: self.html.clone(), user_agent: self.ua.clone(), @@ -122,27 +137,46 @@ mod tests { #[tokio::test] async fn solve_without_harvester_is_not_implemented() { let h = CloudflareHandler::new(); - let page = PageHtml::new("https://x.com", ""); - let oc = h.solve(&page).await.expect("solve"); + let action = SolveAction::new(PageHtml::new("https://x.com", "")); + let oc = h.solve(&action).await.expect("solve"); assert_eq!(oc.status, px_pipeline::HandlerStatus::NotImplemented); } + /// Regression: the request's proxy has to reach the harvester. It used + /// to be parsed at the edge and dropped before any browser saw it. + #[tokio::test] + async fn solve_forwards_the_requested_proxy_to_the_harvester() { + let fake = Arc::new(FakeHarvester::new( + "ua", + vec![cookie("cf_clearance")], + "page", + )); + let h = CloudflareHandler::with_harvester(Arc::clone(&fake) as Arc); + let action = SolveAction::new(PageHtml::new("https://x.com", "")) + .with_proxy(Some("socks5://127.0.0.1:9050".into())); + let _ = h.solve(&action).await.expect("solve"); + assert_eq!( + fake.seen_proxy.lock().await.as_deref(), + Some("socks5://127.0.0.1:9050") + ); + } + #[tokio::test] async fn solve_with_harvester_returns_session_cookies_and_ua() { - let fake = Arc::new(FakeHarvester { - ua: "Mozilla/5.0 Camoufox".into(), - cookies: vec![ + let fake = Arc::new(FakeHarvester::new( + "Mozilla/5.0 Camoufox", + vec![ cookie("cf_clearance"), cookie("__cf_bm"), cookie("_px3"), cookie("_pxhd"), cookie("unrelated_session"), ], - html: "real page".into(), - }); + "real page", + )); let h = CloudflareHandler::with_harvester(fake); - let page = PageHtml::new("https://x.com", ""); - let oc = h.solve(&page).await.expect("solve"); + let action = SolveAction::new(PageHtml::new("https://x.com", "")); + let oc = h.solve(&action).await.expect("solve"); assert_eq!(oc.status, px_pipeline::HandlerStatus::Solved); assert_eq!(oc.user_agent.as_deref(), Some("Mozilla/5.0 Camoufox")); let names: Vec<&str> = oc.cookies.set.iter().map(|c| c.name.as_str()).collect(); diff --git a/px-core/Cargo.toml b/px-core/Cargo.toml index be8221c..fdead31 100644 --- a/px-core/Cargo.toml +++ b/px-core/Cargo.toml @@ -15,5 +15,8 @@ workspace = true serde = { workspace = true } uuid = { workspace = true } +[dev-dependencies] +serde_json = { workspace = true } + [lib] name = "px_core" diff --git a/px-core/src/solve_request.rs b/px-core/src/solve_request.rs index e910f47..e1c9873 100644 --- a/px-core/src/solve_request.rs +++ b/px-core/src/solve_request.rs @@ -1,10 +1,15 @@ use crate::fingerprint::Fingerprint; use serde::{Deserialize, Serialize}; +/// Absent optionals are omitted from the wire, not sent as `null`: this +/// type is what a client serializes, and a `"fingerprint": null` the +/// server has no field for reads as a promise the API does not keep. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SolveRequest { pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] pub proxy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub fingerprint: Option, } @@ -17,11 +22,22 @@ impl SolveRequest { } } + /// Route this solve through one egress proxy, so the returned bundle is + /// bound to an IP the caller can reuse. `scheme://host:port`, where + /// scheme is `http`, `https`, `socks5` or `socks5h`. pub fn with_proxy(mut self, proxy: impl Into) -> Self { self.proxy = Some(proxy.into()); self } + /// [`Self::with_proxy`] for an already-optional value, so an edge that + /// deserializes a nullable `proxy` field forwards it without branching. + #[must_use] + pub fn with_proxy_opt(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } + pub fn with_fingerprint(mut self, fingerprint: Fingerprint) -> Self { self.fingerprint = Some(fingerprint); self @@ -33,6 +49,25 @@ impl SolveRequest { mod tests { use super::*; + #[test] + fn absent_optionals_stay_off_the_wire() { + let json = serde_json::to_string(&SolveRequest::new("https://example.com")) + .expect("serialize request"); + assert_eq!(json, r#"{"url":"https://example.com"}"#); + } + + #[test] + fn a_named_proxy_is_serialized() { + let json = serde_json::to_string( + &SolveRequest::new("https://example.com").with_proxy("socks5://127.0.0.1:9050"), + ) + .expect("serialize request"); + assert!( + json.contains(r#""proxy":"socks5://127.0.0.1:9050""#), + "{json}" + ); + } + #[test] fn builder_assembles_request() { let r = SolveRequest::new("https://example.com").with_proxy("http://1.2.3.4:8080"); diff --git a/px-datadome/src/lib.rs b/px-datadome/src/lib.rs index e14c435..82e0f2e 100644 --- a/px-datadome/src/lib.rs +++ b/px-datadome/src/lib.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml, SolveAction}; pub struct DataDomeHandler; @@ -29,7 +29,7 @@ impl ChallengeHandler for DataDomeHandler { || h.contains("ddg_datadome")) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented(self.name())) } } diff --git a/px-harvester/src/domain/harvester.rs b/px-harvester/src/domain/harvester.rs index bff202a..c791243 100644 --- a/px-harvester/src/domain/harvester.rs +++ b/px-harvester/src/domain/harvester.rs @@ -19,6 +19,14 @@ impl HarvestRequest { wait_ms: 2_500, } } + + /// Pin this harvest to one egress proxy. `None` leaves the choice to + /// the [`crate::ProxyPool`] the implementation was built with. + #[must_use] + pub fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } } #[derive(Debug, Clone)] diff --git a/px-harvester/src/infrastructure/chromiumoxide_pool.rs b/px-harvester/src/infrastructure/chromiumoxide_pool.rs index 7a19e68..cec84d3 100644 --- a/px-harvester/src/infrastructure/chromiumoxide_pool.rs +++ b/px-harvester/src/infrastructure/chromiumoxide_pool.rs @@ -1,5 +1,6 @@ use crate::domain::harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; use crate::domain::stealth::{StealthBundle, default_stealth_bundle}; +use crate::infrastructure::egress::{chromium_proxy_spec, strip_credentials}; use async_trait::async_trait; use chromiumoxide::browser::{Browser, BrowserConfig}; use chromiumoxide::page::Page; @@ -26,6 +27,12 @@ impl Default for PoolConfig { } } +/// The launch flag carrying the egress, rendered by chromiumoxide as +/// `--proxy-server=`. +fn proxy_arg(proxy: &str) -> (&'static str, String) { + ("proxy-server", chromium_proxy_spec(proxy)) +} + pub struct ChromiumoxidePool { config: PoolConfig, stealth: StealthBundle, @@ -47,17 +54,24 @@ impl ChromiumoxidePool { self } - async fn launch_browser(&self) -> Result<(Browser, tokio::task::JoinHandle<()>), AppError> { + async fn launch_browser( + &self, + proxy: Option<&str>, + ) -> Result<(Browser, tokio::task::JoinHandle<()>), AppError> { let mut cfg = BrowserConfig::builder(); if !self.config.headless { cfg = cfg.with_head(); } + if let Some(proxy_url) = proxy { + let (key, value) = proxy_arg(proxy_url); + cfg = cfg.arg((key, value.as_str())); + } let cfg = cfg .build() .map_err(|e| AppError::InternalError(format!("BrowserConfig build: {e}")))?; let (browser, mut handler) = Browser::launch(cfg) .await - .map_err(|e| AppError::InternalError(format!("browser launch: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Browser launch: {e}")))?; let handle = tokio::spawn(async move { while let Some(event) = handler.next().await { if event.is_err() { @@ -72,7 +86,7 @@ impl ChromiumoxidePool { let cookies = page .get_cookies() .await - .map_err(|e| AppError::InternalError(format!("get_cookies: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Get cookies: {e}")))?; Ok(cookies .into_iter() .map(|c| HarvestedCookie { @@ -92,34 +106,40 @@ impl Harvester for ChromiumoxidePool { .permits .acquire() .await - .map_err(|e| AppError::InternalError(format!("semaphore: {e}")))?; - let (mut browser, _handle) = self.launch_browser().await?; + .map_err(|e| AppError::InternalError(format!("Semaphore: {e}")))?; + let proxy = req.proxy.clone().map(strip_credentials); + tracing::info!( + url = %req.url, + proxy = proxy.as_deref().unwrap_or("direct"), + "chromium harvest starting" + ); + let (mut browser, _handle) = self.launch_browser(proxy.as_deref()).await?; let page = browser .new_page("about:blank") .await - .map_err(|e| AppError::InternalError(format!("new_page: {e}")))?; + .map_err(|e| AppError::InternalError(format!("New page: {e}")))?; let script = self.stealth.combined(); if !script.is_empty() { page.evaluate_on_new_document(script.as_str()) .await - .map_err(|e| AppError::InternalError(format!("inject stealth: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Inject stealth: {e}")))?; } let navigate = page.goto(&req.url); tokio::time::timeout(self.config.navigate_timeout, navigate) .await - .map_err(|_| AppError::InternalError("navigate timeout".into()))? - .map_err(|e| AppError::InternalError(format!("goto: {e}")))?; + .map_err(|_| AppError::InternalError("Navigate timeout".into()))? + .map_err(|e| AppError::InternalError(format!("Goto: {e}")))?; tokio::time::sleep(Duration::from_millis(req.wait_ms)).await; let html = page .content() .await - .map_err(|e| AppError::InternalError(format!("content: {e}")))?; + .map_err(|e| AppError::InternalError(format!("Content: {e}")))?; let ua = page .evaluate("navigator.userAgent") .await - .map_err(|e| AppError::InternalError(format!("eval ua: {e}")))? + .map_err(|e| AppError::InternalError(format!("Eval user agent: {e}")))? .into_value::() - .map_err(|e| AppError::InternalError(format!("ua parse: {e}")))?; + .map_err(|e| AppError::InternalError(format!("User agent parse: {e}")))?; let cookies = Self::extract_cookies(&page).await?; let _ = browser.close().await; Ok(HarvestResult { @@ -129,3 +149,24 @@ impl Harvester for ChromiumoxidePool { }) } } + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + /// Regression: the Chromium leg had no proxy argument at all, so a + /// requested egress was dropped without a trace. + #[test] + fn proxy_arg_carries_the_requested_egress() { + let (key, value) = proxy_arg("http://egress.example:8080"); + assert_eq!(key, "proxy-server"); + assert_eq!(value, "http://egress.example:8080"); + } + + #[test] + fn proxy_arg_normalizes_a_scheme_chromium_would_ignore() { + let (_, value) = proxy_arg("socks5h://egress.example:1080"); + assert_eq!(value, "socks5://egress.example:1080"); + } +} diff --git a/px-harvester/src/infrastructure/egress.rs b/px-harvester/src/infrastructure/egress.rs new file mode 100644 index 0000000..8fcc672 --- /dev/null +++ b/px-harvester/src/infrastructure/egress.rs @@ -0,0 +1,93 @@ +//! Egress-proxy handling shared by every [`crate::Harvester`] implementation. +//! +//! A harvest leaves through the proxy named on its [`crate::HarvestRequest`] +//! and no other. Rotation across a pool belongs to long-lived browser +//! sessions, not to solving: a `_px3` bundle is bound to the IP that earned +//! it, so a caller who did not name an egress could not use a bundle +//! harvested through a rotating one. + +/// Drop `user:pass@` from a proxy URL, warning that the credentials are +/// being discarded. +/// +/// Neither browser engine can answer a proxy `407` from userinfo in the +/// URL: geckodriver's W3C `proxy` capability has no credential field, and +/// Chromium's `--proxy-server` ignores them without a CDP +/// `Fetch.authRequired` handler. Passing them through anyway fails the +/// whole harvest with no visible cause, so an authenticated upstream has +/// to be fronted by a local unauthenticated relay (gost, 3proxy). +pub fn strip_credentials(proxy: String) -> String { + let Some((scheme, rest)) = proxy.split_once("://") else { + return proxy; + }; + let Some((_userinfo, host_port)) = rest.rsplit_once('@') else { + return proxy; + }; + let sanitized = format!("{scheme}://{host_port}"); + tracing::warn!( + proxy = %sanitized, + "proxy credentials discarded: no browser engine can authenticate them; front the upstream with a local unauthenticated relay" + ); + sanitized +} + +/// Normalize a proxy URL into a Chromium `--proxy-server` spec. +/// +/// Chromium understands `http`, `https`, `socks4` and `socks5` — but not +/// `socks5h`, which is a curl convention that geckodriver's capability +/// layer accepts. Chromium *silently ignores* a spec it cannot parse and +/// goes direct, which is the failure this whole path exists to remove, so +/// the unknown scheme is rewritten rather than passed through. +pub fn chromium_proxy_spec(proxy: &str) -> String { + match proxy.split_once("://") { + Some(("socks5h", host_port)) => format!("socks5://{host_port}"), + _ => proxy.to_string(), + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn drops_userinfo_from_http_and_socks_urls() { + assert_eq!( + strip_credentials("http://u:p@rotating.example:8080".into()), + "http://rotating.example:8080" + ); + assert_eq!( + strip_credentials("socks5://a:b@x.example:1080".into()), + "socks5://x.example:1080" + ); + } + + #[test] + fn leaves_unauthenticated_urls_alone() { + assert_eq!( + strip_credentials("http://plain.example:8080".into()), + "http://plain.example:8080" + ); + assert_eq!(strip_credentials("host:8080".into()), "host:8080"); + } + + #[test] + fn chromium_spec_rewrites_socks5h_which_chromium_cannot_parse() { + assert_eq!( + chromium_proxy_spec("socks5h://x.example:1080"), + "socks5://x.example:1080" + ); + } + + #[test] + fn chromium_spec_passes_supported_schemes_through() { + for proxy in [ + "http://x.example:8080", + "https://x.example:8443", + "socks4://x.example:1080", + "socks5://x.example:1080", + "x.example:8080", + ] { + assert_eq!(chromium_proxy_spec(proxy), proxy); + } + } +} diff --git a/px-harvester/src/infrastructure/mod.rs b/px-harvester/src/infrastructure/mod.rs index 7809962..3b744ea 100644 --- a/px-harvester/src/infrastructure/mod.rs +++ b/px-harvester/src/infrastructure/mod.rs @@ -1,2 +1,3 @@ pub mod chromiumoxide_pool; +pub mod egress; pub mod stealth_bundle; diff --git a/px-harvester/src/lib.rs b/px-harvester/src/lib.rs index 06b2722..cb8e843 100644 --- a/px-harvester/src/lib.rs +++ b/px-harvester/src/lib.rs @@ -6,3 +6,4 @@ pub use application::harvest_page::HarvestPage; pub use domain::harvester::{HarvestRequest, HarvestResult, HarvestedCookie, Harvester}; pub use domain::stealth::{StealthBundle, default_stealth_bundle}; pub use infrastructure::chromiumoxide_pool::{ChromiumoxidePool, PoolConfig}; +pub use infrastructure::egress::strip_credentials; diff --git a/px-native/src/domain/native_solver.rs b/px-native/src/domain/native_solver.rs index 0e8917e..6a5ea94 100644 --- a/px-native/src/domain/native_solver.rs +++ b/px-native/src/domain/native_solver.rs @@ -7,6 +7,7 @@ pub struct SolveContext { pub url: String, pub app_id: PxAppId, pub fingerprint: Fingerprint, + pub proxy: Option, } impl SolveContext { @@ -15,8 +16,16 @@ impl SolveContext { url: url.into(), app_id, fingerprint, + proxy: None, } } + + /// Route this solve's sensor POST through one egress proxy. + #[must_use] + pub fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } } #[async_trait] diff --git a/px-native/src/infrastructure/handler.rs b/px-native/src/infrastructure/handler.rs index ccb9fb1..21f8653 100644 --- a/px-native/src/infrastructure/handler.rs +++ b/px-native/src/infrastructure/handler.rs @@ -9,7 +9,9 @@ use std::time::Instant; use async_trait::async_trait; use px_core::{CookieJarDelta, Fingerprint, PxAppId}; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerName, HandlerOutcome, PageHtml}; +use px_pipeline::{ + ChallengeHandler, HandlerMetrics, HandlerName, HandlerOutcome, PageHtml, SolveAction, +}; use crate::domain::native_solver::{NativeSolver, SolveContext}; @@ -39,9 +41,10 @@ impl ChallengeHandler for NativePxHandler { Ok(true) } - async fn solve(&self, page: &PageHtml) -> Result { + async fn solve(&self, action: &SolveAction) -> Result { let started = Instant::now(); - let ctx = SolveContext::new(page.url.clone(), self.app_id.clone(), default_fingerprint()); + let ctx = SolveContext::new(action.url(), self.app_id.clone(), default_fingerprint()) + .with_proxy(action.proxy.clone()); let bundle = self.solver.solve(&ctx).await?; let metrics = HandlerMetrics { solve_ms: started.elapsed().as_millis() as u64, @@ -81,12 +84,17 @@ mod tests { use px_core::{NamedCookie, PxCookieBundle}; use px_pipeline::HandlerStatus; use std::time::{Duration, SystemTime}; + use tokio::sync::Mutex; - struct AlwaysOkSolver; + #[derive(Default)] + struct AlwaysOkSolver { + seen_proxy: Mutex>, + } #[async_trait] impl NativeSolver for AlwaysOkSolver { - async fn solve(&self, _ctx: &SolveContext) -> Result { + async fn solve(&self, ctx: &SolveContext) -> Result { + *self.seen_proxy.lock().await = ctx.proxy.clone(); Ok(PxCookieBundle::new( vec![NamedCookie { name: "_px3".into(), @@ -107,12 +115,29 @@ mod tests { #[tokio::test] async fn handler_reports_solved_status() { - let handler = - NativePxHandler::new(Arc::new(AlwaysOkSolver) as Arc, app_id()); - let page = PageHtml::new("https://www.pedidosya.com.ar/", ""); - let out = handler.solve(&page).await.expect("solve"); + let handler = NativePxHandler::new( + Arc::new(AlwaysOkSolver::default()) as Arc, + app_id(), + ); + let action = SolveAction::new(PageHtml::new("https://www.pedidosya.com.ar/", "")); + let out = handler.solve(&action).await.expect("solve"); assert_eq!(out.status, HandlerStatus::Solved); assert_eq!(out.cookies.set.len(), 1); assert_eq!(out.user_agent.as_deref(), Some("ua")); } + + /// The native path must carry the request's proxy into the sensor POST; + /// dropping it here would send the payload from the server's own IP. + #[tokio::test] + async fn handler_forwards_the_requested_proxy_to_the_solver() { + let solver = Arc::new(AlwaysOkSolver::default()); + let handler = NativePxHandler::new(Arc::clone(&solver) as Arc, app_id()); + let action = SolveAction::new(PageHtml::new("https://www.pedidosya.com.ar/", "")) + .with_proxy(Some("http://egress:8080".into())); + let _ = handler.solve(&action).await.expect("solve"); + assert_eq!( + solver.seen_proxy.lock().await.as_deref(), + Some("http://egress:8080") + ); + } } diff --git a/px-native/src/infrastructure/mod.rs b/px-native/src/infrastructure/mod.rs index e03dea1..0306bc9 100644 --- a/px-native/src/infrastructure/mod.rs +++ b/px-native/src/infrastructure/mod.rs @@ -2,6 +2,7 @@ pub mod cookies; pub mod handler; pub mod native_first; pub mod not_implemented; +pub mod proxy_clients; pub mod sensor_solver; pub use handler::NativePxHandler; diff --git a/px-native/src/infrastructure/native_first.rs b/px-native/src/infrastructure/native_first.rs index 3280019..a978219 100644 --- a/px-native/src/infrastructure/native_first.rs +++ b/px-native/src/infrastructure/native_first.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerName, HandlerOutcome, HandlerStatus, PageHtml}; +use px_pipeline::{ + ChallengeHandler, HandlerName, HandlerOutcome, HandlerStatus, PageHtml, SolveAction, +}; pub struct NativeFirstHandler { native: Arc, @@ -33,8 +35,8 @@ impl ChallengeHandler for NativeFirstHandler { self.fallback.detects(page).await } - async fn solve(&self, page: &PageHtml) -> Result { - match self.native.solve(page).await { + async fn solve(&self, action: &SolveAction) -> Result { + match self.native.solve(action).await { Ok(out) if matches!(out.status, HandlerStatus::Solved) => Ok(out), Ok(out) => { tracing::info!( @@ -42,7 +44,7 @@ impl ChallengeHandler for NativeFirstHandler { status = ?out.status, "native handler not solved, falling back" ); - self.fallback.solve(page).await + self.fallback.solve(action).await } Err(e) => { tracing::warn!( @@ -50,7 +52,7 @@ impl ChallengeHandler for NativeFirstHandler { error = %e, "native handler error, falling back" ); - self.fallback.solve(page).await + self.fallback.solve(action).await } } } @@ -74,7 +76,7 @@ mod tests { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::solved_with_ua( self.0, CookieJarDelta::default(), @@ -93,7 +95,7 @@ mod tests { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Err(AppError::InternalError("synthetic".into())) } } @@ -105,7 +107,7 @@ mod tests { Arc::new(SolvedHandler("fallback")), ); let out = h - .solve(&PageHtml::new("https://x/", "")) + .solve(&SolveAction::new(PageHtml::new("https://x/", ""))) .await .expect("solve"); assert_eq!(out.handler, "native"); @@ -115,7 +117,7 @@ mod tests { async fn falls_back_on_error() { let h = NativeFirstHandler::new(Arc::new(FailingHandler), Arc::new(SolvedHandler("fb"))); let out = h - .solve(&PageHtml::new("https://x/", "")) + .solve(&SolveAction::new(PageHtml::new("https://x/", ""))) .await .expect("solve"); assert_eq!(out.handler, "fb"); diff --git a/px-native/src/infrastructure/proxy_clients.rs b/px-native/src/infrastructure/proxy_clients.rs new file mode 100644 index 0000000..e42fbab --- /dev/null +++ b/px-native/src/infrastructure/proxy_clients.rs @@ -0,0 +1,83 @@ +//! Per-proxy `reqwest` clients for the native sensor path. +//! +//! A `Client` owns its connection pool, and the proxy is fixed at build +//! time, so honouring a per-request egress means one client per proxy. +//! They are cached because rebuilding one per solve would throw away the +//! pooled TLS connection to the sensor endpoint. +//! +//! Unlike the browser paths, `reqwest` does implement proxy +//! authentication, so `user:pass@` in the URL is honoured here. + +use std::collections::HashMap; +use std::sync::Mutex; + +use px_errors::AppError; +use reqwest::{Client, Proxy}; + +pub struct ProxyClients { + direct: Client, + proxied: Mutex>, +} + +impl ProxyClients { + pub fn new(direct: Client) -> Self { + Self { + direct, + proxied: Mutex::new(HashMap::new()), + } + } + + /// Client whose egress is `proxy`, or the direct one when `None`. + pub fn for_proxy(&self, proxy: Option<&str>) -> Result { + let Some(proxy) = proxy else { + return Ok(self.direct.clone()); + }; + let mut cache = self + .proxied + .lock() + .map_err(|e| AppError::InternalError(format!("Proxy client cache poisoned: {e}")))?; + if let Some(client) = cache.get(proxy) { + return Ok(client.clone()); + } + let built = Client::builder() + .proxy( + Proxy::all(proxy) + .map_err(|e| AppError::BadRequest(format!("Invalid proxy url: {e}")))?, + ) + .build() + .map_err(|e| AppError::InternalError(format!("Failed to build proxied client: {e}")))?; + cache.insert(proxy.to_string(), built.clone()); + Ok(built) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + fn clients() -> ProxyClients { + ProxyClients::new(Client::builder().build().expect("build direct client")) + } + + #[test] + fn no_proxy_yields_the_direct_client() { + assert!(clients().for_proxy(None).is_ok()); + } + + #[test] + fn a_proxy_url_builds_and_caches_one_client() { + let clients = clients(); + assert!(clients.for_proxy(Some("http://127.0.0.1:8080")).is_ok()); + assert!(clients.for_proxy(Some("http://127.0.0.1:8080")).is_ok()); + assert_eq!(clients.proxied.lock().expect("cache not poisoned").len(), 1); + } + + #[test] + fn a_malformed_proxy_url_is_a_bad_request() { + let err = clients() + .for_proxy(Some("not a url")) + .expect_err("malformed proxy must fail"); + assert!(matches!(err, AppError::BadRequest(_))); + } +} diff --git a/px-native/src/infrastructure/sensor_solver.rs b/px-native/src/infrastructure/sensor_solver.rs index 73fb596..098bd74 100644 --- a/px-native/src/infrastructure/sensor_solver.rs +++ b/px-native/src/infrastructure/sensor_solver.rs @@ -18,11 +18,12 @@ use crate::cipher::encrypt_sensor; use crate::domain::native_solver::{NativeSolver, SolveContext}; use crate::events::{SyntheticIdentity, default_batch}; use crate::infrastructure::cookies::parse_set_cookies; +use crate::infrastructure::proxy_clients::ProxyClients; use crate::profile::TenantProfile; /// HTTP-backed native solver bound to a single tenant profile. pub struct SensorNativeSolver { - client: Client, + clients: ProxyClients, profile: Arc, cookie_ttl: Duration, } @@ -30,7 +31,7 @@ pub struct SensorNativeSolver { impl SensorNativeSolver { pub fn new(client: Client, profile: Arc) -> Self { Self { - client, + clients: ProxyClients::new(client), profile, cookie_ttl: Duration::from_secs(300), } @@ -71,9 +72,9 @@ impl NativeSolver for SensorNativeSolver { ); let sensor_url = self.profile.sensor_url(&origin); let payload = self.build_payload(ctx)?; + let client = self.clients.for_proxy(ctx.proxy.as_deref())?; - let resp = self - .client + let resp = client .post(&sensor_url) .header(USER_AGENT, header(&ctx.fingerprint.user_agent)?) .header(ACCEPT, HeaderValue::from_static("*/*")) @@ -108,7 +109,13 @@ impl NativeSolver for SensorNativeSolver { "sensor POST returned no Set-Cookie headers".into(), )); } - tracing::info!(target: "px_native", url = %sensor_url, count = cookies.len(), "native sensor solved"); + tracing::info!( + target: "px_native", + url = %sensor_url, + count = cookies.len(), + proxy = ctx.proxy.as_deref().unwrap_or("direct"), + "native sensor solved" + ); Ok(PxCookieBundle::new( cookies, ctx.fingerprint.user_agent.clone(), diff --git a/px-native/tests/throughput_soak.rs b/px-native/tests/throughput_soak.rs index 6bb2e00..1242e67 100644 --- a/px-native/tests/throughput_soak.rs +++ b/px-native/tests/throughput_soak.rs @@ -19,7 +19,11 @@ //! [NATIVE_SOAK_CONCURRENCY=8] \ //! [NATIVE_SOAK_TARGET_RPM=40] \ //! [NATIVE_SOAK_PROFILE=px-native/profiles/eT15wiaE.toml] \ +//! [NATIVE_SOAK_PROXY=socks5://... | PX_PROXIES=socks5://...] \ //! cargo test -p pxsolver-native --test throughput_soak -- --ignored --nocapture +//! +//! Without a proxy the soak leaves through the host's own IP, which a +//! rate-limiting tenant will flag long before the target rpm is reached. use std::path::PathBuf; use std::sync::Arc; @@ -61,9 +65,14 @@ async fn native_throughput_soak() { let client = Client::builder().build().expect("client"); let solver: Arc = Arc::new(SensorNativeSolver::new(client, Arc::new(profile))); - let ctx_template = SolveContext::new(url.clone(), app_id.clone(), soak_fingerprint()); + let proxy = soak_proxy(); + let ctx_template = SolveContext::new(url.clone(), app_id.clone(), soak_fingerprint()) + .with_proxy(proxy.clone()); - eprintln!("soak: n={n} concurrency={concurrency} target_rpm={target_rpm} url={url}"); + eprintln!( + "soak: n={n} concurrency={concurrency} target_rpm={target_rpm} url={url} proxy={}", + proxy.as_deref().unwrap_or("direct") + ); let mut latencies_ms: Vec = Vec::with_capacity(n); let mut ok_count: usize = 0; @@ -113,6 +122,21 @@ async fn native_throughput_soak() { ); } +/// Egress for the soak: `NATIVE_SOAK_PROXY`, else the first `PX_PROXIES` +/// entry so an operator who already exported the list for the capture +/// step does not have to restate it. +fn soak_proxy() -> Option { + std::env::var("NATIVE_SOAK_PROXY") + .ok() + .or_else(|| std::env::var("PX_PROXIES").ok()) + .and_then(|raw| { + raw.split(',') + .map(str::trim) + .find(|entry| !entry.is_empty()) + .map(str::to_string) + }) +} + fn soak_fingerprint() -> Fingerprint { Fingerprint { user_agent: "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0".into(), diff --git a/px-perimeterx/src/application/solve_px.rs b/px-perimeterx/src/application/solve_px.rs index f0eeef6..efeb053 100644 --- a/px-perimeterx/src/application/solve_px.rs +++ b/px-perimeterx/src/application/solve_px.rs @@ -20,9 +20,10 @@ impl SolvePx { Self { harvester } } - pub async fn execute(&self, url: &str) -> Result { + pub async fn execute(&self, url: &str, proxy: Option) -> Result { let start = Instant::now(); - let result = self.harvester.harvest(HarvestRequest::new(url)).await?; + let request = HarvestRequest::new(url).with_proxy(proxy); + let result = self.harvester.harvest(request).await?; let px_cookies: Vec = extract_px_cookies(&result.cookies) .into_iter() .map(|c| NamedCookie { diff --git a/px-perimeterx/src/infrastructure/handler.rs b/px-perimeterx/src/infrastructure/handler.rs index e534da6..749dced 100644 --- a/px-perimeterx/src/infrastructure/handler.rs +++ b/px-perimeterx/src/infrastructure/handler.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use px_detector::{Detected, Detector, RegexDetector}; use px_errors::AppError; use px_harvester::Harvester; -use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml, SolveAction}; use std::sync::Arc; pub struct PerimeterxHandler { @@ -30,8 +30,11 @@ impl ChallengeHandler for PerimeterxHandler { Ok(matches!(self.detector.detect(&page.html), Detected::Yes(_))) } - async fn solve(&self, page: &PageHtml) -> Result { - let out = self.solver.execute(&page.url).await?; + async fn solve(&self, action: &SolveAction) -> Result { + let out = self + .solver + .execute(action.url(), action.proxy.clone()) + .await?; let metrics = HandlerMetrics { detect_us: 0, solve_ms: out.solve_ms, diff --git a/px-pipeline/src/application/run_pipeline.rs b/px-pipeline/src/application/run_pipeline.rs index 34a4d74..48217d6 100644 --- a/px-pipeline/src/application/run_pipeline.rs +++ b/px-pipeline/src/application/run_pipeline.rs @@ -1,6 +1,6 @@ use crate::domain::challenge_handler::ChallengeHandler; use crate::domain::handler_outcome::HandlerOutcome; -use crate::domain::page_html::PageHtml; +use crate::domain::solve_action::SolveAction; use px_errors::AppError; use std::sync::Arc; @@ -27,11 +27,11 @@ impl Pipeline { self.handlers.len() } - pub async fn run(&self, page: &PageHtml) -> Result, AppError> { + pub async fn run(&self, action: &SolveAction) -> Result, AppError> { let mut outcomes = Vec::with_capacity(self.handlers.len()); for handler in &self.handlers { - if handler.detects(page).await? { - let outcome = handler.solve(page).await?; + if handler.detects(&action.page).await? { + let outcome = handler.solve(action).await?; let solved = matches!( outcome.status, crate::domain::handler_outcome::HandlerStatus::Solved diff --git a/px-pipeline/src/domain/challenge_handler.rs b/px-pipeline/src/domain/challenge_handler.rs index 22f7e3e..8e80aae 100644 --- a/px-pipeline/src/domain/challenge_handler.rs +++ b/px-pipeline/src/domain/challenge_handler.rs @@ -1,5 +1,6 @@ use crate::domain::handler_outcome::HandlerOutcome; use crate::domain::page_html::PageHtml; +use crate::domain::solve_action::SolveAction; use async_trait::async_trait; use px_errors::AppError; @@ -9,5 +10,5 @@ pub use crate::domain::handler_outcome::HandlerName; pub trait ChallengeHandler: Send + Sync { fn name(&self) -> HandlerName; async fn detects(&self, page: &PageHtml) -> Result; - async fn solve(&self, page: &PageHtml) -> Result; + async fn solve(&self, action: &SolveAction) -> Result; } diff --git a/px-pipeline/src/domain/mod.rs b/px-pipeline/src/domain/mod.rs index 956938c..ac3817f 100644 --- a/px-pipeline/src/domain/mod.rs +++ b/px-pipeline/src/domain/mod.rs @@ -2,3 +2,4 @@ pub mod challenge_handler; pub mod fetcher; pub mod handler_outcome; pub mod page_html; +pub mod solve_action; diff --git a/px-pipeline/src/domain/solve_action.rs b/px-pipeline/src/domain/solve_action.rs new file mode 100644 index 0000000..124046f --- /dev/null +++ b/px-pipeline/src/domain/solve_action.rs @@ -0,0 +1,52 @@ +//! What a [`crate::ChallengeHandler`] is asked to solve. +//! +//! Detection only ever needs the page, but solving also needs the egress +//! the operator picked for this request, so the action carries both. It is +//! built at the infrastructure edge (the HTTP handler) and passed inward; +//! handlers never read transport types. + +use crate::domain::page_html::PageHtml; + +#[derive(Debug, Clone)] +pub struct SolveAction { + pub page: PageHtml, + pub proxy: Option, +} + +impl SolveAction { + pub fn new(page: PageHtml) -> Self { + Self { page, proxy: None } + } + + /// Pin this solve to one egress proxy. `None` leaves the choice to the + /// harvester's rotation. + #[must_use] + pub fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } + + pub fn url(&self) -> &str { + &self.page.url + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn defaults_to_no_proxy() { + let action = SolveAction::new(PageHtml::new("https://example.com", "")); + assert_eq!(action.url(), "https://example.com"); + assert!(action.proxy.is_none()); + } + + #[test] + fn carries_the_requested_proxy() { + let action = SolveAction::new(PageHtml::new("https://example.com", "")) + .with_proxy(Some("socks5://127.0.0.1:9050".into())); + assert_eq!(action.proxy.as_deref(), Some("socks5://127.0.0.1:9050")); + } +} diff --git a/px-pipeline/src/lib.rs b/px-pipeline/src/lib.rs index 4d1f8e6..c7ecdb9 100644 --- a/px-pipeline/src/lib.rs +++ b/px-pipeline/src/lib.rs @@ -6,3 +6,4 @@ pub use domain::challenge_handler::{ChallengeHandler, HandlerName}; pub use domain::fetcher::{FetchRequest, FetchResponse, Fetcher}; pub use domain::handler_outcome::{HandlerMetrics, HandlerOutcome, HandlerStatus}; pub use domain::page_html::PageHtml; +pub use domain::solve_action::SolveAction; diff --git a/px-pipeline/tests/pipeline.rs b/px-pipeline/tests/pipeline.rs index 3ae9e95..8bffabf 100644 --- a/px-pipeline/tests/pipeline.rs +++ b/px-pipeline/tests/pipeline.rs @@ -5,6 +5,7 @@ use px_core::{CookieJarDelta, NamedToken}; use px_errors::AppError; use px_pipeline::{ ChallengeHandler, HandlerMetrics, HandlerOutcome, HandlerStatus, PageHtml, Pipeline, + SolveAction, }; use std::sync::Arc; @@ -18,7 +19,7 @@ impl ChallengeHandler for SkippingHandler { async fn detects(&self, _page: &PageHtml) -> Result { Ok(false) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Err(AppError::InternalError( "should not be called when detects=false".into(), )) @@ -35,7 +36,7 @@ impl ChallengeHandler for SolvingHandler { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::solved( "solve", CookieJarDelta::default(), @@ -55,7 +56,7 @@ impl ChallengeHandler for StubHandler { async fn detects(&self, _page: &PageHtml) -> Result { Ok(true) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented("stub")) } } @@ -64,7 +65,10 @@ impl ChallengeHandler for StubHandler { async fn pipeline_skips_then_solves_and_stops() { let pipeline = Pipeline::new(vec![Arc::new(SkippingHandler), Arc::new(SolvingHandler)]); let page = PageHtml::new("https://x", ""); - let outcomes = pipeline.run(&page).await.expect("run ok"); + let outcomes = pipeline + .run(&SolveAction::new(page.clone())) + .await + .expect("run ok"); assert_eq!(outcomes.len(), 2); assert_eq!(outcomes[0].status, HandlerStatus::Skipped); assert_eq!(outcomes[1].status, HandlerStatus::Solved); @@ -75,7 +79,10 @@ async fn stop_on_solve_false_continues_past_solver() { let pipeline = Pipeline::new(vec![Arc::new(SolvingHandler), Arc::new(StubHandler)]) .with_stop_on_solve(false); let page = PageHtml::new("https://x", ""); - let outcomes = pipeline.run(&page).await.expect("run ok"); + let outcomes = pipeline + .run(&SolveAction::new(page.clone())) + .await + .expect("run ok"); assert_eq!(outcomes.len(), 2); assert_eq!(outcomes[0].status, HandlerStatus::Solved); assert_eq!(outcomes[1].status, HandlerStatus::NotImplemented); @@ -85,7 +92,10 @@ async fn stop_on_solve_false_continues_past_solver() { async fn first_solver_stops_pipeline() { let pipeline = Pipeline::new(vec![Arc::new(SolvingHandler), Arc::new(StubHandler)]); let page = PageHtml::new("https://x", ""); - let outcomes = pipeline.run(&page).await.expect("run ok"); + let outcomes = pipeline + .run(&SolveAction::new(page.clone())) + .await + .expect("run ok"); assert_eq!(outcomes.len(), 1); assert_eq!(outcomes[0].status, HandlerStatus::Solved); } diff --git a/px-server/src/application/routing.rs b/px-server/src/application/routing.rs index e75e605..d8d8026 100644 --- a/px-server/src/application/routing.rs +++ b/px-server/src/application/routing.rs @@ -11,9 +11,9 @@ use crate::application::solve_endpoint::{SolveDispatcher, SolveOutput, domain_from_url}; use async_trait::async_trait; -use px_core::PxCookieBundle; +use px_core::{PxCookieBundle, SolveRequest}; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml, SolveAction}; use std::collections::BTreeMap; use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -68,11 +68,11 @@ impl RoutingDispatcher { #[async_trait] impl SolveDispatcher for RoutingDispatcher { - async fn solve(&self, url: &str) -> Result { - let host = domain_from_url(url)?; + async fn solve(&self, req: SolveRequest) -> Result { + let host = domain_from_url(&req.url)?; let handler = self.resolve(&host); - let page = PageHtml::new(url, ""); - let outcome = handler.solve(&page).await?; + let action = SolveAction::new(PageHtml::new(&req.url, "")).with_proxy(req.proxy); + let outcome = handler.solve(&action).await?; if !matches!(outcome.status, HandlerStatus::Solved) { return Err(AppError::Conflict(format!( "{} returned status {:?}", @@ -106,93 +106,3 @@ pub fn parse_camoufox_domains(raw: Option<&str>) -> Vec { .filter(|s| !s.is_empty()) .collect() } - -#[cfg(test)] -#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] -mod tests { - use super::*; - use px_core::CookieJarDelta; - use px_pipeline::{HandlerMetrics, HandlerOutcome}; - - struct StaticHandler { - name: &'static str, - } - - #[async_trait] - impl ChallengeHandler for StaticHandler { - fn name(&self) -> &'static str { - self.name - } - async fn detects(&self, _page: &PageHtml) -> Result { - Ok(true) - } - async fn solve(&self, _page: &PageHtml) -> Result { - Ok(HandlerOutcome::solved_with_ua( - self.name, - CookieJarDelta::default(), - Vec::new(), - HandlerMetrics::default(), - "ua", - )) - } - } - - #[tokio::test] - async fn default_handler_used_when_no_match() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })).with_route( - "pedidosya.com.ar", - Arc::new(StaticHandler { name: "cloudflare" }), - ); - let out = d - .solve("https://www.havenwellwithin.com/") - .await - .expect("solve"); - assert_eq!(out.handler, "perimeterx"); - } - - #[tokio::test] - async fn exact_host_routes_to_match() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })).with_route( - "pedidosya.com.ar", - Arc::new(StaticHandler { name: "cloudflare" }), - ); - let out = d.solve("https://pedidosya.com.ar/").await.expect("solve"); - assert_eq!(out.handler, "cloudflare"); - } - - #[tokio::test] - async fn subdomain_routes_to_match() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })).with_route( - "pedidosya.com.ar", - Arc::new(StaticHandler { name: "cloudflare" }), - ); - let out = d - .solve("https://www.pedidosya.com.ar/x") - .await - .expect("solve"); - assert_eq!(out.handler, "cloudflare"); - } - - /// Regression: the cookie bundle's `user_agent` must carry the real - /// harvester UA (so cache-hit replies preserve it), not a literal - /// placeholder string. - #[tokio::test] - async fn bundle_user_agent_matches_harvester() { - let d = RoutingDispatcher::new(Arc::new(StaticHandler { name: "perimeterx" })); - let out = d.solve("https://example.com/").await.expect("solve"); - assert_eq!(out.user_agent, "ua"); - assert_eq!(out.bundle.user_agent, "ua"); - } - - #[test] - fn parse_csv_trims_and_lowercases() { - let r = parse_camoufox_domains(Some(" Pedidosya.com.AR , ,foo.com ")); - assert_eq!(r, vec!["pedidosya.com.ar", "foo.com"]); - } - - #[test] - fn parse_csv_empty_unset() { - assert!(parse_camoufox_domains(None).is_empty()); - assert!(parse_camoufox_domains(Some("")).is_empty()); - } -} diff --git a/px-server/src/application/solve_endpoint.rs b/px-server/src/application/solve_endpoint.rs index c12906a..adcb34b 100644 --- a/px-server/src/application/solve_endpoint.rs +++ b/px-server/src/application/solve_endpoint.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; -use px_core::{CacheKey, PxAppId, PxCookieBundle}; +use px_core::{CacheKey, PxAppId, PxCookieBundle, SolveRequest}; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerStatus, PageHtml, SolveAction}; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::sync::Arc; use std::time::{Duration, SystemTime}; use url::Url; #[async_trait] pub trait SolveDispatcher: Send + Sync { - async fn solve(&self, url: &str) -> Result; + async fn solve(&self, req: SolveRequest) -> Result; } #[derive(Debug, Clone)] @@ -37,9 +39,9 @@ impl PxSolveDispatcher { #[async_trait] impl SolveDispatcher for PxSolveDispatcher { - async fn solve(&self, url: &str) -> Result { - let page = PageHtml::new(url, ""); - let outcome = self.handler.solve(&page).await?; + async fn solve(&self, req: SolveRequest) -> Result { + let action = SolveAction::new(PageHtml::new(&req.url, "")).with_proxy(req.proxy); + let outcome = self.handler.solve(&action).await?; if !matches!(outcome.status, HandlerStatus::Solved) { return Err(AppError::Conflict(format!( "{} returned status {:?}", @@ -75,8 +77,25 @@ pub fn cache_key_for(domain: &str, app_id: PxAppId, fp_key: u64) -> CacheKey { CacheKey::new(domain, app_id, fp_key) } -pub fn sentinel_cache_key(domain: &str) -> Result { +/// Cache key for one solve. The egress is part of the identity: PX binds a +/// `_px3` bundle to the IP that earned it, so replaying a proxy-A bundle +/// for a request that asked for proxy B (or for a direct route) hands the +/// caller cookies their traffic will not match. +pub fn sentinel_cache_key(domain: &str, proxy: Option<&str>) -> Result { let app_id = PxAppId::new("Unknown000") .map_err(|e| AppError::InternalError(format!("sentinel app_id invalid: {e}")))?; - Ok(CacheKey::new(domain, app_id, 0)) + Ok(CacheKey::new(domain, app_id, egress_key(proxy))) +} + +/// `0` for a direct route, so keys minted before proxies were honoured +/// keep resolving to the same entry. +fn egress_key(proxy: Option<&str>) -> u64 { + match proxy { + None => 0, + Some(proxy) => { + let mut hasher = DefaultHasher::new(); + proxy.hash(&mut hasher); + hasher.finish() + } + } } diff --git a/px-server/src/infrastructure/http/handlers/solve.rs b/px-server/src/infrastructure/http/handlers/solve.rs index 5c7387e..139b6b7 100644 --- a/px-server/src/infrastructure/http/handlers/solve.rs +++ b/px-server/src/infrastructure/http/handlers/solve.rs @@ -5,6 +5,7 @@ use axum::Json; use axum::extract::State; use axum::http::HeaderMap; use px_auth::{AuditEvent, AuditOutcome}; +use px_core::SolveRequest; use px_errors::AppError; use px_types::SingleResponse; use std::sync::atomic::Ordering; @@ -33,7 +34,8 @@ pub async fn handle( .fetch_add(1, Ordering::Relaxed); })?; state.metrics.solves_total.fetch_add(1, Ordering::Relaxed); - let cache_key = sentinel_cache_key(&domain)?; + let request = SolveRequest::new(&payload.url).with_proxy_opt(payload.proxy); + let cache_key = sentinel_cache_key(&domain, request.proxy.as_deref())?; let cached = state.cache.get(&cache_key).await?; let (out, cache_hit) = if let Some(bundle) = cached { ( @@ -47,7 +49,7 @@ pub async fn handle( true, ) } else { - let result = state.dispatcher.solve(&payload.url).await; + let result = state.dispatcher.solve(request).await; let solved = match result { Ok(v) => v, Err(e) => { diff --git a/px-server/tests/cache.rs b/px-server/tests/cache.rs index 6bd3b91..ad5e43a 100644 --- a/px-server/tests/cache.rs +++ b/px-server/tests/cache.rs @@ -39,3 +39,75 @@ async fn second_solve_for_same_domain_hits_cache() { assert_eq!(dispatcher.calls.load(Ordering::Relaxed), 1); assert_eq!(audit.count.load(Ordering::Relaxed), 2); } + +/// The egress is part of a bundle's identity: PX binds `_px3` to the IP +/// that earned it, so two solves of one domain through different proxies +/// must not share a cache entry. +#[tokio::test] +async fn solves_through_different_proxies_do_not_share_a_cache_entry() { + let audit = Arc::new(CountingAuditSink::default()); + let dispatcher = Arc::new(FakeDispatcher::default()); + let state = build_state_with_dispatcher(audit.clone(), dispatcher.clone()); + let app = build_router(state); + + let first = app + .clone() + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://a.example:8080"}"#, + )) + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::OK); + assert!(body_string(first).await.contains("\"cache_hit\":false")); + assert_eq!( + dispatcher.last_proxy.lock().await.as_deref(), + Some("http://a.example:8080") + ); + + let second = app + .clone() + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://b.example:8080"}"#, + )) + .await + .unwrap(); + assert!(body_string(second).await.contains("\"cache_hit\":false")); + + let repeat = app + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://a.example:8080"}"#, + )) + .await + .unwrap(); + assert!(body_string(repeat).await.contains("\"cache_hit\":true")); + + assert_eq!(dispatcher.calls.load(Ordering::Relaxed), 2); +} + +/// A direct solve must not be served a bundle harvested through a proxy. +#[tokio::test] +async fn a_proxied_bundle_is_not_replayed_for_a_direct_solve() { + let audit = Arc::new(CountingAuditSink::default()); + let dispatcher = Arc::new(FakeDispatcher::default()); + let state = build_state_with_dispatcher(audit.clone(), dispatcher.clone()); + let app = build_router(state); + + let proxied = app + .clone() + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":"http://a.example:8080"}"#, + )) + .await + .unwrap(); + assert!(body_string(proxied).await.contains("\"cache_hit\":false")); + + let direct = app + .oneshot(solve_request( + r#"{"url":"https://pedidosya.com.ar/","proxy":null}"#, + )) + .await + .unwrap(); + assert!(body_string(direct).await.contains("\"cache_hit\":false")); + assert!(dispatcher.last_proxy.lock().await.is_none()); + assert_eq!(dispatcher.calls.load(Ordering::Relaxed), 2); +} diff --git a/px-server/tests/common/mod.rs b/px-server/tests/common/mod.rs index 2d56842..887b0c1 100644 --- a/px-server/tests/common/mod.rs +++ b/px-server/tests/common/mod.rs @@ -19,6 +19,7 @@ use px_server::{AppState, AppStateConfig}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, SystemTime}; +use tokio::sync::Mutex; pub fn hash(secret: &str) -> String { let salt = SaltString::encode_b64(b"px-solver-test-salt-fixed").expect("salt"); @@ -31,12 +32,14 @@ pub fn hash(secret: &str) -> String { #[derive(Default)] pub struct FakeDispatcher { pub calls: AtomicUsize, + pub last_proxy: Mutex>, } #[async_trait] impl SolveDispatcher for FakeDispatcher { - async fn solve(&self, _url: &str) -> Result { + async fn solve(&self, req: px_core::SolveRequest) -> Result { self.calls.fetch_add(1, Ordering::Relaxed); + *self.last_proxy.lock().await = req.proxy; Ok(SolveOutput { bundle: px_core::PxCookieBundle::new( vec![NamedCookie { diff --git a/px-server/tests/routing.rs b/px-server/tests/routing.rs new file mode 100644 index 0000000..5f38ff7 --- /dev/null +++ b/px-server/tests/routing.rs @@ -0,0 +1,130 @@ +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use async_trait::async_trait; +use px_core::{CookieJarDelta, SolveRequest}; +use px_errors::AppError; +use px_pipeline::{ChallengeHandler, HandlerMetrics, HandlerOutcome, PageHtml, SolveAction}; +use px_server::application::routing::{RoutingDispatcher, parse_camoufox_domains}; +use px_server::application::solve_endpoint::SolveDispatcher; +use std::sync::Arc; +use tokio::sync::Mutex; + +#[derive(Default)] +struct StaticHandler { + name: &'static str, + seen_proxy: Mutex>, +} + +impl StaticHandler { + fn new(name: &'static str) -> Arc { + Arc::new(Self { + name, + seen_proxy: Mutex::new(None), + }) + } +} + +#[async_trait] +impl ChallengeHandler for StaticHandler { + fn name(&self) -> &'static str { + self.name + } + + async fn detects(&self, _page: &PageHtml) -> Result { + Ok(true) + } + + async fn solve(&self, action: &SolveAction) -> Result { + *self.seen_proxy.lock().await = action.proxy.clone(); + Ok(HandlerOutcome::solved_with_ua( + self.name, + CookieJarDelta::default(), + Vec::new(), + HandlerMetrics::default(), + "ua", + )) + } +} + +fn dispatcher_with_cf_route() -> RoutingDispatcher { + RoutingDispatcher::new(StaticHandler::new("perimeterx")) + .with_route("pedidosya.com.ar", StaticHandler::new("cloudflare")) +} + +#[tokio::test] +async fn test_solve_unrouted_host_uses_default_handler() { + let out = dispatcher_with_cf_route() + .solve(SolveRequest::new("https://www.havenwellwithin.com/")) + .await + .expect("solve"); + assert_eq!(out.handler, "perimeterx"); +} + +#[tokio::test] +async fn test_solve_exact_host_uses_routed_handler() { + let out = dispatcher_with_cf_route() + .solve(SolveRequest::new("https://pedidosya.com.ar/")) + .await + .expect("solve"); + assert_eq!(out.handler, "cloudflare"); +} + +#[tokio::test] +async fn test_solve_subdomain_uses_routed_handler() { + let out = dispatcher_with_cf_route() + .solve(SolveRequest::new("https://www.pedidosya.com.ar/x")) + .await + .expect("solve"); + assert_eq!(out.handler, "cloudflare"); +} + +/// Regression: the cookie bundle's `user_agent` must carry the real +/// harvester UA (so cache-hit replies preserve it), not a placeholder. +#[tokio::test] +async fn test_solve_bundle_user_agent_matches_harvester() { + let out = RoutingDispatcher::new(StaticHandler::new("perimeterx")) + .solve(SolveRequest::new("https://example.com/")) + .await + .expect("solve"); + assert_eq!(out.user_agent, "ua"); + assert_eq!(out.bundle.user_agent, "ua"); +} + +/// Regression: the request's proxy has to reach the handler. It used to be +/// deserialized at the edge and dropped — no browser ever saw it. +#[tokio::test] +async fn test_solve_forwards_requested_proxy_to_handler() { + let handler = StaticHandler::new("perimeterx"); + let dispatcher = RoutingDispatcher::new(Arc::clone(&handler) as Arc); + dispatcher + .solve(SolveRequest::new("https://example.com/").with_proxy("socks5://127.0.0.1:9050")) + .await + .expect("solve"); + assert_eq!( + handler.seen_proxy.lock().await.as_deref(), + Some("socks5://127.0.0.1:9050") + ); +} + +#[tokio::test] +async fn test_solve_without_proxy_forwards_none() { + let handler = StaticHandler::new("perimeterx"); + let dispatcher = RoutingDispatcher::new(Arc::clone(&handler) as Arc); + dispatcher + .solve(SolveRequest::new("https://example.com/")) + .await + .expect("solve"); + assert!(handler.seen_proxy.lock().await.is_none()); +} + +#[test] +fn test_parse_camoufox_domains_trims_and_lowercases() { + let parsed = parse_camoufox_domains(Some(" Pedidosya.com.AR , ,foo.com ")); + assert_eq!(parsed, vec!["pedidosya.com.ar", "foo.com"]); +} + +#[test] +fn test_parse_camoufox_domains_unset_is_empty() { + assert!(parse_camoufox_domains(None).is_empty()); + assert!(parse_camoufox_domains(Some("")).is_empty()); +} diff --git a/px-turnstile/src/lib.rs b/px-turnstile/src/lib.rs index 8051e3c..d3f8e94 100644 --- a/px-turnstile/src/lib.rs +++ b/px-turnstile/src/lib.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use px_errors::AppError; -use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml}; +use px_pipeline::{ChallengeHandler, HandlerOutcome, PageHtml, SolveAction}; pub struct TurnstileHandler; @@ -27,7 +27,7 @@ impl ChallengeHandler for TurnstileHandler { Ok(h.contains("challenges.cloudflare.com/turnstile") || h.contains("cf-turnstile")) } - async fn solve(&self, _page: &PageHtml) -> Result { + async fn solve(&self, _action: &SolveAction) -> Result { Ok(HandlerOutcome::not_implemented(self.name())) } } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ebc2071..a2dfb65 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -124,6 +124,7 @@ fn bump( if new == manifest { bail!("could not find `{from}` in Cargo.toml"); } + let new = repin_workspace_deps(&new, &next); fs::write(manifest_path, new).context("write root Cargo.toml")?; if !skip_gate { @@ -356,6 +357,30 @@ impl std::fmt::Display for Version { } } +/// Re-pin the internal `px-* = { path, version, package = "pxsolver-*" }` +/// entries to the new workspace version. +/// +/// Every crate carries `version.workspace = true`, so the published +/// artifacts all bump together; a stale pin only survives while the new +/// version still satisfies the old caret requirement. A major bump does +/// not, and `cargo` then fails to resolve the workspace at all. +fn repin_workspace_deps(manifest: &str, next: &Version) -> String { + manifest + .lines() + .map(|line| match line.split_once("version = \"") { + Some((head, rest)) if line.contains("package = \"pxsolver-") => { + match rest.split_once('"') { + Some((_old, tail)) => format!("{head}version = \"{next}\"{tail}"), + None => line.to_string(), + } + } + _ => line.to_string(), + }) + .collect::>() + .join("\n") + + "\n" +} + fn parse_workspace_version(manifest: &str) -> Result { let line = manifest .lines()