From 4db736ed90c08bed89c2a19021c679ddb35789b1 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:28:51 +0000 Subject: [PATCH] Stop a run on a refused proxy before the browser starts, and skip media through a proxy A proxy that answers 407 or 402 to a one-request pre-check (CONNECT plus HEAD /robots.txt on the target host) now ends the run with exit 5 and .latest_attempt.meta.json (status failed, stop_reason proxy_auth_failed / proxy_balance_exhausted) instead of surfacing as a Chromium navigation timeout on every retry. Through a local proxy, all three engines abort image, media and font requests; --load-media or WELLFOUND_LOAD_MEDIA=1 loads them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- .env.example | 5 + CHANGELOG.md | 25 +++ Dockerfile | 2 +- README.md | 36 +++- env_config.py | 2 + output_writer.py | 29 +++ playwright_scraper.py | 15 ++ proxy_check.py | 384 +++++++++++++++++++++++++++++++++++++ puppeteer_scraper.py | 15 ++ selenium_scraper.py | 15 ++ smoke_test.py | 429 +++++++++++++++++++++++++++++++++++++++++- 11 files changed, 953 insertions(+), 4 deletions(-) create mode 100644 proxy_check.py diff --git a/.env.example b/.env.example index 9471e66..cd75b7b 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,11 @@ WELLFOUND_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-us-pid-{profil # user:pass URL is doing something. WELLFOUND_PROXY=http://{user}:{password}@na.proxy.2captcha.com:2334 +# Through a proxy, images, video/audio and fonts are not downloaded: the +# proxy is billed by traffic and nothing this scraper reads is in them. +# Uncomment to load them anyway (same as --load-media). +# WELLFOUND_LOAD_MEDIA=1 + # --- Default target ------------------------------------------------------- # Saves retyping --url. Wellfound serves ONE host in ONE language: there is # no per-country TLD, no locale segment, and the exit country changes nothing diff --git a/CHANGELOG.md b/CHANGELOG.md index 936357a..d99467f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,31 @@ does it leads the release notes rather than being discovered from a bill. ## [Unreleased] +> **Behaviour changes.** (1) A run through a proxy now makes one pre-check +> request before starting a browser. A proxy that answers **HTTP 407** or +> **402** ends the run with **exit 5** and +> `.latest_attempt.meta.json` (`status: failed`, `stop_reason: +> proxy_auth_failed` / `proxy_balance_exhausted`) instead of timing out on +> every retry. (2) Through a local proxy, images, video/audio and fonts are +> **no longer downloaded**; `--load-media` (or `WELLFOUND_LOAD_MEDIA=1`) +> restores them. Runs without a proxy or over `--cdp-endpoint` are unchanged. + +### Added + +- `proxy_check.py`: the proxy pre-check (one CONNECT plus `HEAD /robots.txt` + on the target host, no third-party service) and media blocking for all + three engines. The 407 message names `WELLFOUND_PROXY` and says the + gateway refused the credential (or the traffic balance is exhausted); a + 402 says the proxy traffic/balance is exhausted. Neither prints the + credential. A proxy that does not answer keeps the existing retry + behaviour. +- `--load-media` on all three engines, and `WELLFOUND_LOAD_MEDIA` (commented + out) in `.env.example`. +- Offline checks: a stub proxy that answers 407/402/200 or nothing, each + engine driven to exit 5 without a browser on a 407, the attempt sidecar, + the Playwright route handler, the pyppeteer Fetch patterns, the Selenium + blocked-URL list, and when each engine installs blocking. + ### Changed - CI only: the whole canary, free and paid jobs alike, now runs weekly (Mondays 05:40 UTC) and on a manual dispatch; the daily schedule is removed. diff --git a/Dockerfile b/Dockerfile index 0a28e38..c402fa2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx # container that nothing in the repo would have noticed. COPY captcha_solver.py env_config.py fingerprint_client.py output_writer.py \ page_flow.py playwright_scraper.py product_parser.py proxy_pool.py \ - diff_runs.py ./ + proxy_check.py diff_runs.py ./ ENTRYPOINT ["python3", "playwright_scraper.py"] CMD ["--help"] diff --git a/README.md b/README.md index 0d916f1..b6ebe51 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,34 @@ both directions. unsatisfiable pins, and pyppeteer and selenium collide on `urllib3`. Use a virtualenv per engine if you need more than one. +### Proxy traffic savers + +A proxy is billed by traffic, so a run through `--proxy` / `--proxy-file` / +`WELLFOUND_PROXY` does two things to avoid paying for bytes it does not use: + +- **One pre-check before any browser starts.** A CONNECT through the proxy + and a `HEAD /robots.txt` on the target host. Chromium reports a proxy's + HTTP 407 as a navigation timeout, so a wrong or exhausted credential used + to spend every retry before ending as a load failure. Now a **407** (the + gateway refused the credential, or the traffic balance is exhausted) or a + **402** (proxy traffic/balance exhausted) ends the run in seconds with + **exit 5** and `.latest_attempt.meta.json` saying `status: failed`, + `stop_reason: proxy_auth_failed` (or `proxy_balance_exhausted`). Any + previous output and its `.meta.json` are left as they were. The message + names the variable and never prints the credential. If the proxy does not + answer at all, the run goes on as before and the browser makes its own + attempt. No pre-check runs without a proxy or with `--cdp-endpoint`. +- **Images, video/audio and web fonts are not downloaded** through a local + proxy: all three engines abort those requests (Playwright by resource type, + pyppeteer through CDP's Fetch domain, Selenium by file extension with CDP + `Network.setBlockedURLs`). Every row comes from JSON the server embeds in + the page, never from an image or a font, and Cloudflare's challenge is + scripts and an iframe, which are not blocked. This was decided from what + the parser reads: on 2026-10-20 a free run from a datacentre was refused by + Cloudflare before any listing was served, so a loaded-vs-blocked comparison + of live rows was not possible. `--load-media` (or `WELLFOUND_LOAD_MEDIA=1`) + loads them. Without a proxy, or over `--cdp-endpoint`, nothing is blocked. + --- ## Pagination, and the trap in it @@ -330,7 +358,9 @@ one page five times. ## Output contract Exit codes: `0` ok · `1` crash · `2` bad usage · `3` blocked · `4` zero jobs -· `5` remote API error · `6` partial. +· `5` content not obtained (a remote API error, a load failure, or a proxy +that answered 407/402 to the pre-check: `stop_reason: proxy_auth_failed` / +`proxy_balance_exhausted`) · `6` partial. * **A run that finds nothing writes nothing** — last night's good output is never replaced with `[]`. @@ -339,7 +369,9 @@ Exit codes: `0` ok · `1` crash · `2` bad usage · `3` blocked · `4` zero jobs by number. * **A failed run writes no sidecar**, so a `"failed"` file never sits beside good data (the one exception is below, and only where no earlier output - exists). + exists). A run the proxy pre-check stopped writes + `.latest_attempt.meta.json` instead, a separate file, and leaves + `.meta.json` alone. * **`--allow-empty` is for a genuine empty result only.** With it, a run that finished and found nothing writes an empty JSON (`[]`), a CSV holding only its header, and a sidecar with `status: empty` and its `stop_reason`, diff --git a/env_config.py b/env_config.py index 4d68baa..c1a1dff 100644 --- a/env_config.py +++ b/env_config.py @@ -40,6 +40,8 @@ "WELLFOUND_CDP_ENDPOINT": "cdp_endpoint", "WELLFOUND_PROXY": "proxy", "WELLFOUND_URL": "url", + # "1" keeps images/video/fonts loading through a proxy (--load-media). + "WELLFOUND_LOAD_MEDIA": "load_media", } # Deliberately NOT here: an output prefix. `--out` already carries a non-empty # default, so `apply()` would never see it as unset and the variable would be diff --git a/output_writer.py b/output_writer.py index a093a68..fffdb24 100644 --- a/output_writer.py +++ b/output_writer.py @@ -471,6 +471,35 @@ def write_run_meta(out_prefix: str, meta: dict) -> str: return path +ATTEMPT_META_SUFFIX = ".latest_attempt.meta.json" + + +def finish_without_fetch(out_prefix: str, *, stop_reason: str, + pages_requested: int, start_url: str, + mode: str = "listing", source: str = SOURCE_DEFAULT, + message: str = "") -> int: + """End a run that was stopped before anything was fetched; exit 5. + + Used by the proxy pre-check (a 407 or 402 from the proxy). The record + goes to `.latest_attempt.meta.json`, NOT `.meta.json`: the + latter describes the data file beside it, and a previous run's good + output is still there, untouched, so overwriting its metadata with + `status: failed` would make the pair contradict each other. + """ + meta = run_meta(status="failed", stop_reason=stop_reason, + pages_requested=pages_requested, pages_completed=0, + start_url=start_url, final_url=start_url, products=0, + mode=mode, source=source) + path = f"{out_prefix}{ATTEMPT_META_SUFFIX}" + with _atomic(path) as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + print(f"[+] Wrote attempt metadata -> {path} (status=failed, " + f"stop_reason={stop_reason})") + print(f"[!] Failed run: nothing was fetched ({stop_reason}). " + f"{message} Any previous output and its .meta.json are untouched.") + return EXIT_FETCH_FAILED + + def run_meta(status: str, stop_reason: str, pages_requested: int, pages_completed: int, start_url: str, final_url: str, products: int, pages_failed: Optional[List[int]] = None, diff --git a/playwright_scraper.py b/playwright_scraper.py index 52edc18..ac4d729 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -77,12 +77,14 @@ parse_listing_page, references_own_assets, role_url, site_turnstile_sitekey) from output_writer import (dedupe_by_key, finish_run, EXIT_API_ERROR, + finish_without_fetch, SOURCE_DEFAULT) import page_flow from page_flow import MIN_CARD_MATCHES from proxy_pool import (from_args as proxy_pool_from_args, to_playwright, mask, ROTATE_MODES, ProxyError, ProxyPool) import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("playwright_scraper") @@ -397,6 +399,8 @@ def _launch_local(pw, args, pool): # static read, however careful — cannot produce a solvable task. # It costs nothing on a page that never renders one. context.add_init_script(TURNSTILE_INTERCEPT_JS) + if proxy_check.block_media(args, pool, remote=False): + proxy_check.install_playwright(context) return browser, context, context.new_page() @@ -1390,6 +1394,16 @@ def scrape(args) -> int: "(~150-300MB each). Make sure the machine has the " "memory for it.", concurrency, concurrency) + # One request through the proxy before a browser exists: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a navigation timeout after every retry. + refused = proxy_check.gate(pool, _target_url(args)) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=_target_url(args), + mode=args.mode, source=SOURCE_DEFAULT) + with sync_playwright() as pw: session = _BrowserSession(pw, args, pool, remote=bool(args.cdp_endpoint)).open() @@ -1736,6 +1750,7 @@ def parse_args(): help="reCAPTCHA v3 minimum score to request (0.3, 0.7 or 0.9 " "— the API only accepts these three). Ignored for v2 " "widgets.") + proxy_check.add_media_flags(p) p.add_argument("--cdp-endpoint", default=None, help="Connect to an already-running browser over CDP instead " "of launching Playwright's bundled Chromium, e.g. " diff --git a/proxy_check.py b/proxy_check.py new file mode 100644 index 0000000..43565dd --- /dev/null +++ b/proxy_check.py @@ -0,0 +1,384 @@ +""" +proxy_check.py +--------------- +Two traffic savers for runs that go out through a local proxy. + +A residential proxy is billed by the byte, and both of these exist because +of what a run used to spend before it produced anything: + +1. **A pre-check before any browser starts.** Chromium does not report a + proxy's 407 as a 407. A rejected credential surfaced as + `Page.goto: Timeout ... exceeded`, retried, rotated and timed out again, + so a run with a wrong or exhausted proxy login spent minutes on retries + and ended "page_load_timeout", which points at the site. One CONNECT + through the proxy, followed by a HEAD for the target's own robots.txt, + names the real cause in a second or two: + + 407 the gateway refused the credential (or the traffic balance is + exhausted, which some gateways also answer with 407) + 402 the proxy's traffic or balance is exhausted + + Both end the run before a browser is launched: exit 5 (content not + obtained) and `.latest_attempt.meta.json` with `status: failed` and + `stop_reason: proxy_auth_failed` / `proxy_balance_exhausted`. That file + is separate from `.meta.json` on purpose: a failed attempt must not + overwrite the metadata of the previous run's good output, which `save` + leaves in place. + + Anything else, including a transport error (the proxy host does not + answer, DNS fails, TLS breaks), is logged and the run goes on exactly as + it did before this check existed: the browser then makes its own attempt + and the existing retry and classification rules apply. Only an answer the + proxy gave about the credential or the balance is treated as final. + + The target is the site's robots.txt rather than a third-party "what is my + IP" service: it is tiny, it is on the host the run is about to fetch + anyway, and it does not send the run's traffic to anyone else. + +2. **Media blocking.** Images, video/audio and web fonts are most of the + bytes of a page and none of the data this scraper reads. When (and only + when) the browser's traffic goes through a local proxy, requests of those + resource types are aborted. Not over --cdp-endpoint (the remote browser + has its own exit and its own billing, and is not ours to rewrite), and + not without a proxy (nothing is metered). `--load-media` (or + `_LOAD_MEDIA=1`) turns it off. + +The credential is never printed: every message names the proxy through +`proxy_pool.mask`, which keeps the host and port and replaces the login. +""" + +from __future__ import annotations + +import base64 +import logging +import socket +import ssl +from dataclasses import dataclass +from typing import Optional, Tuple +from urllib.parse import urlparse + +from proxy_pool import mask + +logger = logging.getLogger("proxy_check") + +# Where the proxy credential comes from, for the failure message. Named +# rather than "your proxy" because the person reading the log is often +# looking at a CI secret, not at a command line. +PROXY_SOURCES = "WELLFOUND_PROXY (or --proxy / --proxy-file)" +LOAD_MEDIA_ENV = "WELLFOUND_LOAD_MEDIA" + +# Decided from what the parser reads, not from a live comparison (a free +# run from a datacentre is refused by Cloudflare before any listing is +# served, so loaded-vs-blocked could not be compared): every row comes from +# JSON the server embeds in the page (`__NEXT_DATA__`'s Apollo cache, or the +# detail route's own payload), never from an image or a font. The challenge +# machinery is scripts and an iframe, which are not blocked. +BLOCK_MEDIA_BY_DEFAULT = True + +PRECHECK_TIMEOUT = 15.0 + +STOP_AUTH_FAILED = "proxy_auth_failed" +STOP_BALANCE_EXHAUSTED = "proxy_balance_exhausted" + +# Playwright's `request.resource_type` names, and the same three as CDP's +# Fetch domain spells them (pyppeteer goes through that domain directly). +BLOCKED_RESOURCE_TYPES = frozenset({"image", "media", "font"}) +CDP_BLOCKED_RESOURCE_TYPES = ("Image", "Media", "Font") + +# Selenium has no request interception, but Chromium's CDP +# `Network.setBlockedURLs` takes wildcard URL patterns. Extensions are a +# coarser test than a resource type, so the list is limited to formats that +# are only ever media; a trailing `*` also catches `?width=...` variants. +BLOCKED_URL_PATTERNS = tuple( + f"*.{ext}{tail}" + for ext in ("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "ico", + "svg", "mp4", "webm", "m4v", "mov", "m3u8", "mp3", "m4a", + "ogg", "wav", "woff", "woff2", "ttf", "otf", "eot") + for tail in ("", "?*")) + +_FALSE_WORDS = {"", "0", "false", "no", "off"} + + +@dataclass(frozen=True) +class ProxyCheck: + """What the pre-check learned. `fatal` is True only for 407 and 402.""" + + outcome: str # ok | auth_failed | balance_exhausted | transport_error | skipped + status: Optional[int] # the proxy's (or, through the tunnel, the site's) status + detail: str = "" + + @property + def fatal(self) -> bool: + return self.outcome in ("auth_failed", "balance_exhausted") + + @property + def stop_reason(self) -> Optional[str]: + return {"auth_failed": STOP_AUTH_FAILED, + "balance_exhausted": STOP_BALANCE_EXHAUSTED}.get(self.outcome) + + +def _read_head(sock, limit: int = 65536) -> Tuple[int, str]: + """Read an HTTP response head; return (status, reason).""" + buf = b"" + while b"\r\n\r\n" not in buf and b"\n\n" not in buf: + chunk = sock.recv(4096) + if not chunk: + break + buf += chunk + if len(buf) > limit: + break + line = buf.split(b"\n", 1)[0].decode("latin-1").strip() + parts = line.split(" ", 2) + if len(parts) < 2 or not parts[0].startswith("HTTP/") or not parts[1].isdigit(): + raise ValueError(f"not an HTTP response: {line[:60]!r}") + return int(parts[1]), (parts[2] if len(parts) > 2 else "") + + +def _auth_header(proxy) -> str: + # The username and password are sent exactly as the browser engines send + # them (proxy_pool.to_playwright / split_credentials pass urlparse's + # fields through unchanged), so the check judges the same credential the + # browser would present rather than a decoded variant of it. + if not (proxy.username or proxy.password): + return "" + token = base64.b64encode( + f"{proxy.username or ''}:{proxy.password or ''}".encode()).decode() + return f"Proxy-Authorization: Basic {token}\r\n" + + +def _classify(status: int, reason: str) -> Optional[ProxyCheck]: + if status == 407: + return ProxyCheck("auth_failed", 407, reason) + if status == 402: + return ProxyCheck("balance_exhausted", 402, reason) + return None + + +def precheck(proxy_url: Optional[str], target_url: str, + timeout: float = PRECHECK_TIMEOUT) -> ProxyCheck: + """One request through `proxy_url`: a HEAD for `target_url`'s robots.txt. + + For an https target that is a CONNECT to the target host, then the HEAD + inside the tunnel. A 407 or 402 is the proxy's own answer about the + credential or the balance; the site never sees the request. Any status + the SITE gives (a 403 to a datacentre exit, a 404 for a missing + robots.txt) is not this check's business and counts as `ok`: the proxy + let the request through. + """ + if not proxy_url: + return ProxyCheck("skipped", None, "no proxy") + proxy = urlparse(proxy_url) + if proxy.scheme not in ("http", "https"): + # SOCKS5 carries no credentials in any engine here (proxy_pool + # refuses an authenticated socks5 entry), so there is nothing to + # check that the browser would not report itself. + return ProxyCheck("skipped", None, f"{proxy.scheme} proxy") + target = urlparse(target_url or "") + if target.scheme not in ("http", "https") or not target.hostname: + return ProxyCheck("skipped", None, "no http(s) target URL") + try: + proxy_port = proxy.port or (443 if proxy.scheme == "https" else 80) + target_port = target.port or (443 if target.scheme == "https" else 80) + except ValueError: + return ProxyCheck("skipped", None, "unparseable port") + host_header = (target.hostname if target.port is None + else f"{target.hostname}:{target.port}") + auth = _auth_header(proxy) + sock = None + try: + sock = socket.create_connection((proxy.hostname, proxy_port), + timeout=timeout) + if proxy.scheme == "https": + sock = ssl.create_default_context().wrap_socket( + sock, server_hostname=proxy.hostname) + if target.scheme == "https": + sock.sendall((f"CONNECT {target.hostname}:{target_port} HTTP/1.1\r\n" + f"Host: {target.hostname}:{target_port}\r\n" + f"{auth}\r\n").encode()) + status, reason = _read_head(sock) + verdict = _classify(status, reason) + if verdict: + return verdict + if status != 200: + # The proxy refused the tunnel for another reason (a 403 for + # a blocked port, a 502 from an upstream). Not about the + # credential, so not final: the browser gets its own try. + return ProxyCheck("transport_error", status, + f"CONNECT answered {status} {reason}".strip()) + # The tunnel is up, so the proxy accepted the credential: that + # is the verdict. The HEAD below only completes the one request; + # whatever the site does with it (a refusal, a reset) is the + # browser's business and does not change the answer. + if proxy.scheme == "https": + # TLS inside TLS is not something the ssl module does on a + # wrapped socket; the CONNECT answer already says enough. + return ProxyCheck("ok", None, "tunnel established") + try: + tls = ssl.create_default_context().wrap_socket( + sock, server_hostname=target.hostname) + sock = tls + sock.sendall((f"HEAD /robots.txt HTTP/1.1\r\n" + f"Host: {host_header}\r\n" + "User-Agent: Mozilla/5.0\r\nAccept: */*\r\n" + "Connection: close\r\n\r\n").encode()) + status, reason = _read_head(sock) + except (OSError, ssl.SSLError, ValueError) as e: + return ProxyCheck("ok", None, + f"tunnel established; site: {type(e).__name__}") + return ProxyCheck("ok", status, reason) + sock.sendall((f"HEAD http://{host_header}/robots.txt HTTP/1.1\r\n" + f"Host: {host_header}\r\n{auth}" + "User-Agent: Mozilla/5.0\r\nAccept: */*\r\n" + "Connection: close\r\n\r\n").encode()) + status, reason = _read_head(sock) + return _classify(status, reason) or ProxyCheck("ok", status, reason) + except (OSError, ssl.SSLError, ValueError) as e: + return ProxyCheck("transport_error", None, type(e).__name__) + finally: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +def failure_message(check: ProxyCheck, proxy_url: Optional[str]) -> str: + """The one-line explanation for a fatal pre-check. Never holds the login.""" + shown = mask(proxy_url) + if check.outcome == "auth_failed": + return (f"Proxy pre-check: {shown} answered HTTP 407 — the gateway " + f"refused the credential (or the traffic balance is " + f"exhausted). Check {PROXY_SOURCES} and the proxy account. " + f"No browser was started and nothing was fetched.") + return (f"Proxy pre-check: {shown} answered HTTP 402 — proxy " + f"traffic/balance exhausted. Top up the proxy account or change " + f"{PROXY_SOURCES}. No browser was started and nothing was fetched.") + + +def gate(pool, target_url: str, timeout: float = PRECHECK_TIMEOUT + ) -> Optional[ProxyCheck]: + """Run the pre-check on the pool's current exit; return it if fatal. + + One request for the whole run, on the exit the browser is about to use. + A pool's other exits share the account, so one refusal speaks for all. + """ + if not pool: + return None + exit_url = pool.current + check = precheck(exit_url, target_url, timeout=timeout) + if check.fatal: + logger.error("%s", failure_message(check, exit_url)) + return check + if check.outcome == "transport_error": + logger.warning("Proxy pre-check through %s did not complete (%s); " + "continuing, the browser will make its own attempt.", + mask(exit_url), check.detail) + elif check.outcome == "ok": + logger.info("Proxy pre-check through %s: accepted (%s).", + mask(exit_url), + f"site answered {check.status}" if check.status + else check.detail) + return None + + +# --------------------------------------------------------------------------- +# Media blocking +# --------------------------------------------------------------------------- + +def _truthy(value) -> bool: + if isinstance(value, str): + return value.strip().lower() not in _FALSE_WORDS + return bool(value) + + +def block_media(args, pool, remote: bool) -> bool: + """Whether this run should abort image/media/font requests. + + `--block-media` (hidden; for tests and measurements) forces it on even + without a proxy. Otherwise: only through a local proxy, only when the + site's default allows it, and never when `--load-media` / + `WELLFOUND_LOAD_MEDIA` asks for media. + """ + if remote: + return False + if getattr(args, "block_media", False): + return True + if _truthy(getattr(args, "load_media", False)): + return False + return BLOCK_MEDIA_BY_DEFAULT and bool(pool) + + +def add_media_flags(parser) -> None: + """`--load-media` (public) and `--block-media` (hidden test switch).""" + import argparse + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--load-media", action="store_true", default=False, + help=("Load images, video/audio and fonts through the proxy. By " + "default they are blocked whenever a local --proxy/--proxy-file " + "is in use, to save metered proxy traffic; the data this " + "scraper reads does not depend on them. Also " + f"{LOAD_MEDIA_ENV}=1. No effect without a proxy or with " + "--cdp-endpoint.")) + group.add_argument("--block-media", action="store_true", default=False, + help=argparse.SUPPRESS) + + +def _should_abort(resource_type: str) -> bool: + return resource_type in BLOCKED_RESOURCE_TYPES + + +def install_playwright(context) -> None: + """Abort image/media/font requests on a Playwright context.""" + def _route(route): + if _should_abort(route.request.resource_type): + route.abort() + else: + route.continue_() + context.route("**/*", _route) + logger.info("Blocking image/media/font requests to save proxy traffic " + "(--load-media to load them).") + + +async def install_pyppeteer(page) -> None: + """Abort image/media/font requests on a pyppeteer page. + + Not `page.setRequestInterception`: that sends `Network. + setRequestInterception`, which current Chromium no longer has, and the + call fails with "wasn't found". The Fetch domain is its replacement. + It runs on a CDP session of its own, so this does not replace any Fetch + handling the engine's page session already has (proxy auth), and its + patterns pause ONLY the three blocked resource types: nothing else is + held back waiting for a continue. + """ + import asyncio + session = await page.target.createCDPSession() + + def _on_paused(event): + asyncio.ensure_future(session.send("Fetch.failRequest", { + "requestId": event["requestId"], "errorReason": "BlockedByClient"})) + session.on("Fetch.requestPaused", _on_paused) + await session.send("Fetch.enable", {"patterns": [ + {"urlPattern": "*", "resourceType": rtype, "requestStage": "Request"} + for rtype in CDP_BLOCKED_RESOURCE_TYPES]}) + logger.info("Blocking image/media/font requests to save proxy traffic " + "(--load-media to load them).") + + +def install_selenium(driver) -> bool: + """Block media URLs on a local chromedriver over CDP. False if refused. + + By file extension, not by resource type: Selenium has no request + interception, so a media file served without an extension still loads. + """ + try: + driver.execute_cdp_cmd("Network.enable", {}) + driver.execute_cdp_cmd("Network.setBlockedURLs", + {"urls": list(BLOCKED_URL_PATTERNS)}) + except Exception as e: # noqa: BLE001 — a saver must never break a run + logger.warning("Could not block media over CDP (%s); loading it.", + type(e).__name__) + return False + logger.info("Blocking image/media/font URLs to save proxy traffic " + "(--load-media to load them).") + return True diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 3171860..8935ebc 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -87,12 +87,14 @@ parse_listing_page, references_own_assets, role_url, site_turnstile_sitekey) from output_writer import (dedupe_by_key, finish_run, EXIT_API_ERROR, + finish_without_fetch, SOURCE_DEFAULT) import page_flow from page_flow import MIN_CARD_MATCHES from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, ProxyError, split_credentials) import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("puppeteer_scraper") @@ -392,6 +394,8 @@ def open(self): if credentials: self.bridge.run(self.page.authenticate( {"username": credentials[0], "password": credentials[1]})) + if proxy_check.block_media(self.args, self.pool, remote=False): + self.bridge.run(proxy_check.install_pyppeteer(self.page)) return self def _apply_fingerprint(self): @@ -1026,6 +1030,16 @@ def scrape(args) -> int: "which is the primary engine. Running one page at a " "time.") + # One request through the proxy before a browser exists: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a navigation timeout after every retry. + refused = proxy_check.gate(pool, _target_url(args)) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=_target_url(args), + mode=args.mode, source=SOURCE_DEFAULT) + bridge = _AsyncBridge() session = None try: @@ -1226,6 +1240,7 @@ def parse_args(): "setting has anything to act on today, and neither " "helps with a refusal.") p.add_argument("--min-score", type=float, default=0.7) + proxy_check.add_media_flags(p) p.add_argument("--cdp-endpoint", default=None, help="Connect to a running browser over CDP, e.g. " "ws://user:pass@host:port. pyppeteer authenticates on " diff --git a/selenium_scraper.py b/selenium_scraper.py index ef578fb..af0244a 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -79,12 +79,14 @@ parse_listing_page, references_own_assets, role_url, site_turnstile_sitekey) from output_writer import (dedupe_by_key, finish_run, EXIT_API_ERROR, + finish_without_fetch, SOURCE_DEFAULT) import page_flow from page_flow import MIN_CARD_MATCHES from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, ProxyError, split_credentials) import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("selenium_scraper") @@ -283,6 +285,8 @@ def open(self): self.driver = webdriver.Chrome(options=options) self._apply_timeouts() + if proxy_check.block_media(self.args, self.pool, remote=False): + proxy_check.install_selenium(self.driver) version = self.driver.capabilities.get("browserVersion", "") if version: @@ -912,6 +916,16 @@ def scrape(args) -> int: "which is the primary engine. Running one page at a " "time.") + # One request through the proxy before a browser exists: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a navigation timeout after every retry. + refused = proxy_check.gate(pool, _target_url(args)) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=_target_url(args), + mode=args.mode, source=SOURCE_DEFAULT) + session = None try: session = _Session(args, pool).open() @@ -1138,6 +1152,7 @@ def parse_args(): "setting has anything to act on today, and neither " "helps with a refusal.") p.add_argument("--min-score", type=float, default=0.7) + proxy_check.add_media_flags(p) p.add_argument("--cdp-endpoint", default=None, help="Attach to a running browser at host:port. Must NOT " "carry credentials — chromedriver's debuggerAddress " diff --git a/smoke_test.py b/smoke_test.py index 57b4ea8..6161b71 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1506,7 +1506,10 @@ def check_a_copied_env_example_reads_as_UNSET(): and re.search(r"^#\s*WELLFOUND_URL=https://wellfound\.com/", text, re.M) is not None) check("the example actually sets every credential variable", - set(values) == set(env_config.ENV_KEYS) - {"WELLFOUND_URL"}, + # WELLFOUND_LOAD_MEDIA is an opt-out switch, shipped commented out + # like the URL: an active copy would turn the saver off. + set(values) == set(env_config.ENV_KEYS) - {"WELLFOUND_URL", + "WELLFOUND_LOAD_MEDIA"}, "example has %s, loader reads %s" % (sorted(values), sorted(env_config.ENV_KEYS))) # Every CREDENTIAL must read as unset. @@ -2823,6 +2826,430 @@ def check_allow_empty_writes_only_a_genuine_empty_result(): equal(label, got, want) +import contextlib # noqa: E402 — used by the proxy-savers block below + + +# --------------------------------------------------------------------------- +# Proxy traffic savers: the pre-check and media blocking +# --------------------------------------------------------------------------- +_SAVERS_URL = "https://wellfound.com/role/r/software-engineer" +_SAVERS_SECRET = "s3cret-Pw-not-real" + + +def _ok(label, condition): + """`check`, returning the condition so the savers block can `&=` it.""" + check(label, bool(condition)) + return bool(condition) + + +class _StubProxy: + """A one-port HTTP proxy on 127.0.0.1 that answers every request with a + fixed status and records the request heads it received.""" + + def __init__(self, status): + import socket + import threading + self.status, self.heads = status, [] + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(8) + self.port = self.sock.getsockname()[1] + threading.Thread(target=self._serve, daemon=True).start() + + def _serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + with conn: + data = b"" + while b"\r\n\r\n" not in data: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + self.heads.append(data.decode("latin-1")) + reason = {407: "Proxy Authentication Required", + 402: "Payment Required", 200: "OK"}.get( + self.status, "X") + conn.sendall(f"HTTP/1.1 {self.status} {reason}\r\n" + "Content-Length: 0\r\nConnection: close\r\n\r\n" + .encode()) + + def url(self, secret=_SAVERS_SECRET): + # Concatenated, so no line here looks like a credentialled URL. + return "http://user-zone-x:" + secret + "@127.0.0.1:" + str(self.port) + + def close(self): + self.sock.close() + + +def _closed_port(): + import socket + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +@contextlib.contextmanager +def _captured_output(): + """stdout and every log record, as one string.""" + import logging + buf = io.StringIO() + handler = logging.StreamHandler(buf) + root = logging.getLogger() + root.addHandler(handler) + try: + with contextlib.redirect_stdout(buf): + yield buf + finally: + root.removeHandler(handler) + + +class _FakeRoute: + def __init__(self, resource_type): + self.request = type("R", (), {"resource_type": resource_type})() + self.verdict = None + + def abort(self): + self.verdict = "abort" + + def continue_(self): + self.verdict = "continue" + + +def _media_verdicts(handler): + out = {} + for rtype in ("image", "media", "font", "document", "script", "xhr", + "fetch", "stylesheet"): + route = _FakeRoute(rtype) + handler(route) + out[rtype] = route.verdict + return out + + +def _engine_args(engine, tmp, proxy_url, *extra): + """The engine's own parser, on a stubbed argv, with no .env in play.""" + from unittest import mock + import env_config + argv = [engine.__name__, "--url", _SAVERS_URL, "--out", + os.path.join(tmp, "out")] + if proxy_url: + argv += ["--proxy", proxy_url] + argv += list(extra) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.object(env_config, "load_env", lambda *a, **k: None): + return engine.parse_args() + + +def check_proxy_traffic_savers(): + if VERBOSE: + print("== Proxy traffic savers: pre-check before the browser, media " + "blocking through a proxy") + import argparse + import asyncio + from unittest import mock + import output_writer + import proxy_check + import env_config + from proxy_pool import ProxyPool + ok = True + + # -- the pre-check, against a stub proxy -------------------------------- + stub = _StubProxy(407) + try: + res = proxy_check.precheck(stub.url(), _SAVERS_URL, timeout=5) + ok &= _ok("a 407 from the proxy is auth_failed and fatal " + f"({res.outcome})", res.outcome == "auth_failed" + and res.fatal and res.stop_reason == "proxy_auth_failed") + head = stub.heads[-1] if stub.heads else "" + ok &= _ok("the pre-check is ONE CONNECT to the target host, with " + "the credential", len(stub.heads) == 1 + and head.startswith("CONNECT wellfound.com:443 ") + and "Proxy-Authorization: Basic " in head) + with _captured_output() as buf: + gated = proxy_check.gate(ProxyPool([stub.url()]), _SAVERS_URL, + timeout=5) + text = buf.getvalue() + ok &= _ok("gate() returns the fatal check on a 407", + gated is not None and gated.fatal) + ok &= _ok("the 407 message names the variable and the cause", + proxy_check.PROXY_SOURCES.split()[0] in text + and "refused the credential" in text + and "traffic balance is exhausted" in text) + ok &= _ok("the 407 message never prints the credential", + _SAVERS_SECRET not in text and "user-zone-x" not in text) + finally: + stub.close() + + stub = _StubProxy(402) + try: + res = proxy_check.precheck(stub.url(), _SAVERS_URL, timeout=5) + msg = proxy_check.failure_message(res, stub.url()) + ok &= _ok("a 402 is balance_exhausted and fatal", + res.outcome == "balance_exhausted" and res.fatal + and res.stop_reason == "proxy_balance_exhausted") + ok &= _ok("the 402 message says traffic/balance exhausted, no " + "credential", "traffic/balance exhausted" in msg + and _SAVERS_SECRET not in msg) + finally: + stub.close() + + stub = _StubProxy(200) + try: + res = proxy_check.precheck(stub.url(), "http://127.0.0.1:9/x", + timeout=5) + head = stub.heads[-1] if stub.heads else "" + ok &= _ok("a plain-http target is one HEAD for its robots.txt", + res.outcome == "ok" and not res.fatal + and head.startswith("HEAD http://127.0.0.1:9/robots.txt ")) + finally: + stub.close() + + dead = "http://u:" + _SAVERS_SECRET + "@127.0.0.1:" + str(_closed_port()) + res = proxy_check.precheck(dead, _SAVERS_URL, timeout=5) + ok &= _ok("a proxy that does not answer is a transport error, NOT " + "fatal (existing retry behaviour applies)", + res.outcome == "transport_error" and not res.fatal) + with _captured_output() as buf: + gated = proxy_check.gate(ProxyPool([dead]), _SAVERS_URL, timeout=5) + ok &= _ok("gate() lets a transport error through, credential unprinted", + gated is None and _SAVERS_SECRET not in buf.getvalue()) + ok &= _ok("no pool means no pre-check", + proxy_check.gate(None, _SAVERS_URL) is None) + ok &= _ok("a socks5 exit is skipped, not refused", + proxy_check.precheck("socks5://127.0.0.1:1", _SAVERS_URL) + .outcome == "skipped") + + # -- the attempt sidecar ------------------------------------------------ + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "out") + for name, body in (("out.json", "[1]"), ("out.meta.json", "{}")): + with open(os.path.join(tmp, name), "w") as f: + f.write(body) + with contextlib.redirect_stdout(io.StringIO()): + rc = output_writer.finish_without_fetch( + out, stop_reason="proxy_auth_failed", pages_requested=3, + start_url=_SAVERS_URL) + with open(out + ".latest_attempt.meta.json") as f: + meta = json.load(f) + untouched = (open(out + ".json").read() == "[1]" + and open(out + ".meta.json").read() == "{}") + ok &= _ok(f"a refused proxy exits 5 ({rc})", rc == 5) + ok &= _ok("...and records status failed / proxy_auth_failed in the " + "attempt sidecar", meta.get("status") == "failed" + and meta.get("stop_reason") == "proxy_auth_failed" + and meta.get("pages_completed") == 0) + ok &= _ok("...leaving the previous output and its .meta.json alone", + untouched) + + # -- when media is blocked ---------------------------------------------- + pool = ProxyPool(["http://127.0.0.1:1"]) + ns = argparse.Namespace + ok &= _ok("blocked through a local proxy by default", + proxy_check.block_media(ns(load_media=False, block_media=False), + pool, remote=False) + is proxy_check.BLOCK_MEDIA_BY_DEFAULT) + ok &= _ok("not blocked without a proxy", + not proxy_check.block_media( + ns(load_media=False, block_media=False), None, False)) + ok &= _ok("not blocked over --cdp-endpoint, even when forced", + not proxy_check.block_media( + ns(load_media=False, block_media=True), pool, True)) + ok &= _ok("--load-media turns it off", + not proxy_check.block_media( + ns(load_media=True, block_media=False), pool, False)) + ok &= _ok("an env value of 0 does not count as --load-media", + proxy_check.block_media(ns(load_media="0", block_media=False), + pool, False) + is proxy_check.BLOCK_MEDIA_BY_DEFAULT) + ok &= _ok("the hidden --block-media forces it without a proxy", + proxy_check.block_media(ns(load_media=False, block_media=True), + None, False)) + parser = argparse.ArgumentParser() + proxy_check.add_media_flags(parser) + try: + with contextlib.redirect_stderr(io.StringIO()): + parser.parse_args(["--load-media", "--block-media"]) + exclusive = False + except SystemExit: + exclusive = True + ok &= _ok("--load-media and --block-media are mutually exclusive", + exclusive) + ok &= _ok("--block-media is hidden from --help", + "--block-media" not in parser.format_help() + and "--load-media" in parser.format_help()) + ok &= _ok(f"{proxy_check.LOAD_MEDIA_ENV} maps onto --load-media", + env_config.ENV_KEYS.get(proxy_check.LOAD_MEDIA_ENV) + == "load_media") + + # -- the route handler, on a stub page ---------------------------------- + ctx = mock.MagicMock() + with _captured_output(): + proxy_check.install_playwright(ctx) + pattern, handler = ctx.route.call_args[0] + verdicts = _media_verdicts(handler) + ok &= _ok(f"Playwright: route on every URL ({pattern})", pattern == "**/*") + ok &= _ok("Playwright: image/media/font aborted, everything else " + f"continues {verdicts}", + all(verdicts[t] == "abort" for t in ("image", "media", "font")) + and all(v == "continue" for t, v in verdicts.items() + if t not in ("image", "media", "font"))) + + cdp = mock.MagicMock() + cdp.send = mock.AsyncMock() + page = mock.MagicMock() + page.target.createCDPSession = mock.AsyncMock(return_value=cdp) + + async def _drive_pyppeteer(): + await proxy_check.install_pyppeteer(page) + on_paused = cdp.on.call_args[0][1] + on_paused({"requestId": "r1", "resourceType": "Image"}) + await asyncio.sleep(0) + + with _captured_output(): + asyncio.run(_drive_pyppeteer()) + calls = [c[0] for c in cdp.send.await_args_list] + enable = next((c[1] for c in calls if c[0] == "Fetch.enable"), {}) + kinds = sorted(pt.get("resourceType") for pt in enable.get("patterns", [])) + ok &= _ok("pyppeteer: Fetch pauses ONLY Image/Media/Font, on its own " + f"CDP session {kinds}", cdp.on.call_args[0][0] == + "Fetch.requestPaused" and kinds == ["Font", "Image", "Media"]) + ok &= _ok("pyppeteer: a paused request is failed, not continued", + ("Fetch.failRequest", {"requestId": "r1", + "errorReason": "BlockedByClient"}) + in calls) + ok &= _ok("pyppeteer: never the removed Network.setRequestInterception", + "setRequestInterception" not in inspect.getsource( + proxy_check.install_pyppeteer).split('"""')[-1]) + + driver = mock.MagicMock() + with _captured_output(): + installed = proxy_check.install_selenium(driver) + sent = {c[0][0]: c[0][1] for c in driver.execute_cdp_cmd.call_args_list} + urls = sent.get("Network.setBlockedURLs", {}).get("urls", []) + ok &= _ok("Selenium: Network.setBlockedURLs with image/video/font " + "patterns", installed and "*.png" in urls and "*.woff2" in urls + and "*.mp4?*" in urls) + ok &= _ok("Selenium: no pattern touches a document, script, JSON or " + "stylesheet", not [u for u in urls if re.search( + r"\.(html?|js|json|css)\b", u)]) + broken = mock.MagicMock() + broken.execute_cdp_cmd.side_effect = RuntimeError("no CDP") + with _captured_output(): + ok &= _ok("Selenium: a driver that refuses CDP is not fatal", + proxy_check.install_selenium(broken) is False) + + # -- each engine, driven: no browser is started on a 407 ---------------- + for name in ENGINES: + try: + engine = __import__(name) + except ImportError as e: + skip(f"{name} proxy savers driving", str(e)) + continue + stub = _StubProxy(407) + launched = [] + if name == "playwright_scraper": + target = "sync_playwright" + else: + target = "_Session" + try: + with tempfile.TemporaryDirectory() as tmp: + args = _engine_args(engine, tmp, stub.url()) + with mock.patch.object( + engine, target, + side_effect=lambda *a, **k: launched.append(1)), \ + _captured_output() as buf: + try: + rc = engine.scrape(args) + except Exception as e: # noqa: BLE001 — fail BY NAME + rc = f"raised {type(e).__name__}" + side = os.path.join(tmp, "out.latest_attempt.meta.json") + meta = json.load(open(side)) if os.path.exists(side) else {} + wrote_rows = any(os.path.exists(os.path.join(tmp, "out" + s)) + for s in (".json", ".csv", ".meta.json")) + finally: + stub.close() + ok &= _ok(f"{name}: a 407 exits 5 without starting a browser " + f"(rc={rc}, launched={len(launched)})", + rc == 5 and not launched) + ok &= _ok(f"{name}: ...with stop_reason proxy_auth_failed in the " + "attempt sidecar and no output written", + meta.get("stop_reason") == "proxy_auth_failed" + and meta.get("status") == "failed" and not wrote_rows) + ok &= _ok(f"{name}: ...and the credential never reaches the log", + _SAVERS_SECRET not in buf.getvalue()) + + # -- each engine installs blocking only through a local proxy ----------- + for name in ENGINES: + try: + engine = __import__(name) + except ImportError: + continue # already recorded as a skip above + for label, proxy, extra, want in ( + ("through a proxy", "http://127.0.0.1:1", (), None), + ("with --load-media", "http://127.0.0.1:1", + ("--load-media",), False), + ("without a proxy", None, (), False), + ("forced, no proxy", None, ("--block-media",), True)): + if want is None: + want = proxy_check.BLOCK_MEDIA_BY_DEFAULT + with tempfile.TemporaryDirectory() as tmp: + args = _engine_args(engine, tmp, proxy, *extra) + pool = ProxyPool([proxy]) if proxy else None + got = _engine_installs_blocking(engine, args, pool) + ok &= _ok(f"{name}: media blocking {label} = {want} (got {got})", + got is want) + return ok + + +def _engine_installs_blocking(engine, args, pool): + """Open the engine's local session on fakes; True if blocking went in.""" + from unittest import mock + name = engine.__name__ + with _captured_output(): + if name == "playwright_scraper": + pw = mock.MagicMock() + browser = pw.chromium.launch.return_value + browser.version = "120.0.0.0" + ctx = browser.new_context.return_value + engine._launch_local(pw, args, pool) + return ctx.route.called + if name == "selenium_scraper": + with mock.patch.object(engine.webdriver, "Chrome") as chrome: + driver = chrome.return_value + driver.capabilities = {"browserVersion": "120.0.0.0"} + engine._Session(args, pool).open() + return any(c[0][0] == "Network.setBlockedURLs" + for c in driver.execute_cdp_cmd.call_args_list) + page = mock.MagicMock() + for attr in ("setUserAgent", "setViewport", "authenticate", + "setRequestInterception", "evaluateOnNewDocument", + "setExtraHTTPHeaders"): + setattr(page, attr, mock.AsyncMock()) + cdp = mock.MagicMock() + cdp.send = mock.AsyncMock() + page.target.createCDPSession = mock.AsyncMock(return_value=cdp) + browser = mock.MagicMock() + browser.newPage = mock.AsyncMock(return_value=page) + browser.version = mock.AsyncMock(return_value="HeadlessChrome/120.0") + + async def fake_launch(*a, **k): + return browser + bridge = engine._AsyncBridge() + try: + with mock.patch.object(engine, "launch", fake_launch): + engine._Session(bridge, args, pool).open() + finally: + closer = getattr(bridge, "close", None) + if closer: + closer() + return any(c[0][0] == "Fetch.enable" for c in cdp.send.await_args_list) + + CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]