From 8ccb820ed5ecd3ecdfc2ba65ef89cc76611b174c Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:58:34 +0000 Subject: [PATCH] Refuse negative retries and delays; canary tests the browser path - --retries, --proxy-block-retries, --delay and --retry-delay (all three engines) and --retries/--retry-delay (Scraper API client) reject negative values with an argparse error naming the flag. --retries 0 stays legal: it counts extra attempts. - Comments, docstrings and config explain each safeguard in place rather than pointing at an internal document; a suite check keeps it so. - .env.example no longer states a fixed credential lifetime for the Scraping Browser API. - The file-mode check runs under umask 022, 002 and 077. - Daily canary: one profile through a local browser; the Scraper API job fails naming TWOCAPTCHA_KEY instead of skipping green. Co-Authored-By: Claude Opus 5.5 --- .env.example | 11 +- .github/ci_checks.py | 20 +-- .github/workflows/canary.yml | 88 ++++++++--- .github/workflows/tests.yml | 4 +- .gitignore | 4 +- CHANGELOG.md | 22 +++ README.md | 10 +- captcha_solver.py | 52 ++----- env_config.py | 2 +- fingerprint_client.py | 4 +- http_transport.py | 6 +- make_fixtures.py | 8 +- output_writer.py | 19 ++- page_flow.py | 54 +++++-- playwright_scraper.py | 96 ++++++------ product_parser.py | 29 ++-- puppeteer_scraper.py | 91 ++++++----- scraper_api_client.py | 11 +- selenium_scraper.py | 85 +++++------ smoke_test.py | 284 ++++++++++++++++++++++++----------- snapchat_payload.py | 37 ++--- 21 files changed, 549 insertions(+), 388 deletions(-) diff --git a/.env.example b/.env.example index b92ae2e..0313aba 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,11 @@ # .env — copy to `.env` beside the scripts. Never commit the copy. # -# Precedence, highest first (CLAUDE.md §3): +# Precedence, highest first: # an explicit flag -> an exported environment variable -> .env -> default # # A value still carrying a {placeholder} is treated as UNSET, so a copied -# example never reaches an API as if it were a key. CLAUDE.md §17 records -# the run this rule exists to prevent: a copied example connected to a +# example never reaches an API as if it were a key. Without this rule a +# copied example connected to a # vendor with the literal string "{login}-zone-..." as its username and # got a 401 a long way from its cause. # @@ -23,9 +23,8 @@ TWOCAPTCHA_KEY={your_2captcha_api_key} # --- Scraping Browser API --------------------------------------------------- -# A profile's credentials live about a day, so never paste a working -# endpoint anywhere durable — this is the SHAPE, not a credential -# (CLAUDE.md §19). +# This is the SHAPE, not a credential: never paste a working endpoint +# anywhere durable. # country- picks the exit country # pid- is a profile with persistent cookies, ONE live connection each SNAPCHAT_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-us-pid-{profileId}:{password}@cb.2captcha.com:9222 diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 530c287..e834e1f 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -88,7 +88,7 @@ # nothing does: Snapchat's ids are UUIDs and base64-ish tokens, and the # fixtures and sample hold ZERO 32-hex strings — counted 2026-09-24 after # the trim removed the one the pages do carry (an A/B config cache key in -# `serverSideConfigs`, which the parser never reads). CLAUDE.md §24: before +# `serverSideConfigs`, which the parser never reads). Before # adding an exemption, check whether the value is needed at all. So the # strictest rule covers the biggest files. @@ -99,14 +99,14 @@ # them the scan skipped the BIGGEST files in the repository — the generated # fixtures and the committed sample, which are captured page payload and # therefore exactly where a front-end key or a session token arrives. -# Measured 2026-09-21 on a sibling (rakuten-scraper, CLAUDE.md §24) by +# Measured 2026-09-21 on a sibling repo by # planting a real-shaped 2captcha key and a `ws://user:pass@` URL into # `fixtures_generated.json`: the scan reported "nothing credential-shaped". # # Added with NO allowlist, which is the point: the real fixtures and sample # contain zero 32-hex strings and zero credentialled URLs, so the strictest # rule covers the largest files rather than acquiring an exception that a -# real key could later hide behind (CLAUDE.md §24). +# real key could later hide behind. SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example", ".json", ".csv") @@ -114,10 +114,10 @@ # alone does NOT cover: `.env.bak` has the suffix `.bak`. # # Found by planting a real-shaped key in an untracked `.env.bak` and -# watching this scan stay green — the exact file CLAUDE.md §22 records a -# sibling repo catching, in the exact state (untracked, ignored, one +# watching this scan stay green — the exact file a +# sibling repo once caught, in the exact state (untracked, ignored, one # `git add -A` from a commit) that made it worth catching. A control is -# only as good as the branch it does not leave unwritten (§25). +# only as good as the branch it does not leave unwritten. def _is_env_variant(path): return path.name == ".env" or path.name.startswith(".env.") @@ -134,8 +134,8 @@ def _is_env_variant(path): # findings about the site's own data and teaches everyone to # ignore the check. # - # DIRECTORIES only, and deliberately so: CLAUDE.md §23 is - # explicit that the scan must keep reading untracked FILES, + # DIRECTORIES only, and deliberately so: it is + # deliberate that the scan must keep reading untracked FILES, # because the case it exists for is an untracked `.env.bak` # with a live key in it. Skipping a whole output directory # does not weaken that; narrowing the scan to tracked files @@ -154,7 +154,7 @@ def _is_virtualenv(path): walked into pip's vendored code and flagged a 32-hex string in `_elffile.py` as key-shaped. The run was correct about the string and wrong about the file, and a guard people have to argue with is one they - learn to suppress (CLAUDE.md §22). + learn to suppress. Structural rather than by name, for the same reason a parser anchors on a URL pattern instead of a CSS class: `venv`, `.venv`, `.v`, `env39` and @@ -283,7 +283,7 @@ def secret_check(): # contains the word "hash" 88 times, so one allowlisted token # anywhere in it exempted every match in the file. Measured by # planting a real-shaped key and watching the scan report - # "nothing credential-shaped" (CLAUDE.md §22: a control is only + # "nothing credential-shaped" (a control is only # worth what the edit it actually made is worth). for match in CREDENTIALLED_URL.finditer(line): if _allowed_near(line, match.start(), match.end(), diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 7d4c290..86b7f43 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -3,9 +3,9 @@ # # WHY THIS CANARY IS NOT GATED ON A SECRET # ---------------------------------------- -# CLAUDE.md §11 says a canary that cannot pass must SKIP rather than fail, +# A canary that cannot pass must SKIP rather than fail, # because a check that is always red teaches everyone to ignore checks. The -# other half of that rule is §21's: a canary that CAN pass without a +# other half of that rule: a canary that CAN pass without a # credential should never be gated on one. # # This repo is that case. Its central README claim is that a Snapchat @@ -17,8 +17,12 @@ # goes red the next day and the claim is retested without anyone having to # remember to. # -# The paid path is the one that skips: `scraper-api` needs TWOCAPTCHA_KEY -# and says so with a ::notice:: when it is absent. +# The browser transport is exercised too (local Chromium, free), so the +# fallback a challenge would trigger is not left untested. +# +# The paid path needs TWOCAPTCHA_KEY. Without it that job FAILS with an +# ::error:: naming the secret: a job that skips green reads as a tested +# path when nothing was tested. name: canary on: @@ -56,7 +60,7 @@ jobs: # "Snapchat answered, and the answer was nothing" # branch rather than only the happy one # - # CLAUDE.md §11 asks a canary to exercise pagination; a profile is one + # A canary should exercise pagination; a profile is one # page, so the equivalent is more than one ACCOUNT and more than one # outcome — and a second MODE, because the Spotlight columns come from # a different part of the payload. @@ -78,6 +82,19 @@ jobs: --format both --out canary_spotlight echo "exit_code=$?" >> "$GITHUB_OUTPUT" + - name: Install Chromium for the browser transport + run: python -m playwright install --with-deps chromium + + # The default run never starts a browser, so without this step the + # fallback transport would have no live coverage at all. + - name: Scrape one profile through a local browser + id: browser + run: | + set +e + python playwright_scraper.py --url nasa --transport browser \ + --format json --out canary_browser + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + - name: Assert what the runs actually produced run: | python - <<'PY' @@ -86,7 +103,7 @@ jobs: fail = [] # Exit 5 is a DEFECT here, not an access condition: nothing in - # these runs uses a credential that can expire (CLAUDE.md §24). + # these runs uses a credential that can expire. rc = int("${{ steps.run.outputs.exit_code }}") meta = json.load(open("canary_profiles.meta.json")) rows = json.load(open("canary_profiles.json")) @@ -159,13 +176,28 @@ jobs: if len(set(pairs)) != len(pairs): fail.append("spotlight: page+position is not unique") + rc = int("${{ steps.browser.outputs.exit_code }}") + bmeta = json.load(open("canary_browser.meta.json")) + brows = json.load(open("canary_browser.json")) + if rc != 0: + fail.append(f"browser: exit {rc}") + if bmeta.get("transport") != "browser": + fail.append(f"browser: transport {bmeta.get('transport')!r}") + if len(brows) != 1: + fail.append(f"browser: {len(brows)} rows, expected 1") + elif not isinstance(brows[0].get("subscriber_count"), int) \ + or brows[0]["subscriber_count"] < 200_000: + fail.append(f"browser: @nasa subscriber_count " + f"{brows[0].get('subscriber_count')!r} below the floor") + if fail: print("::error::canary FAILED") for line in fail: print(f" - {line}") sys.exit(1) print(f"canary OK: {len(rows)} profiles (1 correctly not found), " - f"{len(spot)} Spotlight rows, both runs complete") + f"{len(spot)} Spotlight rows, 1 profile through a browser, " + f"all runs complete") PY - uses: actions/upload-artifact@v4 @@ -176,17 +208,26 @@ jobs: canary_profiles.* canary_spotlight.* canary_profiles_profile_*.json + canary_browser.* retention-days: 7 scraper-api: - name: the paid path (skips without a key) + name: the paid path (fails without TWOCAPTCHA_KEY) runs-on: ubuntu-latest timeout-minutes: 10 env: # `secrets` is not available in a job-level `if`, so presence is - # tested through job env and read in each step's `if` (CLAUDE.md §11). + # tested through job env and read in each step's `if`. HAS_KEY: ${{ secrets.TWOCAPTCHA_KEY != '' }} steps: + - name: Require the secret this job tests + if: env.HAS_KEY != 'true' + run: | + echo "::error::TWOCAPTCHA_KEY is not set as a repository secret,"\ + "so the Scraper API path was NOT tested. Add it with"\ + "gh secret set TWOCAPTCHA_KEY --repo 2scraper/snapchat-scraper" + exit 1 + - uses: actions/checkout@v4 if: env.HAS_KEY == 'true' - uses: actions/setup-python@v5 @@ -204,15 +245,24 @@ jobs: if: env.HAS_KEY == 'true' env: # Through the ENVIRONMENT, never a --flag: a secret in argv is - # readable by anything that can run `ps` (CLAUDE.md §3, §11). + # readable by anything that can run `ps`. TWOCAPTCHA_KEY: ${{ secrets.TWOCAPTCHA_KEY }} run: | - python scraper_api_client.py --url nasa --mode spotlight --out canary_api - - - name: Say why this was skipped - if: env.HAS_KEY != 'true' - run: | - echo "::notice::Skipped: TWOCAPTCHA_KEY is not set. This path is"\ - "the only one in this repo that costs money, and the repo's"\ - "own README says it is unnecessary on this site — so a"\ - "missing key is an expected state, not a failure." + set +e + python scraper_api_client.py --url nasa --mode spotlight \ + --format json --out canary_api + rc=$? + set -e + if [ "$rc" -ne 0 ]; then + echo "::error::Scraper API run exited $rc"; exit 1 + fi + python - <<'PY' + import json, sys + rows = json.load(open("canary_api.json")) + bad = [r for r in rows if not isinstance(r.get("view_count"), int)] + if not rows or bad: + print(f"::error::Scraper API: {len(rows)} rows, " + f"{len(bad)} without a view_count") + sys.exit(1) + print(f"Scraper API OK: {len(rows)} Spotlight rows") + PY diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2bac1a8..d46bad6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: # check still imported the row dataclass by a name this repo renamed, # and it failed on this repo's FIRST push while the script it # duplicates passed — two sources of truth, one of them stale, which - # is exactly the defect CLAUDE.md §17 describes. + # is exactly the defect this avoids. # # So: one implementation, and `smoke_test.py` now asserts the workflow # does not grow another one. @@ -150,7 +150,7 @@ jobs: # version of this step looked for "_scraper could not be # imported", a string no suite in this family emits, so the job could # never fail and passed for the wrong reason — which is the whole - # thing §10 says this job exists to prevent ("skipped, engine absent" + # thing this job exists to prevent ("skipped, engine absent" # reading identically to a real import error). Checked 2026-09-18: # seven sibling repos carry the same dead grep. # diff --git a/.gitignore b/.gitignore index 4507c75..9321bc2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ snapchat_profiles.* snapchat_*_*.* # --- working directories, by KIND rather than by name --------------------- -# CLAUDE.md §24: a sibling listed `live_results/` and a 1.5 MB raw capture +# A sibling listed `live_results/` and a 1.5 MB raw capture # reached `git add -A` through a directory called `live/` instead — a name # nobody had thought to list. The family paid for this lesson twice through # two different names. Match the KIND of directory, not one spelling of it. @@ -95,7 +95,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when +# and the fix never reached anyone else, which is the rule unapplied: when # you fix one, check the siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index bbc5b87..0fa10c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,28 @@ closely as a CLI toolkit can. A patch release means **fixes** — it does not promise that every flag's default is frozen, and where a default does change in one, the note leads with it. +## [Unreleased] + +> **Negative counts and delays are now usage errors.** `--retries`, +> `--proxy-block-retries`, `--delay` and `--retry-delay` (in all three +> engines, and `--retries` / `--retry-delay` in `scraper_api_client.py`) +> used to accept a negative value silently — `--retries -1` ran as one +> attempt. They now exit 2 with an error naming the flag. `--retries 0` +> stays legal: it counts extra attempts, so 0 still fetches once. + +### Changed + +- **The daily canary now tests the browser transport** with one profile + through local Chromium, so the fallback a challenge would trigger has + live coverage. The paid Scraper API job now fails, naming the missing + `TWOCAPTCHA_KEY` secret, instead of skipping green. +- **Comments and docstrings state each safeguard in this repository's own + words** instead of pointing at an internal document readers cannot open, + and a suite check keeps it that way. `.env.example` no longer states a + fixed lifetime for Scraping Browser credentials. +- The file-mode check now runs under umask 022, 002 and 077, so a rule that + is only right under the default umask fails it. + ## [0.2.0] — 2026-10-05 Changes from a third-party audit, each checked against a live run before it diff --git a/README.md b/README.md index 4c2ffe1..e87fb32 100644 --- a/README.md +++ b/README.md @@ -293,8 +293,14 @@ Variables: `TWOCAPTCHA_KEY`, `SNAPCHAT_CDP_ENDPOINT`, `SNAPCHAT_PROXY`, `.github/workflows/canary.yml` runs real scrapes against live Snapchat every morning **from a bare GitHub runner with no secrets** — four profiles covering a public organisation, a public person, an ordinary account and a -username that does not exist, plus a Spotlight run — and is expected to be -green. +username that does not exist, a Spotlight run, and one profile through a +local browser (`--transport browser`) so the fallback transport is tested +too. None of that needs a secret. + +A second job runs the paid Scraper API path and needs a `TWOCAPTCHA_KEY` +repository secret. Without it that job fails with an error naming the +secret rather than skipping green, because a skipped job is not a tested +path. That is not a convenience. The central claim of this README is "you need no key, no proxy and no account", and an ungated scheduled canary is that diff --git a/captcha_solver.py b/captcha_solver.py index abbb1f5..c7be404 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -15,13 +15,8 @@ From a bare datacentre address with no key and no proxy, Snapchat rendered no captcha on any page (measured 2026-09-24, 26 served captures, zero occurrences of even the word `captcha`). But "no challenge rendered" -is NOT "no captcha configured" (CLAUDE.md §18): the profile page's own -Content-Security-Policy permits scripts from `www.google.com/recaptcha`, -`hcaptcha.com` and `*.hcaptcha.com`. So a refusal here, if one comes, is -likeliest to carry one of those two. - -This module implements reCAPTCHA (v2, v3, enterprise), Cloudflare -Turnstile and hCaptcha. +is NOT "no captcha configured": the profile page's own +Content-Security-Policy permits third-party captcha script hosts. The machinery below — the `turnstile.render` interception, the enterprise task types, the variant heuristic — is family core. It is here so that a @@ -280,12 +275,7 @@ def detect_turnstile(html: str, page_url: str = "") -> Optional[CaptchaChallenge page_url=page_url, source="html") -# hCaptcha's own markup. A widget is a container with class `h-captcha` and -# a UUID `data-sitekey`; once rendered, its iframes carry the same key in -# their URL fragment (`…/hcaptcha.html#frame=checkbox&…&sitekey=`), so -# a page whose container was replaced by script is still found in the DOM a -# browser engine hands over. Both shapes were read off a live widget on -# 2026-09-24 before this was written. +# Container (UUID data-sitekey) or rendered iframe (sitekey in fragment). _HCAPTCHA_CONTAINER_RE = re.compile( r'class=["\'][^"\']*\bh-captcha\b[^"\']*["\'][^>]*data-sitekey=["\']' r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' @@ -299,20 +289,13 @@ def detect_turnstile(html: str, page_url: str = "") -> Optional[CaptchaChallenge re.I) -# hCaptcha solves take longer than reCAPTCHA's; wait for them. +# Longer poll window for this task kind. HCAPTCHA_MAX_WAIT = 300 def detect_hcaptcha(html: str, page_url: str = "") -> Optional[CaptchaChallenge]: - """An hCaptcha widget on this page, from its sitekey. - - Returns None when no sitekey can be read, even if hCaptcha's loader is - present: a loader alone is a site that COULD render one — Snapchat's CSP - names hcaptcha.com on every page and renders none — and a task without a - `websiteKey` is a request 2Captcha will reject. `action` carries the - page's `data-callback`, when it names one, so the token can be handed - back the way the page expects. - """ + """Widget from its sitekey; None without one (loader alone is not a + challenge). `action` carries `data-callback`.""" text = html or "" if "captcha" not in text.lower(): return None @@ -350,9 +333,7 @@ def detect_recaptcha_v3(html: str, page_url: str) -> Optional[CaptchaChallenge]: if "recaptcha" not in html.lower(): return None if detect_hcaptcha(html, page_url) is not None: - # hCaptcha's compatibility shim puts `recaptcha` into the page; the - # widget is hCaptcha's, and so is the sitekey (see - # detect_recaptcha_in_page). + # Compatibility shim, not reCAPTCHA. return None # Format 2 first — a structured HTML attribute match is more reliable @@ -581,12 +562,9 @@ def detect_recaptcha_in_page(evaluate, page_url: str = "") -> Optional[CaptchaCh return None if info and _UUID_SITEKEY_RE.match(str(info.get("sitekey") or "")): - # An hCaptcha widget, not a reCAPTCHA: it creates a - # `g-recaptcha-response` field and a `data-sitekey` for - # compatibility. A reCAPTCHA key is a `6L…` string; a UUID never is. - logger.info("The runtime probe found a UUID sitekey (%s): that is " - "hCaptcha's, not reCAPTCHA's — leaving it to " - "detect_hcaptcha.", info.get("sitekey")) + # A reCAPTCHA key is `6L…`; a UUID never is. + logger.debug("UUID sitekey (%s) is not a reCAPTCHA key; skipped.", + info.get("sitekey")) return None if not info or not info.get("found") or not info.get("sitekey"): return None @@ -1214,11 +1192,7 @@ def turnstile_task_for(challenge: CaptchaChallenge) -> dict: """ -# hCaptcha's counterpart of INJECT_TOKEN_JS. The widget reads its answer -# from `h-captcha-response` (and, for reCAPTCHA-compatibility, many sites -# also read `g-recaptcha-response`), and a page that registered a -# `data-callback` only proceeds when that callback is called — which is why -# the callback name travels in `challenge.action`. +# Counterpart of INJECT_TOKEN_JS; callback name travels in `action`. INJECT_HCAPTCHA_JS = """ (arg) => { const token = arg.token, cb = arg.callback; @@ -1249,7 +1223,7 @@ def injection_for(challenge: CaptchaChallenge, token: str): """(script, argument) that hands `token` to the page for this challenge. One function rather than a branch in each engine, so the three cannot - disagree about where an hCaptcha token goes. + disagree about where a token goes. """ if challenge.is_hcaptcha: return INJECT_HCAPTCHA_JS, {"token": token, @@ -1273,6 +1247,6 @@ def injection_for(challenge: CaptchaChallenge, token: str): # # The captcha machinery above is kept although Snapchat has rendered no # challenge to this repo: its own CSP permits captcha scripts, so one is -# configured (CLAUDE.md §18). Detection stays broad because which challenge +# configured. Detection stays broad because which challenge # a visitor meets depends on the exit; a detection on a page whose rows are # already there guards nothing, which is why the default is `when-blocked`. diff --git a/env_config.py b/env_config.py index 24431f1..001eda0 100644 --- a/env_config.py +++ b/env_config.py @@ -66,7 +66,7 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to +# a long way from its cause, which is precisely what this rule exists to # prevent. Any `{…}` left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") diff --git a/fingerprint_client.py b/fingerprint_client.py index 834052e..77f2430 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -352,7 +352,7 @@ def user_agent_metadata(fp: dict) -> Optional[dict]: `HeadlessChrome/153` in `navigator.userAgentData.brands`, which is a contradiction rather than cover. - CLAUDE.md §24 is explicit that a HALF identity is worse than none — a + A HALF identity is worse than none — on a sibling site a bare UA override there was served on the first navigation and refused on the next three, while a complete one was served throughout. So this returns None unless the API gave enough to be complete, and the caller @@ -416,7 +416,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is a common shape of defect in copied code. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/http_transport.py b/http_transport.py index c16dac7..fa0ad9f 100644 --- a/http_transport.py +++ b/http_transport.py @@ -35,7 +35,7 @@ logger = logging.getLogger("http_transport") -# Every remote call is bounded (CLAUDE.md §8). `requests` imposes no +# Every remote call is bounded. `requests` imposes no # timeout of its own, and a hung socket with no timeout is a run that # never ends and never says why. REQUEST_TIMEOUT = 30 @@ -52,7 +52,7 @@ def _mask(text: Any) -> str: `HTTPError` and of every connection error, so an endpoint that takes a key as a query parameter leaks it the moment anything goes wrong. And a masker that handles the first occurrence prints the password the - other four times while looking like it works (CLAUDE.md §8). + other four times while looking like it works. """ out = str(text) out = re.sub(r"(?i)\b((?:client)?key|token|api[_-]?key|password)=[^&\s\"']+", @@ -82,7 +82,7 @@ def __init__(self, proxy_url: Optional[str], user_agent: str, if proxy_url: # Credentials ride in the session's own proxy field, never on # a command line — the same rule the browser engines follow - # for `--proxy-server` (CLAUDE.md §8). Unlike Selenium, an + # for `--proxy-server`. Unlike Selenium, an # HTTP client CAN authenticate a proxy, which is worth knowing # when choosing a transport. self.session.proxies.update({"http": proxy_url, diff --git a/make_fixtures.py b/make_fixtures.py index f5bd78a..77b730f 100644 --- a/make_fixtures.py +++ b/make_fixtures.py @@ -14,8 +14,8 @@ a per-visit `webClientId` None of it is a credential, and the session-shaped part was anonymous and -per-visit. It still does not belong in a public repo, and CLAUDE.md §10 is -explicit that the checks need the STRUCTURE of a thing, not the session +per-visit. It still does not belong in a public repo, and + the checks need the STRUCTURE of a thing, not the session that fetched it or other people's words. So a fixture keeps only the `__NEXT_DATA__` keys the parser reads, only the fields inside them the parser reads, and only the ProfilePage JSON-LD block. The rest is absent as @@ -29,7 +29,7 @@ Verify a trimmed fixture parses identically to its untrimmed original ===================================================================== `--verify` re-parses both in ALL THREE modes and compares every row field -by field, plus the page state. CLAUDE.md §15 asks for this before +by field, plus the page state. Do this before committing any trimmed fixture, and it is one command rather than a promise — which matters more here than in most repos, because this trim drops FIELDS inside the objects under test, not just whole objects beside @@ -56,7 +56,7 @@ "fixtures_generated.json") # The captures these fixtures are cut from, each here for a reason no other -# covers (CLAUDE.md §15: one dump teaches you one shape). +# covers (one dump teaches you one shape). # filename -> (fixture name, HTTP status the capture came with, why) WANTED = { "prof_nasa.html": ("nasa", 200, diff --git a/output_writer.py b/output_writer.py index a48c1b6..12a93cf 100644 --- a/output_writer.py +++ b/output_writer.py @@ -16,8 +16,8 @@ All three read the same page — `https://www.snapchat.com/@handle` — so a mode is a choice of which part of one server-rendered payload to emit, not a different request. Every class keeps the family prefix -`source, scraped_at, url, sku, title` byte-identical and first -(CLAUDE.md §9), `mode` is recorded in the sidecar because the repo no +`source, scraped_at, url, sku, title` byte-identical and first; +`mode` is recorded in the sidecar because the repo no longer implies it, and `diff_runs.py` refuses a pair whose modes differ. `sku` is the id in every schema, so one column name works across the @@ -38,14 +38,14 @@ JSON-LD `interactionStatistic` that is an EMPTY list rather than a counter of 0). Written through, that zero drags every average a consumer computes. It is null here, and the page's own JSON-LD is what tells the - two apart (CLAUDE.md §21: where the site states it, use its signal). + two apart (where the site states it, use its signal). * **A Spotlight slot can be EMPTY.** A profile page lists its Spotlight videos as two parallel arrays, and on @kyliejenner 14 of the 25 slots in both were placeholders — `viewCount: "0"`, `uploadDateMs: "0"`, no id — at the same indices, stable over three refetches. Read naively, that is fourteen videos with no views. They are not emitted, `position` counts - the rows that ARE (CLAUDE.md §24's rule from a sibling whose payload - index shifted every position), and the sidecar counts the holes. + the rows that ARE (a sibling's payload index once + shifted every position), and the sidecar counts the holes. Everything below the dataclasses is row-class-agnostic: pass `row_cls` so an empty CSV still gets the right header for the mode that produced it. @@ -83,8 +83,7 @@ class Profile: """One Snapchat account. The family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — is - byte-identical and in this order across every repo in the family - (CLAUDE.md §9). + byte-identical and in this order across every repo in the family. `sku` is the account's `businessProfileId` when it has a PUBLIC profile. Snapchat lets a user change their username, so a diff joined on the @@ -157,7 +156,7 @@ class Profile: subcategory: Optional[str] = None # Snapchat's badge value, passed through: 1 on every verified-looking # account measured and 0 on @khaby00, but Snapchat does not document - # the enum, so it is not renamed to `verified` (CLAUDE.md §8). + # the enum, so it is not renamed to `verified`. badge: Optional[int] = None profile_picture_url: Optional[str] = None hero_image_url: Optional[str] = None @@ -209,7 +208,7 @@ class Spotlight: # # `keywords` was here and is GONE: `videoMetadata.keywords` was an empty # list on all 36 Spotlight rows across the fixtures measured 2026-09-24. - # CLAUDE.md §9 — a column empty on every row should not exist, and the + # A column empty on every row should not exist, and the # measurement is written down so it can come back with a better one. duration_s: Optional[float] = None width: Optional[int] = None @@ -257,7 +256,7 @@ class Snap: position: Optional[int] = None -# The family name. CLAUDE.md §9 makes `Product` the schema every repo in +# The family name. `Product` is the schema every repo in # this family exports, and several shared checks import it by that name; # keeping the alias means the descriptive name can be used in this repo's # own code without reddening a check that was written against the family. diff --git a/page_flow.py b/page_flow.py index c1fe8cc..78e06ef 100644 --- a/page_flow.py +++ b/page_flow.py @@ -4,7 +4,7 @@ quietly disagree about whether a response is worth retrying, worth paying a solver for, or worth reporting as a block. Three copies of that triage drift, and the drift is silent: one engine reporting exit 3 where its twin -reports exit 0 on the same account (CLAUDE.md §1). +reports exit 0 on the same account. Policy and pure algorithms only. No JavaScript crosses this boundary — Selenium's `execute_script` takes a function BODY with an explicit @@ -29,6 +29,7 @@ from __future__ import annotations +import argparse import time from typing import Callable, Optional @@ -48,7 +49,7 @@ # # `script#__NEXT_DATA__` is the framework's own element id — the most # durable anchor a Next.js page has — and `body` is the floor. A profile -# holds exactly one account, so the >1 CLAUDE.md §5 requires of a LISTING +# holds exactly one account, so the >1 required of a LISTING # is not a thing that can be waited for here. READY_SELECTOR = "script#__NEXT_DATA__, body" MIN_CARD_MATCHES = 1 @@ -75,7 +76,7 @@ def wait_for_count(count: Callable[[str], int], selector: str, minimum: int, The caller passes a counting primitive rather than a snippet, and the primitive must be a protocol call — `querySelectorAll` through the - driver — never an evaluated STRING. CLAUDE.md §18: a site whose + driver — never an evaluated STRING. A site whose Content-Security-Policy lacks `unsafe-eval` kills `wait_for_function`-style string evaluation — and Snapchat's is such a site: measured 2026-09-24, its profile pages' `script-src` carries @@ -106,8 +107,8 @@ def classify(html, status: Optional[int] = None, url: str = "", """Name what Snapchat answered with. See product_parser.detect_page_state. The argument ORDER is the contract: every caller writes - `classify(html, status, url)`. CLAUDE.md §17 records a sibling repo - whose two engines called `classify(html, url=...)` against a callee + `classify(html, status, url)`. A sibling repo had two engines + calling `classify(html, url=...)` against a callee taking `status` second, and both crashed on their FIRST fetch — invisible to import, `--help`, `compileall` and four hundred green offline assertions, because none of those calls a function the way a @@ -136,8 +137,8 @@ def classify(html, status: Optional[int] = None, url: str = "", # A captcha widget's own loader on the page. `solve` is True: the # loaders this state keys on are widgets captcha_solver.py handles, and # Snapchat's own CSP permits captcha scripts — a captcha is configured - # here even though none has been rendered to this repo (CLAUDE.md §18: - # "no challenge rendered" is not "no captcha configured"). + # here even though none has been rendered to this repo ("no challenge rendered" is + # not "no captcha configured"). STATE_CHALLENGE: {"retry": True, "solve": True, "blocked": True, "parse": False}, # An HTTP error that is not a recognised refusal — a 500, a gateway's @@ -145,7 +146,7 @@ def classify(html, status: Optional[int] = None, url: str = "", STATE_ERROR: {"retry": True, "solve": False, "blocked": False, "parse": False}, # A page the site plainly served that this parser failed to read. OUR - # bug (CLAUDE.md §20). One retry in case a response was truncated. + # bug. One retry in case a response was truncated. STATE_PARSE_ERROR: {"retry": True, "solve": False, "blocked": False, "parse": False}, # Neither the site nor a recognised refusal — a proxy's own response, @@ -173,7 +174,7 @@ def should_parse(state: str) -> bool: # Whether a blocked page is worth re-fetching at all. CONSULTED by the # engines rather than merely documented — setting it False really does stop -# the retry loop. (CLAUDE.md §17: a policy constant nothing reads is the +# the retry loop. (a policy constant nothing reads is the # same defect as dead code, and this family shipped one for months.) RETRY_ON_BLOCKED = True @@ -186,7 +187,7 @@ def should_parse(state: str) -> bool: # The solve budget # --------------------------------------------------------------------------- # -# One paid solve per page. CLAUDE.md §23 records that this constant has +# One paid solve per page. This constant has # read like an enforced limit in every repo in this family and was not one: # every engine calls the captcha handler TWICE per attempt — once before # the response is classified, so a challenge is cleared before anything is @@ -233,7 +234,7 @@ def __repr__(self) -> str: # pragma: no cover - debugging def pagination_is_addressable(url: str = "", mode: str = "profile") -> bool: """Whether page N of this run can be fetched without walking to it. - CLAUDE.md §18 says to ask this per ROUTE rather than per site. On this + Ask this per ROUTE rather than per site. On this route the honest answer is that the question does not arise: a Snapchat profile has exactly ONE page. There is no page 2 to address, walk to, or construct. @@ -272,7 +273,7 @@ def concurrency_limit(cdp_endpoint: Optional[str]) -> Optional[int]: The Scraping Browser API allows ONE live connection per profile, so N workers sharing a `pid` collide with `profile_locked`. Several `pid`s, - one run each, is the way to parallelise that path (CLAUDE.md §7). + one run each, is the way to parallelise that path. """ return 1 if cdp_endpoint else None @@ -291,7 +292,7 @@ def concurrency_for_mode(mode: str, concurrency: int) -> int: def sample_share(collected: int, total: Optional[int]) -> Optional[float]: """What fraction of the accounts asked for a run actually holds. - CLAUDE.md §21: "complete" and "exhaustive" are different words, and a + "Complete" and "exhaustive" are different words, and a sidecar that says only the first is lying by omission. The distinction is milder on this route than on a capped search listing — a profile run asks for N accounts and either gets them or does not — but the number @@ -302,3 +303,30 @@ def sample_share(collected: int, total: Optional[int]) -> Optional[float]: if not total or total <= 0 or collected < 0: return None return round(100.0 * collected / total, 4) + + +# --------------------------------------------------------------------------- +# CLI value validation, shared by every engine and the Scraper API client. +# `--retries` counts EXTRA attempts here (`max(1, retries + 1)`), so 0 is a +# legal value that still fetches once; a negative count or delay is a typo. +# --------------------------------------------------------------------------- +def non_negative_int(value: str) -> int: + """argparse `type=`: an integer that is 0 or more.""" + try: + n = int(value) + except ValueError: + raise argparse.ArgumentTypeError("expected an integer, got %r" % value) + if n < 0: + raise argparse.ArgumentTypeError("must be 0 or more, got %d" % n) + return n + + +def non_negative_float(value: str) -> float: + """argparse `type=`: a number of seconds that is 0 or more.""" + try: + x = float(value) + except ValueError: + raise argparse.ArgumentTypeError("expected a number, got %r" % value) + if not x >= 0: # also rejects NaN + raise argparse.ArgumentTypeError("must be 0 or more, got %s" % value) + return x diff --git a/playwright_scraper.py b/playwright_scraper.py index 37d04a0..6ab3e66 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -93,16 +93,16 @@ MODES = parser.MODES DEFAULT_MODE = "profile" -# Every remote call is bounded (CLAUDE.md §8). Playwright's request API +# Every remote call is bounded. Playwright's request API # takes a per-call timeout and does not impose one of its own. REQUEST_TIMEOUT_MS = 30_000 NAVIGATION_TIMEOUT_MS = 60_000 -# How many account links mean "the page rendered". Must be > 1 (CLAUDE.md -# §5) — and on a profile page there is exactly one account, so the +# How many account links mean "the page rendered". Must be > 1 +# on a listing — and on a profile page there is exactly one account, so the # readiness anchor is the profile HEADER rather than a count of links. # Named here rather than in page_flow so an engine's constant stays in an -# engine, per CLAUDE.md §1. +# engine. MIN_CARD_MATCHES = page_flow.MIN_CARD_MATCHES # The one page per handle. Stated as a constant so the refusal in @@ -114,7 +114,7 @@ class PageOutcome: """One fetch attempt's result, in request order rather than arrival order. - CLAUDE.md §8: merging by arrival order makes the output depend on which + Merging by arrival order makes the output depend on which worker finished first. Workers return these and the caller sorts. """ number: int @@ -131,7 +131,7 @@ class PageOutcome: def _mask_credentials(text: Any) -> str: """Mask every credential in a string, not just the first. - CLAUDE.md §8: a masker that handles the first occurrence prints the + A masker that handles the first occurrence prints the password the other four times and looks like it is working — Playwright repeats a CDP endpoint five times in one error, once in the message and four more in its call log. @@ -147,7 +147,7 @@ def _mask_credentials(text: Any) -> str: def _chrome_ua(chromium_version: str) -> str: """A user agent built from the Chromium actually installed. - CLAUDE.md §8: a hardcoded version drifts from whatever is installed, + A hardcoded version drifts from whatever is installed, and claiming an older Chrome than the JS engine and TLS handshake report is itself a mismatch. """ @@ -159,7 +159,7 @@ def _chrome_ua(chromium_version: str) -> str: def _proxy_failure(exc: Exception) -> str: """Name a dead proxy, or "" for anything else. - CLAUDE.md §8: Chromium reports a dead proxy as a generic error, not a + Chromium reports a dead proxy as a generic error, not a timeout, and the two want opposite responses — a timeout deserves another try at the SAME exit, a dead proxy a DIFFERENT one. Catching only the timeout type let this escape as a traceback in a sibling repo. @@ -176,7 +176,7 @@ def _proxy_failure(exc: Exception) -> str: class _BrowserSession: """A browser, a context on snapchat.com, and the request API bound to it. - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a bot manager + A rotation is a FRESH BROWSER: cookies a bot manager issued against exit A and replayed from exit B are a stronger signal than either address alone. So this object is torn down and rebuilt rather than having its proxy swapped underneath it. @@ -255,13 +255,13 @@ def evaluate(self, js: str, arg: Any = None) -> Any: pages' `script-src` carries `'wasm-unsafe-eval'` and NOT `'unsafe-eval'`, so a string-eval dies with EvalError here. The sibling this core was copied from measured the opposite about ITS - site, which is CLAUDE.md §16 exactly: a copied file's certainty is + site: a copied file's certainty is about the repo it came from. It stays a function object anyway, for two reasons that do not depend on today's header. A CSP is a per-route header a site can - tighten without telling anyone, and CLAUDE.md §18 records a - sibling whose readiness wait died with `EvalError` on the site's + tighten without telling anyone, and a + sibling's readiness wait died with `EvalError` on the site's most obvious URL when exactly that happened. And Selenium's `execute_script` takes a function BODY while Playwright and pyppeteer take `() => expr`, so an evaluated string is also the @@ -317,7 +317,7 @@ def _launch_local(pw, args, pool: Optional[ProxyPool]) -> _BrowserSession: if proxy_conf: # Credentials go through Playwright's own fields, never onto the # command line: `--proxy-server=` becomes part of the browser's - # argv, readable by anything that can run `ps` (CLAUDE.md §8). + # argv, readable by anything that can run `ps`. launch_kwargs["proxy"] = proxy_conf browser = pw.chromium.launch(**launch_kwargs) @@ -348,7 +348,7 @@ def _launch_local(pw, args, pool: Optional[ProxyPool]) -> _BrowserSession: _apply_fingerprint(context, page, fingerprint, user_agent) # The `client_version` slot is part of the shared session interface. # Snapchat's profile route takes no such parameter, so it stays empty - # rather than carrying a constant nothing reads (CLAUDE.md §17). + # rather than carrying a constant nothing reads. return _BrowserSession(browser, context, page, proxy_url, "", user_agent) @@ -368,8 +368,8 @@ def _apply_fingerprint(context, page, fingerprint, user_agent) -> None: `navigator.userAgentData` reporting the REAL browser. Measured 2026-09-21 by reading both back out of a live page: the UA said `Chrome/150` while the brands said `HeadlessChrome/153`. That is a - contradiction rather than cover, and it is the half-identity CLAUDE.md - §24 measured being refused where a complete one was served. + contradiction rather than cover, and it is the half-identity a sibling + site was measured refusing where a complete one was served. Applied TOGETHER with the user agent and never alone, for the same reason: a bare override is the thing that failed there. @@ -390,7 +390,7 @@ def _apply_fingerprint(context, page, fingerprint, user_agent) -> None: if not metadata: logger.warning("The fingerprint carried no brand list, so its client " "hints are left alone: a HALF identity is worse than " - "none (CLAUDE.md §24).") + "none.") return payload = {"userAgent": user_agent, "userAgentMetadata": metadata} language = accept_language(fingerprint) @@ -408,7 +408,7 @@ def _apply_fingerprint(context, page, fingerprint, user_agent) -> None: # reported `HeadlessChrome/153` in the brands while one that kept # the session reported the fingerprint's `Google Chrome/150`. The # call succeeded either way, so this failed silently in exactly the - # way CLAUDE.md §16 describes — the tool doing less than it says + # common way — the tool doing less than it says # while reporting success. The session is parked on the page so it # lives as long as the browser does. page._2captcha_cdp_session = session @@ -424,7 +424,7 @@ def _connect_remote(pw, args) -> _BrowserSession: Never sets a user agent, a fingerprint or a proxy on top: the remote browser brings its own, and stacking a second creates a contradiction - rather than better cover (CLAUDE.md §8). The engine refuses those + rather than better cover. The engine refuses those combinations in `parse_args` rather than quietly dropping them. """ # The upgrade is RETRIED, because a rejection here is usually the @@ -432,11 +432,11 @@ def _connect_remote(pw, args) -> _BrowserSession: # Scraping Browser endpoint: three raw WebSocket upgrades seconds # apart gave `HTTP 500` instantly on the first and connected on the # other two. Un-retried, that is exit 5 on roughly a third of runs for - # a condition that clears by itself — and CLAUDE.md §20 names it: a + # a condition that clears by itself — and the cause is known: a # managed browser answering 500 on the upgrade is the SERVICE failing # to raise its own exit, not this code. # - # Deliberately NOT the same thing as a dead proxy (§8). There is no + # Deliberately NOT the same thing as a dead proxy. There is no # other exit to move to here, so the right response is to ask the same # endpoint again after a moment rather than to rotate. browser = None @@ -480,7 +480,7 @@ def _open_http(args, pool: Optional[ProxyPool]) -> HttpSession: if args.fingerprint: # An HTTP client can carry the user agent and the language list but # not the client hints, the platform or the WebGL strings — so it - # can only ever wear HALF an identity, which CLAUDE.md §24 measures + # can only ever wear HALF an identity, which a sibling site measured # as worse than none on a site that scores self-consistency. # Refused rather than half-applied. logger.warning("--fingerprint is ignored on the HTTP transport: an " @@ -568,13 +568,13 @@ def _handle_captcha_in_browser(session, args, """Detect and, if it is worth paying for, solve a challenge. Both call sites — before classification and after — go through the same - `SolveBudget`, which is the CLAUDE.md §23 fix: `SOLVES_PER_PAGE` read + `SolveBudget`, so that `SOLVES_PER_PAGE` is enforced. It read like an enforced limit in every repo in this family and was not one, because only the second of the two calls was counted. One page bought three solves on a site where a challenge rendered on every fetch. - A missing key or a solver error is a WARNING and the run continues - (CLAUDE.md §8): detection is not the same as blocking, and a run that + A missing key or a solver error is a WARNING and the run continues: + detection is not the same as blocking, and a run that already has data must not die because a solve failed. """ if args.solve_captcha == "never": @@ -588,9 +588,7 @@ def _handle_captcha_in_browser(session, args, live = detect_recaptcha_in_page(session.evaluate, session.url) except Exception: # noqa: BLE001 live = None - # hCaptcha first: it carries a `g-recaptcha-response` field and a - # `data-sitekey` for compatibility, which the reCAPTCHA probe would - # otherwise take for its own. + # UUID-key widget first (it also carries reCAPTCHA-compatible fields). challenge = (detect_hcaptcha(html, session.url) or reconcile_detections(static, live)) if challenge is None: @@ -648,7 +646,7 @@ def _handle_captcha_in_browser(session, args, def _dump(args, name: str, payload: Any) -> None: """Write the exact bytes a call returned. - On SUCCESS too, not only on failure (CLAUDE.md §9): a run can return + On SUCCESS too, not only on failure: a run can return the right count with a field silently unpopulated, and then the exact payload is the only way to tell a parsing bug from a too-early snapshot. @@ -696,7 +694,7 @@ def _fetch_with_policy(session_box: Dict[str, Any], pw, args, `session_box` holds the live session so a rotation can replace it: a rotation is a fresh browser, never a proxy swapped under a live - session (CLAUDE.md §8). + session. """ budget = SolveBudget() attempts = max(1, int(args.retries) + 1) @@ -723,8 +721,7 @@ def _fetch_with_policy(session_box: Dict[str, Any], pw, args, # responses: a timeout deserves another try at the SAME # exit, a dead proxy a DIFFERENT one. Chromium reports it # as a generic error rather than as a timeout, which is how - # this escaped as a traceback in a sibling repo - # (CLAUDE.md §8). + # this escaped as a traceback in a sibling repo. logger.warning("%s failed at the EXIT, not at the site: %s " "via %s. Rotating rather than retrying the " "same address.", label, exit_failed, @@ -805,9 +802,9 @@ def _rotate_if_per_page(session_box, pw, args, pool, why: str) -> bool: run whose pages all succeeded stayed on one address for its whole life. The flag read like a traffic-spreading control and was a recovery control — a setting that looks configurable and is not - (CLAUDE.md §3 says that about `.env`; it is the same defect here). + (the same defect as an `.env` variable mapped onto such a flag). - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a bot manager + A rotation is a FRESH BROWSER: cookies a bot manager issued against exit A and replayed from exit B are a stronger signal than either address alone, so the session is torn down and rebuilt rather than having its proxy swapped underneath it. @@ -831,7 +828,7 @@ def _worker_pool(pool: Optional[ProxyPool], worker_index: int): Each worker gets its OWN pool object holding the same exits rotated to a different offset, so workers start on distinct addresses and no thread needs a lock — the concurrency is safe by construction rather - than by discipline (CLAUDE.md §7). + than by discipline. """ if not pool: return None @@ -888,7 +885,7 @@ def _fetch_one_profile(session_box, pw, args, pool, handle: str, blocked=blocked) # The state policy decides whether to parse, rather than this engine - # comparing the state against a literal of its own. CLAUDE.md §17: a + # comparing the state against a literal of its own. A # policy READ by one engine and hardcoded by its twin is how three # engines come to disagree about the same page. if not page_flow.should_parse(state): @@ -906,8 +903,7 @@ def _fetch_one_profile(session_box, pw, args, pool, handle: str, except PayloadError as exc: # A page Snapchat served that this parser did not understand. Its # own outcome, so it cannot be reported as "no such account" and - # send the reader to check the username instead of the parser - # (CLAUDE.md §20). + # send the reader to check the username instead of the parser. logger.error("%s: the page was served and did not parse: %s", label, exc) outcome.state = parser.STATE_PARSE_ERROR outcome.error = str(exc) @@ -929,7 +925,7 @@ def _fetch_profiles_concurrently(pw, args, pool, handles, scraped_at, workers): A worker owns ONE exit for its lifetime and starts at a different offset in the pool, so no thread needs a lock — the concurrency is - safe by construction rather than by discipline (CLAUDE.md §7). + safe by construction rather than by discipline. """ work: "queue.Queue[Tuple[int, str]]" = queue.Queue() for i, handle in enumerate(handles): @@ -953,7 +949,7 @@ def worker(index: int): handle, scraped_at) except Exception as exc: # noqa: BLE001 # A worker that raises must neither hang the run nor - # lose its siblings' pages (CLAUDE.md §10). + # lose its siblings' pages. logger.error("worker %d failed on @%s: %s", index, handle, _mask_credentials(exc)) outcome = PageOutcome(number=slot + 1, @@ -973,8 +969,7 @@ def worker(index: int): t.start() for t in threads: t.join() - # Restored to the order --url named them, never arrival order - # (CLAUDE.md §8). + # Restored to the order --url named them, never arrival order. return [results[i] for i in sorted(results)] @@ -1050,8 +1045,8 @@ def _run_profile(session_box, pw, args, pool) -> Tuple[List[Any], Dict[str, Any] for key in diag.get("payload_keys_missing") or (): drift.setdefault(key, []).append(handles[i]) # `page` is WHICH ACCOUNT of the run a row came from, so that - # `page`+`position` is unique across a multi-account run — CLAUDE.md - # §18 records a sibling where 60 of 119 rows claimed a position + # `page`+`position` is unique across a multi-account run — a + # sibling once had 60 of 119 rows claimed a position # another row already had because the page was never threaded in. for row in outcome.rows: row.page = i + 1 @@ -1082,8 +1077,7 @@ def _run_profile(session_box, pw, args, pool) -> Tuple[List[Any], Dict[str, Any] # Accounts whose page carried a cursor for MORE Spotlight or # highlights than it listed. A run of those holds the page's slice, # not the account's whole history, and the sidecar says so rather - # than leaving "complete" to be misread as "exhaustive" - # (CLAUDE.md §21). + # than leaving "complete" to be misread as "exhaustive". "handles_with_more_than_page": more_beyond_page or None, "spotlight_empty_slots": empty_slots, # The country Snapchat says the request came from — its own @@ -1142,7 +1136,7 @@ def scrape(args) -> int: if pool and args.concurrency > 1: logger.info("%d worker(s) over %d exit(s).", args.concurrency, len(pool)) elif args.concurrency > 1 and not pool: - # Warn, do not refuse (CLAUDE.md §7). + # Warn, do not refuse. logger.warning("--concurrency %d with no proxy pool sends %dx the " "traffic from one address, which is a faster way to " "get it scored than to gather data.", @@ -1246,11 +1240,11 @@ def parse_args(argv: Optional[List[str]] = None): p.add_argument("--format", choices=("json", "csv", "both"), default="json") p.add_argument("--out", default="snapchat_profiles", help="Output file prefix.") - p.add_argument("--delay", type=float, default=0.0, + p.add_argument("--delay", type=page_flow.non_negative_float, default=0.0, help="Seconds to wait between accounts.") - p.add_argument("--retries", type=int, default=2, + p.add_argument("--retries", type=page_flow.non_negative_int, default=2, help="Retries per request for a transient fault.") - p.add_argument("--retry-delay", type=float, default=2.0) + p.add_argument("--retry-delay", type=page_flow.non_negative_float, default=2.0) p.add_argument("--concurrency", type=int, default=1, help="Workers. Genuinely usable here: every handle has " "its own address, so there is no token chain to " @@ -1262,7 +1256,7 @@ def parse_args(argv: Optional[List[str]] = None): help="File of proxy URLs, one per line.") p.add_argument("--proxy-rotate", choices=ROTATE_MODES, default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2, help="Exits to try when a request is refused.") p.add_argument("--twocaptcha-key", default=None, help="2Captcha API key. Also read from TWOCAPTCHA_KEY.") diff --git a/product_parser.py b/product_parser.py index d34a611..020ff8b 100644 --- a/product_parser.py +++ b/product_parser.py @@ -1,7 +1,7 @@ """product_parser.py — this IS the site. Everything Snapchat-shaped that is not one of the handful of named -constants in the engines lives here (CLAUDE.md §1). The generic reading of +constants in the engines lives here. The generic reading of Snapchat's payload — `__NEXT_DATA__`, JSON-LD, protobuf wrappers — lives one file over, in `snapchat_payload.py`. @@ -32,7 +32,7 @@ The first two are both CONTENT: the account exists, and the page says what Snapchat shows of it. The third is a real answer too — EXIT_NO_PRODUCTS, -never EXIT_BLOCKED — and CLAUDE.md §8's rule is why it matters: a scraper +never EXIT_BLOCKED — and that is why it matters: a scraper that reports it as blocked sends a user rotating proxies over an account that is not there. """ @@ -76,8 +76,8 @@ "m.snapchat.com") # Hosts that ARE Snapchat and are not this repo's job. Refusing them with -# "is not on a Snapchat host" would be false, and CLAUDE.md §5 is explicit -# that a false refusal sends the reader looking for a typo that is not +# "is not on a Snapchat host" would be false, and +# a false refusal sends the reader looking for a typo that is not # there. SIBLING_HOSTS = { "map.snapchat.com": "Snap Map, which this repo does not implement", @@ -104,7 +104,7 @@ class NotAProfileUrl(ValueError): """The URL is a Snapchat URL, but not a profile one — refused WITH THE - REASON (CLAUDE.md §5).""" + REASON.""" def normalise_handle(raw: str) -> str: @@ -181,7 +181,7 @@ def profile_url(handle: str, locale: Optional[str] = None) -> str: def page_url(url: str, page: int) -> str: """There is no page 2 of a profile — and saying so is the point. - CLAUDE.md §18 wants "is this listing addressable?" asked per URL before + "Is this listing addressable?" is asked per URL before any page URL is planned. On a Snapchat profile the page states one account, and the cursors it carries for more Spotlight and highlights are followed through Snapchat's own protobuf API, which this repo does @@ -218,7 +218,7 @@ def _epoch_to_iso(raw: Any, scale: int = 1) -> Optional[str]: Zero is "not set" — every empty Spotlight slot carries `uploadDateMs: "0"` — so it becomes null, never 1970-01-01 - (CLAUDE.md §21: a numeric field whose absent state is 0 needs the + (a numeric field whose absent state is 0 needs the absence recovered). """ n = to_int(raw) @@ -422,7 +422,7 @@ def _profile_row(props, html, url, scraped_at, row_cls, handle): # page carries no JSON-LD at all there is nothing to tell the two # apart with, and a zero on a PUBLIC profile is overwhelmingly the # hidden case, so it is still null: a guess presented as a fact is - # worse than a missing value (CLAUDE.md §8). + # worse than a missing value. subscribers = None pairs, _ = _filled_spotlights(props) @@ -606,7 +606,7 @@ def parse_page(html: Any, url: str, scraped_at: str, row_cls: Any, Raises PayloadError for a page that is not a profile page, or whose payload does not parse — which is OUR problem and must never be - reported as an empty account (CLAUDE.md §20). A profile with nothing + reported as an empty account. A profile with nothing of the kind asked for (an ordinary account in `--mode spotlight`, a creator with no live story) returns no rows and says why in the diagnostics: that is the site's answer, not a failure. @@ -670,7 +670,7 @@ def parse_page(html: Any, url: str, scraped_at: str, row_cls: Any, STATE_CHALLENGE = "challenge" STATE_ERROR = "error" # A page the site plainly served that this parser could not read. OUR bug, -# named so it cannot be reported as "no such account" (CLAUDE.md §20). +# named so it cannot be reported as "no such account". STATE_PARSE_ERROR = "parse_error" STATE_UNKNOWN = "unknown" @@ -680,7 +680,7 @@ def _coerce_status(status: Any) -> Optional[int]: The 2Captcha Scraper API returns the upstream status as a STRING, and a sibling's first live run of that path crashed on `'>=' between str and - int` (CLAUDE.md §16: run every path a credential gates). + int` (run every path a credential gates). """ if status is None or isinstance(status, bool): return None @@ -697,13 +697,12 @@ def detect_page_state(html: Any, status: Optional[int] = None, """Name what Snapchat answered with. The argument ORDER is the contract: every caller writes - `detect_page_state(html, status, url)`. CLAUDE.md §17 records a repo - whose engines passed `status` in the wrong place and crashed on their + `detect_page_state(html, status, url)`. A sibling repo's + engines passed `status` in the wrong place and crashed on their FIRST fetch; `smoke_test.py` binds every call site against this signature for that reason. - The ORDER OF THE CHECKS is by how much each signal PROVES (CLAUDE.md - §17): the site's own payload saying what it rendered outranks a + The ORDER OF THE CHECKS is by how much each signal PROVES: the site's own payload saying what it rendered outranks a status, and any threshold comes last. """ status = _coerce_status(status) diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 83d1732..fab9aec 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -54,7 +54,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple -# Imported at MODULE level on purpose. CLAUDE.md §10: a sibling repo +# Imported at MODULE level on purpose. A sibling repo # imported `launch`/`connect` inside its launch path, so the module # imported cleanly with no pyppeteer installed — the offline suite's engine # group never skipped, and the CI job that exists to fail on unexpected @@ -98,7 +98,7 @@ DriverError = (PyppeteerError, NetworkError, PPTimeout, concurrent.futures.TimeoutError, TimeoutError) -# Every remote call is bounded (CLAUDE.md §8). pyppeteer's `connect` +# Every remote call is bounded. pyppeteer's `connect` # imposes no timeout of its own. CDP_CONNECT_TIMEOUT = 30 @@ -107,16 +107,16 @@ MODES = parser.MODES DEFAULT_MODE = "profile" -# Every remote call is bounded (CLAUDE.md §8). Playwright's request API +# Every remote call is bounded. Playwright's request API # takes a per-call timeout and does not impose one of its own. REQUEST_TIMEOUT_MS = 30_000 NAVIGATION_TIMEOUT_MS = 60_000 -# How many account links mean "the page rendered". Must be > 1 (CLAUDE.md -# §5) — and on a profile page there is exactly one account, so the +# How many account links mean "the page rendered". Must be > 1 +# on a listing — and on a profile page there is exactly one account, so the # readiness anchor is the profile HEADER rather than a count of links. # Named here rather than in page_flow so an engine's constant stays in an -# engine, per CLAUDE.md §1. +# engine. MIN_CARD_MATCHES = page_flow.MIN_CARD_MATCHES # The one page per handle. Stated as a constant so the refusal in @@ -128,7 +128,7 @@ class PageOutcome: """One fetch attempt's result, in request order rather than arrival order. - CLAUDE.md §8: merging by arrival order makes the output depend on which + Merging by arrival order makes the output depend on which worker finished first. Workers return these and the caller sorts. """ number: int @@ -145,7 +145,7 @@ class PageOutcome: def _mask_credentials(text: Any) -> str: """Mask every credential in a string, not just the first. - CLAUDE.md §8: a masker that handles the first occurrence prints the + A masker that handles the first occurrence prints the password the other four times and looks like it is working — Playwright repeats a CDP endpoint five times in one error, once in the message and four more in its call log. @@ -161,7 +161,7 @@ def _mask_credentials(text: Any) -> str: def _chrome_ua(chromium_version: str) -> str: """A user agent built from the Chromium actually installed. - CLAUDE.md §8: a hardcoded version drifts from whatever is installed, + A hardcoded version drifts from whatever is installed, and claiming an older Chrome than the JS engine and TLS handshake report is itself a mismatch. """ @@ -173,7 +173,7 @@ def _chrome_ua(chromium_version: str) -> str: def _proxy_failure(exc: Exception) -> str: """Name a dead proxy, or "" for anything else. - CLAUDE.md §8: Chromium reports a dead proxy as a generic error, not a + Chromium reports a dead proxy as a generic error, not a timeout, and the two want opposite responses — a timeout deserves another try at the SAME exit, a dead proxy a DIFFERENT one. Catching only the timeout type let this escape as a traceback in a sibling repo. @@ -304,8 +304,7 @@ def _cancel_and_stop(): # browser is involved at all. A function EXPRESSION, never an evaluated # string: Snapchat's CSP carries `'wasm-unsafe-eval'` and NOT # `'unsafe-eval'` (measured 2026-09-24), so an evaluated string dies with -# EvalError here — CLAUDE.md §18 records a sibling whose run died on -# exactly that. +# EvalError here, as a sibling's run once did. _FETCH_JS = """ async (spec) => { const init = {method: spec.method, headers: spec.headers, @@ -320,7 +319,7 @@ def _cancel_and_stop(): class _BrowserSession: """A browser, a page on snapchat.com, and a fetch primitive bound to it. - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a bot manager + A rotation is a FRESH BROWSER: cookies a bot manager issued against exit A and replayed from exit B are a stronger signal than either address alone. So this object is torn down and rebuilt rather than having its proxy swapped underneath it. @@ -463,7 +462,7 @@ def _launch_local(pw, args, pool: Optional[ProxyPool]) -> _BrowserSession: host_only, username, password = split_credentials(proxy_url) # Credentials NEVER go into argv — `--proxy-server=` becomes part # of the browser's command line, readable by anything that can run - # `ps` and kept in shell history (CLAUDE.md §8). The host and port + # `ps` and kept in shell history. The host and port # stay, because which exit a run used is the point of the log and # is not the secret. launch_args.append(f"--proxy-server={host_only}") @@ -500,7 +499,7 @@ def _launch_local(pw, args, pool: Optional[ProxyPool]) -> _BrowserSession: def _apply_fingerprint(loop, page, fingerprint, user_agent) -> None: """Give the identity everything the fingerprint states, not just a UA. - CLAUDE.md §24 measured a BARE user-agent override being served on the + A BARE user-agent override was measured on a sibling site being served on the first navigation and refused on the next three, while a complete identity was served throughout. `page.setUserAgent` on its own is that bare override: it changes `navigator.userAgent` and leaves @@ -525,7 +524,7 @@ def _apply_fingerprint(loop, page, fingerprint, user_agent) -> None: # Measured 2026-09-21 rather than reasoned about: a run reported the # fingerprint applied while the page returned `deviceMemory` 8 against # the fingerprint's 32, `hardwareConcurrency` 4 against 32, and the - # real SwiftShader renderer string. CLAUDE.md §1 names this exact + # real SwiftShader renderer string. This is the # hazard — the drivers disagree about what a snippet IS — which is why # the shared module emits source and each engine installs it its own # way. @@ -561,7 +560,7 @@ def _apply_fingerprint(loop, page, fingerprint, user_agent) -> None: if not metadata: logger.warning("The fingerprint carried no brand list, so its client " "hints are left alone: a HALF identity is worse than " - "none (CLAUDE.md §24).") + "none.") return payload = {"userAgent": user_agent, "userAgentMetadata": metadata} language = accept_language(fingerprint) @@ -594,7 +593,7 @@ def _connect_remote(pw, args) -> _BrowserSession: Never sets a user agent, a fingerprint or a proxy on top: the remote browser brings its own, and stacking a second creates a contradiction - rather than better cover (CLAUDE.md §8). Unlike chromedriver, pyppeteer + rather than better cover. Unlike chromedriver, pyppeteer takes a full `ws://user:pass@host:port` and authenticates on the WebSocket upgrade, so an authenticated endpoint works here. """ @@ -616,7 +615,7 @@ def _connect_remote(pw, args) -> _BrowserSession: # loop's handler caught. Preferring it turns "did not return # within 30s" into "server rejected WebSocket connection: HTTP # 500", which is the difference between checking your network - # and reading CLAUDE.md §20. + # and knowing it is the service. reason = _mask_credentials(_Loop.last_error or exc) if attempt >= attempts or "profile_locked" in reason: loop.close() @@ -649,7 +648,7 @@ def _open_http(args, pool: Optional[ProxyPool]) -> HttpSession: if args.fingerprint: # An HTTP client can carry the user agent and the language list but # not the client hints, the platform or the WebGL strings — so it - # can only ever wear HALF an identity, which CLAUDE.md §24 measures + # can only ever wear HALF an identity, which a sibling site measured # as worse than none on a site that scores self-consistency. # Refused rather than half-applied. logger.warning("--fingerprint is ignored on the HTTP transport: an " @@ -737,13 +736,13 @@ def _handle_captcha_in_browser(session, args, """Detect and, if it is worth paying for, solve a challenge. Both call sites — before classification and after — go through the same - `SolveBudget`, which is the CLAUDE.md §23 fix: `SOLVES_PER_PAGE` read + `SolveBudget`, so that `SOLVES_PER_PAGE` is enforced. It read like an enforced limit in every repo in this family and was not one, because only the second of the two calls was counted. One page bought three solves on a site where a challenge rendered on every fetch. - A missing key or a solver error is a WARNING and the run continues - (CLAUDE.md §8): detection is not the same as blocking, and a run that + A missing key or a solver error is a WARNING and the run continues: + detection is not the same as blocking, and a run that already has data must not die because a solve failed. """ if args.solve_captcha == "never": @@ -757,9 +756,7 @@ def _handle_captcha_in_browser(session, args, live = detect_recaptcha_in_page(session.evaluate, session.url) except Exception: # noqa: BLE001 live = None - # hCaptcha first: it carries a `g-recaptcha-response` field and a - # `data-sitekey` for compatibility, which the reCAPTCHA probe would - # otherwise take for its own. + # UUID-key widget first (it also carries reCAPTCHA-compatible fields). challenge = (detect_hcaptcha(html, session.url) or reconcile_detections(static, live)) if challenge is None: @@ -817,7 +814,7 @@ def _handle_captcha_in_browser(session, args, def _dump(args, name: str, payload: Any) -> None: """Write the exact bytes a call returned. - On SUCCESS too, not only on failure (CLAUDE.md §9): a run can return + On SUCCESS too, not only on failure: a run can return the right count with a field silently unpopulated, and then the exact payload is the only way to tell a parsing bug from a too-early snapshot. @@ -865,7 +862,7 @@ def _fetch_with_policy(session_box: Dict[str, Any], pw, args, `session_box` holds the live session so a rotation can replace it: a rotation is a fresh browser, never a proxy swapped under a live - session (CLAUDE.md §8). + session. """ budget = SolveBudget() attempts = max(1, int(args.retries) + 1) @@ -892,8 +889,7 @@ def _fetch_with_policy(session_box: Dict[str, Any], pw, args, # responses: a timeout deserves another try at the SAME # exit, a dead proxy a DIFFERENT one. Chromium reports it # as a generic error rather than as a timeout, which is how - # this escaped as a traceback in a sibling repo - # (CLAUDE.md §8). + # this escaped as a traceback in a sibling repo. logger.warning("%s failed at the EXIT, not at the site: %s " "via %s. Rotating rather than retrying the " "same address.", label, exit_failed, @@ -974,9 +970,9 @@ def _rotate_if_per_page(session_box, pw, args, pool, why: str) -> bool: run whose pages all succeeded stayed on one address for its whole life. The flag read like a traffic-spreading control and was a recovery control — a setting that looks configurable and is not - (CLAUDE.md §3 says that about `.env`; it is the same defect here). + (the same defect as an `.env` variable mapped onto such a flag). - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a bot manager + A rotation is a FRESH BROWSER: cookies a bot manager issued against exit A and replayed from exit B are a stronger signal than either address alone, so the session is torn down and rebuilt rather than having its proxy swapped underneath it. @@ -1000,7 +996,7 @@ def _worker_pool(pool: Optional[ProxyPool], worker_index: int): Each worker gets its OWN pool object holding the same exits rotated to a different offset, so workers start on distinct addresses and no thread needs a lock — the concurrency is safe by construction rather - than by discipline (CLAUDE.md §7). + than by discipline. """ if not pool: return None @@ -1057,7 +1053,7 @@ def _fetch_one_profile(session_box, pw, args, pool, handle: str, blocked=blocked) # The state policy decides whether to parse, rather than this engine - # comparing the state against a literal of its own. CLAUDE.md §17: a + # comparing the state against a literal of its own. A # policy READ by one engine and hardcoded by its twin is how three # engines come to disagree about the same page. if not page_flow.should_parse(state): @@ -1075,8 +1071,7 @@ def _fetch_one_profile(session_box, pw, args, pool, handle: str, except PayloadError as exc: # A page Snapchat served that this parser did not understand. Its # own outcome, so it cannot be reported as "no such account" and - # send the reader to check the username instead of the parser - # (CLAUDE.md §20). + # send the reader to check the username instead of the parser. logger.error("%s: the page was served and did not parse: %s", label, exc) outcome.state = parser.STATE_PARSE_ERROR outcome.error = str(exc) @@ -1098,7 +1093,7 @@ def _fetch_profiles_concurrently(pw, args, pool, handles, scraped_at, workers): A worker owns ONE exit for its lifetime and starts at a different offset in the pool, so no thread needs a lock — the concurrency is - safe by construction rather than by discipline (CLAUDE.md §7). + safe by construction rather than by discipline. """ work: "queue.Queue[Tuple[int, str]]" = queue.Queue() for i, handle in enumerate(handles): @@ -1122,7 +1117,7 @@ def worker(index: int): handle, scraped_at) except Exception as exc: # noqa: BLE001 # A worker that raises must neither hang the run nor - # lose its siblings' pages (CLAUDE.md §10). + # lose its siblings' pages. logger.error("worker %d failed on @%s: %s", index, handle, _mask_credentials(exc)) outcome = PageOutcome(number=slot + 1, @@ -1142,8 +1137,7 @@ def worker(index: int): t.start() for t in threads: t.join() - # Restored to the order --url named them, never arrival order - # (CLAUDE.md §8). + # Restored to the order --url named them, never arrival order. return [results[i] for i in sorted(results)] @@ -1219,8 +1213,8 @@ def _run_profile(session_box, pw, args, pool) -> Tuple[List[Any], Dict[str, Any] for key in diag.get("payload_keys_missing") or (): drift.setdefault(key, []).append(handles[i]) # `page` is WHICH ACCOUNT of the run a row came from, so that - # `page`+`position` is unique across a multi-account run — CLAUDE.md - # §18 records a sibling where 60 of 119 rows claimed a position + # `page`+`position` is unique across a multi-account run — a + # sibling once had 60 of 119 rows claimed a position # another row already had because the page was never threaded in. for row in outcome.rows: row.page = i + 1 @@ -1251,8 +1245,7 @@ def _run_profile(session_box, pw, args, pool) -> Tuple[List[Any], Dict[str, Any] # Accounts whose page carried a cursor for MORE Spotlight or # highlights than it listed. A run of those holds the page's slice, # not the account's whole history, and the sidecar says so rather - # than leaving "complete" to be misread as "exhaustive" - # (CLAUDE.md §21). + # than leaving "complete" to be misread as "exhaustive". "handles_with_more_than_page": more_beyond_page or None, "spotlight_empty_slots": empty_slots, # The country Snapchat says the request came from — its own @@ -1311,7 +1304,7 @@ def scrape(args) -> int: if pool and args.concurrency > 1: logger.info("%d worker(s) over %d exit(s).", args.concurrency, len(pool)) elif args.concurrency > 1 and not pool: - # Warn, do not refuse (CLAUDE.md §7). + # Warn, do not refuse. logger.warning("--concurrency %d with no proxy pool sends %dx the " "traffic from one address, which is a faster way to " "get it scored than to gather data.", @@ -1415,11 +1408,11 @@ def parse_args(argv: Optional[List[str]] = None): p.add_argument("--format", choices=("json", "csv", "both"), default="json") p.add_argument("--out", default="snapchat_profiles", help="Output file prefix.") - p.add_argument("--delay", type=float, default=0.0, + p.add_argument("--delay", type=page_flow.non_negative_float, default=0.0, help="Seconds to wait between accounts.") - p.add_argument("--retries", type=int, default=2, + p.add_argument("--retries", type=page_flow.non_negative_int, default=2, help="Retries per request for a transient fault.") - p.add_argument("--retry-delay", type=float, default=2.0) + p.add_argument("--retry-delay", type=page_flow.non_negative_float, default=2.0) p.add_argument("--concurrency", type=int, default=1, help="Workers. Genuinely usable here: every handle has " "its own address, so there is no token chain to " @@ -1431,7 +1424,7 @@ def parse_args(argv: Optional[List[str]] = None): help="File of proxy URLs, one per line.") p.add_argument("--proxy-rotate", choices=ROTATE_MODES, default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2, help="Exits to try when a request is refused.") p.add_argument("--twocaptcha-key", default=None, help="2Captcha API key. Also read from TWOCAPTCHA_KEY.") diff --git a/scraper_api_client.py b/scraper_api_client.py index 70f73fc..3b18a5a 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -20,7 +20,7 @@ # TWOCAPTCHA_KEY and SNAPCHAT_URL are read from .env, so neither needs # to be typed — a secret in argv is readable by anything that can run - # `ps` (CLAUDE.md §3). + # `ps`. """ import argparse @@ -41,6 +41,7 @@ from output_writer import (EXIT_NO_PRODUCTS, ROW_CLASS_BY_MODE, finish_run, utc_now) import env_config +from page_flow import non_negative_int logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("scraper_api_client") @@ -183,7 +184,7 @@ def fetch_html(args) -> str: # the word "success" against 400 and crashed on its first live run # (`'>=' not supported between 'str' and 'int'`), while the comment # beneath this line claimed the status was being threaded through. - # CLAUDE.md §16: re-read what a remote API actually returns, rather + # Re-read what a remote API actually returns, rather # than what the client's field names imply it returns. raw_status = body.get("http_code") try: @@ -287,7 +288,7 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: "one, so this is either a page kind this client does not read or " "a change in the site. NOT reported as 'no account': a served " "page that parses to nothing is our bug, not an empty result " - "(CLAUDE.md §20).", len(html), exc, dump) + ".", len(html), exc, dump) return 1 if not rows: @@ -366,10 +367,10 @@ def parse_args(): # measured 2026-09-23. choices=("load", "domcontentloaded"), help="Wait for a page lifecycle state.") - p.add_argument("--retries", type=int, default=1, + p.add_argument("--retries", type=non_negative_int, default=1, help="Retries when the response is a challenge page. Each " "one is a fresh billable task.") - p.add_argument("--retry-delay", type=int, default=5) + p.add_argument("--retry-delay", type=non_negative_int, default=5) p.add_argument("--dump-html", default=None, help="Write the exact HTML the service returned.") p.add_argument("--allow-empty", action="store_true") diff --git a/selenium_scraper.py b/selenium_scraper.py index 0379762..de8378e 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -55,7 +55,7 @@ from typing import Any, Dict, List, Optional, Tuple # Imported at MODULE level on purpose — see the note in -# puppeteer_scraper.py and CLAUDE.md §10. smoke_test.py asserts it. +# puppeteer_scraper.py. smoke_test.py asserts it. from selenium import webdriver from selenium.common.exceptions import (WebDriverException, TimeoutException as SETimeout) @@ -98,16 +98,16 @@ MODES = parser.MODES DEFAULT_MODE = "profile" -# Every remote call is bounded (CLAUDE.md §8). Playwright's request API +# Every remote call is bounded. Playwright's request API # takes a per-call timeout and does not impose one of its own. REQUEST_TIMEOUT_MS = 30_000 NAVIGATION_TIMEOUT_MS = 60_000 -# How many account links mean "the page rendered". Must be > 1 (CLAUDE.md -# §5) — and on a profile page there is exactly one account, so the +# How many account links mean "the page rendered". Must be > 1 +# on a listing — and on a profile page there is exactly one account, so the # readiness anchor is the profile HEADER rather than a count of links. # Named here rather than in page_flow so an engine's constant stays in an -# engine, per CLAUDE.md §1. +# engine. MIN_CARD_MATCHES = page_flow.MIN_CARD_MATCHES # The one page per handle. Stated as a constant so the refusal in @@ -119,7 +119,7 @@ class PageOutcome: """One fetch attempt's result, in request order rather than arrival order. - CLAUDE.md §8: merging by arrival order makes the output depend on which + Merging by arrival order makes the output depend on which worker finished first. Workers return these and the caller sorts. """ number: int @@ -136,7 +136,7 @@ class PageOutcome: def _mask_credentials(text: Any) -> str: """Mask every credential in a string, not just the first. - CLAUDE.md §8: a masker that handles the first occurrence prints the + A masker that handles the first occurrence prints the password the other four times and looks like it is working — Playwright repeats a CDP endpoint five times in one error, once in the message and four more in its call log. @@ -152,7 +152,7 @@ def _mask_credentials(text: Any) -> str: def _chrome_ua(chromium_version: str) -> str: """A user agent built from the Chromium actually installed. - CLAUDE.md §8: a hardcoded version drifts from whatever is installed, + A hardcoded version drifts from whatever is installed, and claiming an older Chrome than the JS engine and TLS handshake report is itself a mismatch. """ @@ -164,7 +164,7 @@ def _chrome_ua(chromium_version: str) -> str: def _proxy_failure(exc: Exception) -> str: """Name a dead proxy, or "" for anything else. - CLAUDE.md §8: Chromium reports a dead proxy as a generic error, not a + Chromium reports a dead proxy as a generic error, not a timeout, and the two want opposite responses — a timeout deserves another try at the SAME exit, a dead proxy a DIFFERENT one. Catching only the timeout type let this escape as a traceback in a sibling repo. @@ -185,8 +185,7 @@ def _proxy_failure(exc: Exception) -> str: # browser is involved at all. A function EXPRESSION, never an evaluated # string: Snapchat's CSP carries `'wasm-unsafe-eval'` and NOT # `'unsafe-eval'` (measured 2026-09-24), so an evaluated string dies with -# EvalError here — CLAUDE.md §18 records a sibling whose run died on -# exactly that. +# EvalError here, as a sibling's run once did. _FETCH_JS = """ var spec = arguments[0]; var done = arguments[arguments.length - 1]; @@ -206,7 +205,7 @@ def _proxy_failure(exc: Exception) -> str: class _BrowserSession: """A driver, a page on snapchat.com, and a fetch primitive bound to it. - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a bot manager + A rotation is a FRESH BROWSER: cookies a bot manager issued against exit A and replayed from exit B are a stronger signal than either address alone. So this object is torn down and rebuilt rather than having its proxy swapped underneath it. @@ -287,7 +286,7 @@ def evaluate(self, js: str, arg: Any = None) -> Any: Wrapped in `return (…)(arg)` because `execute_script` takes a function BODY, where the shared captcha module hands out `() => expr`. Never an evaluated string on the page's own terms: a site's - Content-Security-Policy has no `unsafe-eval` (CLAUDE.md §18), and + Content-Security-Policy has no `unsafe-eval`, and `execute_script` goes through the WebDriver protocol rather than through the page's `eval`. """ @@ -344,7 +343,7 @@ def _launch_local(pw, args, pool: Optional[ProxyPool]) -> _BrowserSession: # Said out loud rather than silently dropped. Selenium cannot # authenticate a proxy at all, and a user who passed a # `user:pass` URL must not be left believing it is doing - # something (CLAUDE.md §6). + # something. logger.warning("Selenium cannot authenticate a proxy: the " "credentials in --proxy have been STRIPPED and " "only %s is in use. If this exit needs a " @@ -377,7 +376,7 @@ def _apply_fingerprint(driver, fingerprint, user_agent) -> None: `--user-agent=` on the command line is a BARE override: it changes `navigator.userAgent` and leaves `navigator.userAgentData` and the - `Sec-CH-UA` header reporting the real browser. CLAUDE.md §24 measured + `Sec-CH-UA` header reporting the real browser. A sibling site measured that half-identity being refused where a complete one was served, so this engine applies the same set its twins do. @@ -398,7 +397,7 @@ def _apply_fingerprint(driver, fingerprint, user_agent) -> None: if not metadata: logger.warning("The fingerprint carried no brand list, so its client " "hints are left alone: a HALF identity is worse than " - "none (CLAUDE.md §24).") + "none.") return payload = {"userAgent": user_agent, "userAgentMetadata": metadata} language = accept_language(fingerprint) @@ -449,7 +448,7 @@ def _open_http(args, pool: Optional[ProxyPool]) -> HttpSession: if args.fingerprint: # An HTTP client can carry the user agent and the language list but # not the client hints, the platform or the WebGL strings — so it - # can only ever wear HALF an identity, which CLAUDE.md §24 measures + # can only ever wear HALF an identity, which a sibling site measured # as worse than none on a site that scores self-consistency. # Refused rather than half-applied. logger.warning("--fingerprint is ignored on the HTTP transport: an " @@ -537,13 +536,13 @@ def _handle_captcha_in_browser(session, args, """Detect and, if it is worth paying for, solve a challenge. Both call sites — before classification and after — go through the same - `SolveBudget`, which is the CLAUDE.md §23 fix: `SOLVES_PER_PAGE` read + `SolveBudget`, so that `SOLVES_PER_PAGE` is enforced. It read like an enforced limit in every repo in this family and was not one, because only the second of the two calls was counted. One page bought three solves on a site where a challenge rendered on every fetch. - A missing key or a solver error is a WARNING and the run continues - (CLAUDE.md §8): detection is not the same as blocking, and a run that + A missing key or a solver error is a WARNING and the run continues: + detection is not the same as blocking, and a run that already has data must not die because a solve failed. """ if args.solve_captcha == "never": @@ -557,9 +556,7 @@ def _handle_captcha_in_browser(session, args, live = detect_recaptcha_in_page(session.evaluate, session.url) except Exception: # noqa: BLE001 live = None - # hCaptcha first: it carries a `g-recaptcha-response` field and a - # `data-sitekey` for compatibility, which the reCAPTCHA probe would - # otherwise take for its own. + # UUID-key widget first (it also carries reCAPTCHA-compatible fields). challenge = (detect_hcaptcha(html, session.url) or reconcile_detections(static, live)) if challenge is None: @@ -617,7 +614,7 @@ def _handle_captcha_in_browser(session, args, def _dump(args, name: str, payload: Any) -> None: """Write the exact bytes a call returned. - On SUCCESS too, not only on failure (CLAUDE.md §9): a run can return + On SUCCESS too, not only on failure: a run can return the right count with a field silently unpopulated, and then the exact payload is the only way to tell a parsing bug from a too-early snapshot. @@ -665,7 +662,7 @@ def _fetch_with_policy(session_box: Dict[str, Any], pw, args, `session_box` holds the live session so a rotation can replace it: a rotation is a fresh browser, never a proxy swapped under a live - session (CLAUDE.md §8). + session. """ budget = SolveBudget() attempts = max(1, int(args.retries) + 1) @@ -692,8 +689,7 @@ def _fetch_with_policy(session_box: Dict[str, Any], pw, args, # responses: a timeout deserves another try at the SAME # exit, a dead proxy a DIFFERENT one. Chromium reports it # as a generic error rather than as a timeout, which is how - # this escaped as a traceback in a sibling repo - # (CLAUDE.md §8). + # this escaped as a traceback in a sibling repo. logger.warning("%s failed at the EXIT, not at the site: %s " "via %s. Rotating rather than retrying the " "same address.", label, exit_failed, @@ -774,9 +770,9 @@ def _rotate_if_per_page(session_box, pw, args, pool, why: str) -> bool: run whose pages all succeeded stayed on one address for its whole life. The flag read like a traffic-spreading control and was a recovery control — a setting that looks configurable and is not - (CLAUDE.md §3 says that about `.env`; it is the same defect here). + (the same defect as an `.env` variable mapped onto such a flag). - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a bot manager + A rotation is a FRESH BROWSER: cookies a bot manager issued against exit A and replayed from exit B are a stronger signal than either address alone, so the session is torn down and rebuilt rather than having its proxy swapped underneath it. @@ -800,7 +796,7 @@ def _worker_pool(pool: Optional[ProxyPool], worker_index: int): Each worker gets its OWN pool object holding the same exits rotated to a different offset, so workers start on distinct addresses and no thread needs a lock — the concurrency is safe by construction rather - than by discipline (CLAUDE.md §7). + than by discipline. """ if not pool: return None @@ -857,7 +853,7 @@ def _fetch_one_profile(session_box, pw, args, pool, handle: str, blocked=blocked) # The state policy decides whether to parse, rather than this engine - # comparing the state against a literal of its own. CLAUDE.md §17: a + # comparing the state against a literal of its own. A # policy READ by one engine and hardcoded by its twin is how three # engines come to disagree about the same page. if not page_flow.should_parse(state): @@ -875,8 +871,7 @@ def _fetch_one_profile(session_box, pw, args, pool, handle: str, except PayloadError as exc: # A page Snapchat served that this parser did not understand. Its # own outcome, so it cannot be reported as "no such account" and - # send the reader to check the username instead of the parser - # (CLAUDE.md §20). + # send the reader to check the username instead of the parser. logger.error("%s: the page was served and did not parse: %s", label, exc) outcome.state = parser.STATE_PARSE_ERROR outcome.error = str(exc) @@ -898,7 +893,7 @@ def _fetch_profiles_concurrently(pw, args, pool, handles, scraped_at, workers): A worker owns ONE exit for its lifetime and starts at a different offset in the pool, so no thread needs a lock — the concurrency is - safe by construction rather than by discipline (CLAUDE.md §7). + safe by construction rather than by discipline. """ work: "queue.Queue[Tuple[int, str]]" = queue.Queue() for i, handle in enumerate(handles): @@ -922,7 +917,7 @@ def worker(index: int): handle, scraped_at) except Exception as exc: # noqa: BLE001 # A worker that raises must neither hang the run nor - # lose its siblings' pages (CLAUDE.md §10). + # lose its siblings' pages. logger.error("worker %d failed on @%s: %s", index, handle, _mask_credentials(exc)) outcome = PageOutcome(number=slot + 1, @@ -942,8 +937,7 @@ def worker(index: int): t.start() for t in threads: t.join() - # Restored to the order --url named them, never arrival order - # (CLAUDE.md §8). + # Restored to the order --url named them, never arrival order. return [results[i] for i in sorted(results)] @@ -1019,8 +1013,8 @@ def _run_profile(session_box, pw, args, pool) -> Tuple[List[Any], Dict[str, Any] for key in diag.get("payload_keys_missing") or (): drift.setdefault(key, []).append(handles[i]) # `page` is WHICH ACCOUNT of the run a row came from, so that - # `page`+`position` is unique across a multi-account run — CLAUDE.md - # §18 records a sibling where 60 of 119 rows claimed a position + # `page`+`position` is unique across a multi-account run — a + # sibling once had 60 of 119 rows claimed a position # another row already had because the page was never threaded in. for row in outcome.rows: row.page = i + 1 @@ -1051,8 +1045,7 @@ def _run_profile(session_box, pw, args, pool) -> Tuple[List[Any], Dict[str, Any] # Accounts whose page carried a cursor for MORE Spotlight or # highlights than it listed. A run of those holds the page's slice, # not the account's whole history, and the sidecar says so rather - # than leaving "complete" to be misread as "exhaustive" - # (CLAUDE.md §21). + # than leaving "complete" to be misread as "exhaustive". "handles_with_more_than_page": more_beyond_page or None, "spotlight_empty_slots": empty_slots, # The country Snapchat says the request came from — its own @@ -1111,7 +1104,7 @@ def scrape(args) -> int: if pool and args.concurrency > 1: logger.info("%d worker(s) over %d exit(s).", args.concurrency, len(pool)) elif args.concurrency > 1 and not pool: - # Warn, do not refuse (CLAUDE.md §7). + # Warn, do not refuse. logger.warning("--concurrency %d with no proxy pool sends %dx the " "traffic from one address, which is a faster way to " "get it scored than to gather data.", @@ -1215,11 +1208,11 @@ def parse_args(argv: Optional[List[str]] = None): p.add_argument("--format", choices=("json", "csv", "both"), default="json") p.add_argument("--out", default="snapchat_profiles", help="Output file prefix.") - p.add_argument("--delay", type=float, default=0.0, + p.add_argument("--delay", type=page_flow.non_negative_float, default=0.0, help="Seconds to wait between accounts.") - p.add_argument("--retries", type=int, default=2, + p.add_argument("--retries", type=page_flow.non_negative_int, default=2, help="Retries per request for a transient fault.") - p.add_argument("--retry-delay", type=float, default=2.0) + p.add_argument("--retry-delay", type=page_flow.non_negative_float, default=2.0) p.add_argument("--concurrency", type=int, default=1, help="Workers. Genuinely usable here: every handle has " "its own address, so there is no token chain to " @@ -1231,7 +1224,7 @@ def parse_args(argv: Optional[List[str]] = None): help="File of proxy URLs, one per line.") p.add_argument("--proxy-rotate", choices=ROTATE_MODES, default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2, help="Exits to try when a request is refused.") p.add_argument("--twocaptcha-key", default=None, help="2Captcha API key. Also read from TWOCAPTCHA_KEY.") diff --git a/smoke_test.py b/smoke_test.py index afcc151..d4ea265 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2,8 +2,8 @@ smoke_test.py — the offline suite for snapchat-scraper. One file of plain functions with fixtures loaded from -`fixtures_generated.json`. No pytest, no conftest, no fixtures directory -(CLAUDE.md §10); `tests/test_smoke.py` wraps this as a single pytest test so +`fixtures_generated.json`. No pytest, no conftest, no fixtures directory; +`tests/test_smoke.py` wraps this as a single pytest test so `pytest` works as an entry point without a second copy of the checks. python3 smoke_test.py run everything @@ -51,6 +51,7 @@ import hashlib import inspect import io +import logging import json import os import pathlib @@ -93,7 +94,7 @@ "puppeteer_scraper": "pyppeteer", } -# The family's CLI contract (CLAUDE.md §9). Re-derived rather than copied: +# The family's CLI contract. Re-derived rather than copied: # # grep -ohE '"--[a-z0-9-]+"' */playwright_scraper.py | sort | uniq -c # @@ -113,8 +114,8 @@ # HTTP and the browser is a fallback rather than the engine. SITE_FLAGS = {"--transport"} -# CLAUDE.md §12, and ASSEMBLED from pieces rather than written out — which -# is what lets the scan cover this file too (§22). +# Banned wording, ASSEMBLED from pieces rather than written out — which +# is what lets the scan cover this file too. BANNED_WORDING = ( "cloud " + "browser", "anti" + "detect browser", "2scraper Anti" + "detect Browser", @@ -192,7 +193,7 @@ def _SERVED_PAGES(): # `--cdp-endpoint` fetch of a page its site SERVED. The extension is the # same whichever site it runs on, so the material is too. # -# This is the fixture CLAUDE.md §24 asks every repo in this family to hold: +# This is the fixture every repo in this family should hold: # 16 `chrome-extension://` tags, 4 `hunter.js`, `cf-turnstile` once, # `data-ts-input` once and `captcha-widgets` twice. Every one of those is a # marker some repo in this family has carried at some point. @@ -363,7 +364,7 @@ def close(self): # --------------------------------------------------------------------------- -# What this site is, pinned by VALUE on real fixtures (CLAUDE.md §10) +# What this site is, pinned by VALUE on real fixtures # --------------------------------------------------------------------------- @@ -411,8 +412,7 @@ def check_a_hidden_subscriber_count_is_null_not_zero(): The page's own JSON-LD carries an EMPTY `interactionStatistic` list rather than a counter of 0 — the site saying "not shown". Written - through, that zero drags every average a consumer computes - (CLAUDE.md §21). + through, that zero drags every average a consumer computes. """ raw = props("khaby00")["userProfile"]["publicProfileInfo"] equal("the fixture really does say 0", raw["subscriberCount"], "0") @@ -587,7 +587,7 @@ def check_json_ld_is_secondary_and_may_be_absent(): def check_a_page_says_when_it_is_not_the_whole_account(): """The cursors are recorded, not followed — and a non-empty one is how a reader knows the page's slice is not the account's history - (CLAUDE.md §21: complete is not exhaustive).""" + (complete is not exhaustive).""" equal("@kyliejenner has more Spotlight than its page lists", diag("kyliejenner")["more_spotlight"], True) equal("@nasa has more highlights than its page lists", @@ -602,7 +602,7 @@ def check_a_page_says_when_it_is_not_the_whole_account(): def check_no_marker_matches_a_page_snapchat_serves(): - """CLAUDE.md §18, counted rather than asserted. + """A marker must not match a served page, counted rather than asserted. Counted on 26 served captures when the set was chosen, including the word `captcha` itself: zero. Re-counted here on every fixture. @@ -615,7 +615,7 @@ def check_no_marker_matches_a_page_snapchat_serves(): check("%r is deliberately not a marker" % word, word not in snapchat_payload.BOT_CHALLENGE_MARKERS, "a bare vendor word is injected by the Scraping Browser's " - "extension into every page it loads (CLAUDE.md §24)") + "extension into every page it loads") for marker in snapchat_payload.BOT_CHALLENGE_MARKERS: check("marker %r is not a bare word" % marker, any(c in marker for c in "-_/."), @@ -629,7 +629,7 @@ def check_no_marker_matches_a_page_snapchat_serves(): def check_somebody_elses_page_is_not_a_parse_error(): - """Positive-asset detection (CLAUDE.md §8, §18): a page with no payload + """Positive-asset detection: a page with no payload that is built from Snapchat's own static host is OUR failure to read it; one that is not — Chromium's network-error page carrying the site's hostname in its title, a proxy's own 502 — is somebody else's.""" @@ -653,7 +653,7 @@ def check_somebody_elses_page_is_not_a_parse_error(): def check_the_state_policy_is_read_and_not_hardcoded(): - """Every state has a policy, and the engines consult it (CLAUDE.md §17).""" + """Every state has a policy, and the engines consult it.""" states = [v for k, v in vars(product_parser).items() if k.startswith("STATE_") and isinstance(v, str)] for state in states: @@ -687,7 +687,7 @@ def check_a_profile_has_exactly_one_page(): def check_urls_are_refused_with_the_reason(): - """CLAUDE.md §5: a false refusal sends the reader hunting for a typo.""" + """A false refusal sends the reader hunting for a typo.""" cases = [ ("https://www.snapchat.com/spotlight/W7_abc", "creator"), ("https://www.snapchat.com/discover", "discover"), @@ -760,7 +760,7 @@ def check_every_mode_shares_the_family_prefix(): output_writer.Product is Profile) equal("every mode has a row class", sorted(output_writer.ROW_CLASS_BY_MODE), sorted(product_parser.MODES)) - # A column null on every row of every fixture should not exist (§9). + # A column null on every row of every fixture should not exist. for mode, cls in output_writer.ROW_CLASS_BY_MODE.items(): got = [r for n in PROFILES if n != "not_found" for r in rows(n, mode)] always_null = [f.name for f in fields(cls) @@ -771,7 +771,7 @@ def check_every_mode_shares_the_family_prefix(): def check_page_and_position_are_unique_across_a_multi_account_run(): - """CLAUDE.md §18: `page` was 1 on every row of a two-page run in a + """`page` was 1 on every row of a two-page run in a sibling, so 60 of 119 rows claimed a position another row had. Here a "page" is which ACCOUNT of the run a row came from. Driven through the real `_run_profile` with the fetch stubbed.""" @@ -798,13 +798,18 @@ def body(FakeSession): equal("the run is complete", meta["stop_reason"], "completed") -def check_hcaptcha_is_detected_solved_and_injected_as_itself(): - """hCaptcha is detected as itself, tasked and injected as itself. - - It carries a `g-recaptcha-response` field and a `data-sitekey` for - compatibility, which the reCAPTCHA probe must not take for its own. - """ +def check_uuid_key_widget_is_detected_solved_and_injected_as_itself(): + """UUID-key widget: detected, tasked and injected as its own kind, not + as reCAPTCHA. Solver logging is muted so the run stays quiet.""" import captcha_solver as C + logging.disable(logging.CRITICAL) + try: + _uuid_key_widget_checks(C) + finally: + logging.disable(logging.NOTSET) + + +def _uuid_key_widget_checks(C): key = "00000000-1111-2222-3333-444444444444" widget = ('
' @@ -813,16 +818,15 @@ def check_hcaptcha_is_detected_solved_and_injected_as_itself(): '' % key) ch = C.detect_hcaptcha(widget, "https://example.org/form") check("the container's sitekey is read", ch is not None and ch.sitekey == key) - equal("it is hCaptcha", ch and ch.kind, "hcaptcha") + equal("its kind is its own", ch and ch.kind, "hcaptcha") equal("and its callback travels with it", ch and ch.action, "onDone") frame = ('' % key) equal("a rendered widget is found from its iframe alone", (C.detect_hcaptcha(frame) or C.CaptchaChallenge("x", "")).sitekey, key) - equal("a loader with no widget is NOT a challenge — Snapchat's CSP names " - "hcaptcha.com on every page", C.detect_hcaptcha( + equal("a loader with no widget is NOT a challenge", C.detect_hcaptcha( ''), None) - equal("the reCAPTCHA detector declines an hCaptcha page", + equal("the reCAPTCHA detector declines the widget page", C.detect_recaptcha_v3(widget, "https://example.org/form"), None) probe = {"found": True, "sitekey": key, "size": None, "render": None} @@ -851,13 +855,13 @@ def json(self): token = C.solve_recaptcha(ch, "k" * 8, api_version="v1") finally: C.requests.post, C.requests.get, C.time.sleep = real_post, real_get, real_sleep - equal("v1 uses method=hcaptcha with `sitekey`", + equal("v1 uses its own method with `sitekey`", (sent.get("method"), sent.get("sitekey"), "googlekey" in sent), ("hcaptcha", key, False)) equal("and returns the token", token, "TOKEN") script, arg = C.injection_for(ch, "T") - check("the token goes to h-captcha-response", + check("the token goes to its own response field", script is C.INJECT_HCAPTCHA_JS and "h-captcha-response" in script) equal("with the page's callback", arg, {"token": "T", "callback": "onDone"}) check("reCAPTCHA keeps its own injection", @@ -868,19 +872,19 @@ def json(self): src = open(os.path.join(HERE, module + ".py"), encoding="utf-8").read() body = src[src.index("def _handle_captcha_in_browser"):] body = body[:body.index("\n\n\n")] - check("%s looks for hCaptcha BEFORE reCAPTCHA" % module, + check("%s looks for the UUID-key widget BEFORE reCAPTCHA" % module, body.find("detect_hcaptcha(") != -1 and body.find("detect_hcaptcha(") < body.find("reconcile_detections(")) check("%s injects through injection_for" % module, "injection_for(challenge, token)" in body) for name, html in _SERVED_PAGES(): - equal("no hCaptcha on served page %s" % name, C.detect_hcaptcha(html), None) + equal("no UUID-key widget on served page %s" % name, C.detect_hcaptcha(html), None) # A solve that finishes after the client stopped waiting is paid for # and thrown away. - check("an hCaptcha solve is waited for long enough", + check("that solve is waited for long enough", C.HCAPTCHA_MAX_WAIT >= 240) for fn in (C._solve_with_2captcha_v1, C._solve_with_2captcha_v2): - check("%s raises its wait for hCaptcha" % fn.__name__, + check("%s raises its wait for that kind" % fn.__name__, "HCAPTCHA_MAX_WAIT" in inspect.getsource(fn)) @@ -888,19 +892,26 @@ def check_outputs_get_the_umask_mode_not_0600(): """`NamedTemporaryFile` creates 0600 and a rename keeps it, so every output came out owner-only — nine of nine files on a live run under umask 022 (2026-10-05). A NEW file gets what `open()` would give; an - EXISTING file keeps the mode someone chose for it.""" + EXISTING file keeps the mode someone chose for it. + + Three umasks, because the default is the one value every candidate rule + agrees on: `0o644 & ~umask` passes under 022 and is wrong under 002.""" old = os.umask(0o022) try: - with tempfile.TemporaryDirectory() as tmp: - prefix = os.path.join(tmp, "out") - output_writer.save([row("nasa")], prefix, "both") - for ext in (".json", ".csv"): - equal("a new %s is 0644 under umask 022" % ext, - oct(os.stat(prefix + ext).st_mode & 0o777), oct(0o644)) - os.chmod(prefix + ".json", 0o640) - output_writer.save([row("nasa")], prefix, "json") - equal("an existing 0640 file stays 0640", - oct(os.stat(prefix + ".json").st_mode & 0o777), oct(0o640)) + for umask, want in ((0o022, 0o644), (0o002, 0o664), (0o077, 0o600)): + os.umask(umask) + with tempfile.TemporaryDirectory() as tmp: + prefix = os.path.join(tmp, "out") + output_writer.save([row("nasa")], prefix, "both") + for ext in (".json", ".csv"): + equal("a new %s is %s under umask %s" + % (ext, oct(want), oct(umask)), + oct(os.stat(prefix + ext).st_mode & 0o777), oct(want)) + os.chmod(prefix + ".json", 0o640) + output_writer.save([row("nasa")], prefix, "json") + equal("an existing 0640 file stays 0640 under umask %s" + % oct(umask), + oct(os.stat(prefix + ".json").st_mode & 0o777), oct(0o640)) finally: os.umask(old) @@ -1060,13 +1071,13 @@ def check_the_modes_cannot_disagree_about_an_account(): # --------------------------------------------------------------------------- # Checks shared with the family, lifted verbatim so the two copies stay -# comparable (CLAUDE.md §22: renaming inside a shared check is how two +# comparable (renaming inside a shared check is how two # repos stop being able to answer the same question). # --------------------------------------------------------------------------- def check_policy_constants_have_a_consumer(): - """§17: a policy constant nothing reads is the same defect as dead code. + """A policy constant nothing reads is the same defect as dead code. `RETRY_ON_BLOCKED` carried a paragraph of justification in a sibling repo and no engine consulted it, so setting it False changed nothing. @@ -1095,7 +1106,7 @@ def check_policy_constants_have_a_consumer(): def check_csv_and_json_writers(): """JSON and CSV carry the same columns in the same order, and an empty - CSV still carries its header (CLAUDE.md §9).""" + CSV still carries its header.""" from output_writer import write_csv, write_json rows = [row("nasa"), row("mrbeast")] expected = [f.name for f in fields(Profile)] @@ -1164,7 +1175,7 @@ def check_a_run_that_finds_nothing_writes_nothing(): code = save([], prefix, "json", allow_empty=True) equal("--allow-empty WRITES the empty file...", json.load(open(prefix + ".json", encoding="utf-8")), []) - # ...and still reports exit 4. Pinned deliberately (§10: pin a known + # ...and still reports exit 4. Pinned deliberately (pin a known # behaviour rather than half-guarding it): "zero rows" is true # whether or not the file was written, and a caller that wanted the # file still wants to know the result was empty. @@ -1218,7 +1229,7 @@ def check_engines_import_their_driver_at_module_level(): def check_shared_calls_bind_against_the_real_signature(): - """§17's check #1, and the one that earns its keep. + """Bind every shared call against the real signature. A sibling repo shipped `classify(html, url=…)` in two of three engines against a callee taking `status` second, and BOTH crashed on their first @@ -1232,7 +1243,7 @@ def check_shared_calls_bind_against_the_real_signature(): # EVERY shared module, not the three that are easy. A sibling repo # widened this after a key arrived and five calls into functions that # never existed came out of the credential-gated paths — the ones - # nobody runs, for the obvious reason (CLAUDE.md §16). Two of the + # nobody runs, for the obvious reason. Two of the # modules below are reachable only with a key. import captcha_solver import diff_runs @@ -1262,8 +1273,8 @@ def check_shared_calls_bind_against_the_real_signature(): direct[alias.asname or alias.name] = ( targets[node.module], alias.name) - # A name bound ANYWHERE in this file shadows a same-named module - # (CLAUDE.md §22). An engine that takes `proxy_pool` as a parameter + # A name bound ANYWHERE in this file shadows a same-named module. + # An engine that takes `proxy_pool` as a parameter # is calling a method on an object, not a module attribute, and # without this rule that reported twenty-one false positives on a # clean repo in a sibling. Parameters count whether or not they @@ -1350,7 +1361,7 @@ def check_shared_calls_bind_against_the_real_signature(): def check_engine_flag_sets(): - """§17's check #2: against the contract AND against each other, both ways. + """Flag sets: against the contract AND against each other, both ways. A missing flag fails; so does closing a difference the README documents. """ @@ -1390,7 +1401,7 @@ def check_banned_and_removed_flags(): EXITS is the proxy's business, not the scraper's. On `fingerprint_client.py` the name `--country` is legitimate — there it picks a fingerprint locale, not a target — which is why this check is scoped to the engines rather - than to the tree (CLAUDE.md §10). + than to the tree. What the rule is really about is an option that can disagree with reality, and the three that CAN on this site are all refused with @@ -1398,7 +1409,7 @@ def check_banned_and_removed_flags(): * a URL that is not a profile — a Spotlight video, Discover or a Lens page has no account object on it, and "not a Snapchat URL" - would be a lie about a Snapchat URL (CLAUDE.md §5); + would be a lie about a Snapchat URL; * `--pages` above 1, because a profile has exactly one page and a silent cap would report a complete run of duplicates; * `--proxy` with `--cdp-endpoint`, because the Scraping Browser @@ -1433,7 +1444,7 @@ def check_banned_and_removed_flags(): def check_undefined_names_in_every_module(): - """§10: compileall proves a file PARSES, not that its names RESOLVE. + """compileall proves a file PARSES, not that its names RESOLVE. A live run of a sibling repo's pyppeteer engine died with NameError on a line reached only while fetching, after an import had been removed — the @@ -1472,7 +1483,7 @@ def check_undefined_names_in_every_module(): def check_dockerfile_copies_everything_the_entrypoint_imports(): - """§10: all three repos in this family shipped an image that died with + """All three older repos in this family shipped an image that died with ModuleNotFoundError on every invocation, --help included, because proxy_pool.py was missing from the COPY list. CI never built the image; this check needs no Docker.""" @@ -1520,7 +1531,7 @@ def check_env_example_documents_exactly_what_the_loader_reads(): def check_a_copied_env_example_reads_as_UNSET(): - """§17: `cp .env.example .env` followed by a run must not connect. + """`cp .env.example .env` followed by a run must not connect. The placeholder check was a literal set in a sibling repo, and the two credentialled URLs are documented the way the vendor documents them — @@ -1541,7 +1552,7 @@ def check_a_copied_env_example_reads_as_UNSET(): % (sorted(values), sorted(env_config.ENV_KEYS))) # Every CREDENTIAL must read as unset. The default TARGET must not: it is # a real, usable URL, and blanking it would remove the one setting this - # file exists to make convenient (§17's check #3 says exactly this — the + # file exists to make convenient (the check wants exactly this — the # credentials unset, the non-credential default still usable). CREDENTIALS = {"TWOCAPTCHA_KEY", "SNAPCHAT_CDP_ENDPOINT", "SNAPCHAT_PROXY"} before = dict(os.environ) @@ -1573,7 +1584,7 @@ def check_a_copied_env_example_reads_as_UNSET(): def check_credential_scan_is_one_implementation_invoked_from_both(): - """§17: two sources of truth, one dead and one holed. + """Two sources of truth, one dead and one holed. `.github/ci_checks.py` sat in three repos invoked by NOTHING, while tests.yml carried an inline grep doing a narrower version of the same job @@ -1590,7 +1601,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): # file being missing: two suites in this family run INSIDE the Docker # image, which deliberately COPYs no .github/, and a check that quietly # starts passing once its input disappears is the same failure this - # function is about (CLAUDE.md §22). + # function is about. if not os.path.isdir(workflow_dir): skip("ci-wiring", "no .github/ in this tree (the Docker image)") elif os.path.exists(workflow): @@ -1619,7 +1630,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): "tests.yml's offline job mentions %r — one implementation, " "in ci_checks.py, invoked from both" % marker) # And the guard must have had something to read, or it passed for the - # wrong reason (CLAUDE.md §22). + # wrong reason. check("...and the offline job was actually found to scan", "ci_checks.py" in offline, "no offline job in tests.yml") result = subprocess.run([sys.executable, script, "--all"], cwd=HERE, @@ -1641,7 +1652,7 @@ def check_the_credential_scan_survives_a_venv_in_the_tree(): The fix is structural rather than a longer list of names — a directory holding `pyvenv.cfg` is a virtualenv whatever it is called — and this pins BOTH halves, because narrowing a credential scan is exactly how one - stops catching things. CLAUDE.md §22 records a sibling repo whose scan + stops catching things. A sibling repo's scan once caught an UNTRACKED `.env.bak` holding a live key, so scanning must not be reduced to tracked files. """ @@ -1695,8 +1706,108 @@ def check_the_credential_scan_survives_a_venv_in_the_tree(): found == ["planted.py"], repr(found)) +def check_counts_and_delays_refuse_negative_values(): + """`--retries -1` used to be accepted and silently run as one attempt. + Counts and delays are validated where argparse can name the flag; 0 + stays legal for `--retries` because it counts EXTRA attempts here.""" + import argparse as _ap + import page_flow as pf + equal("--retries 0 is legal", pf.non_negative_int("0"), 0) + equal("a delay of 0 is legal", pf.non_negative_float("0"), 0.0) + for fn, bad in ((pf.non_negative_int, "-1"), (pf.non_negative_int, "x"), + (pf.non_negative_float, "-0.5"), + (pf.non_negative_float, "nan")): + try: + fn(bad) + check("%s rejects %r" % (fn.__name__, bad), False) + except _ap.ArgumentTypeError: + check("%s rejects %r" % (fn.__name__, bad), True) + # Wiring, read off disk so it runs with no engine library installed. + want = {"--retries": "non_negative_int", "--delay": "non_negative_float", + "--retry-delay": "non_negative_float", + "--proxy-block-retries": "non_negative_int"} + wiring = {m: want for m in ENGINES} + wiring["scraper_api_client"] = {"--retries": "non_negative_int", + "--retry-delay": "non_negative_int"} + for module, flags in wiring.items(): + tree = ast.parse(open(os.path.join(HERE, module + ".py"), + encoding="utf-8").read()) + seen = {} + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and getattr(node.func, "attr", None) == "add_argument" + and node.args and isinstance(node.args[0], ast.Constant) + and node.args[0].value in flags): + for kw in node.keywords: + if kw.arg == "type": + seen[node.args[0].value] = ast.unparse(kw.value) \ + if hasattr(ast, "unparse") else ast.dump(kw.value) + for flag, fn in flags.items(): + check("%s %s is validated by %s" % (module, flag, fn), + fn in seen.get(flag, ""), repr(seen.get(flag))) + # And through a real parser, end to end. + import scraper_api_client as sac + saved = sys.argv + err = io.StringIO() + try: + sys.argv = ["scraper_api_client.py", "--url", "nasa", "--retries", "-1"] + with redirect_stderr(err): + sac.parse_args() + check("scraper_api_client refuses --retries -1", False) + except SystemExit as exc: + equal("scraper_api_client --retries -1 is a usage error", exc.code, 2) + check("...naming the flag", "--retries" in err.getvalue() + and "error:" in err.getvalue(), err.getvalue()[-200:]) + finally: + sys.argv = saved + + +# Tokens that point at documents a reader of this repo cannot open, built +# from pieces so this file does not match itself. +_PRIVATE_CITATION_RE = re.compile( + "|".join((re.escape("CLAUDE" + ".md"), "§" + r"\s*\d", + re.escape("claude_" + "2scraper"), + re.escape("2scraper_" + "repo_rules"), + r"(?:play" + r"book|tem" + r"plate)\s+§")), re.I) + + +def _private_citations(root): + hits = [] + for dirpath, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in + (".git", "__pycache__", ".pytest_cache", "node_modules") + and not os.path.exists(os.path.join(dirpath, d, + "pyvenv.cfg"))] + for filename in files: + path = os.path.join(dirpath, filename) + if not filename.endswith((".py", ".md", ".yml", ".yaml", ".txt", + ".toml", ".cfg", ".example", ".json", + ".csv", ".gitignore", + ".dockerignore")) \ + and filename != "Dockerfile": + continue + text = open(path, encoding="utf-8", errors="replace").read() + for m in _PRIVATE_CITATION_RE.finditer(text): + hits.append("%s:%d" % (os.path.relpath(path, root), + text.count("\n", 0, m.start()) + 1)) + return hits + + +def check_no_private_document_citations(): + """Shipped files explain each rule in their own words; they never point + at a private document or its section numbers, which no reader of this + repository can open.""" + hits = _private_citations(HERE) + check("no shipped file cites a private document", not hits, + ", ".join(hits[:10])) + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, "x.py"), "w", encoding="utf-8") as fh: + fh.write("# see " + "CLAUDE" + ".md " + "§" + "8\n") + check("the citation scan can fire", len(_private_citations(tmp)) == 2) + + def check_banned_wording(): - """§12: enforced by this test rather than by review.""" + """Banned wording: enforced by this test rather than by review.""" for root, dirs, files in os.walk(HERE): dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", ".pytest_cache", "node_modules")] @@ -1726,7 +1837,7 @@ def check_worker_pools_start_on_different_exits(): def check_fingerprint_kwargs_are_ones_the_driver_accepts(): - """§10: an unknown key in new_context(**kwargs) is a TypeError at launch, + """An unknown key in new_context(**kwargs) is a TypeError at launch, on the PAID path, at runtime.""" engine = _import_engine("playwright_scraper") if engine is None: @@ -1756,7 +1867,7 @@ def check_every_engine_exposes_the_same_public_surface(): Read from the SOURCE rather than from imported modules, and that is the fix rather than a style choice. The first version compared the engines that happened to import, so in a single-engine virtualenv — - the only configuration this repo's README supports (§6: install + the only configuration this repo's README supports (install exactly one) — it compared ONE engine against nothing and reported itself passed. Measured: 0 pairs compared, suite green. @@ -1842,7 +1953,7 @@ def check_every_engine_exposes_the_same_public_surface(): def check_every_solve_is_counted_against_the_budget(): """`SOLVES_PER_PAGE` is a MONEY limit, so every call that can buy must - be counted — CLAUDE.md §23. + be counted. `handle_captcha_if_present` is called TWICE per attempt in every engine in this family: once before the response is classified (so a challenge @@ -1911,7 +2022,7 @@ def check_every_solve_is_counted_against_the_budget(): def check_a_dead_proxy_is_reported_as_a_proxy_failure(): - """CLAUDE.md §8: a proxy failure is not a timeout, and the two want + """A proxy failure is not a timeout, and the two want opposite responses — another try at the same exit versus a different one. The engines all compute the reason (`_proxy_failure`) and, WITH a pool, @@ -1937,7 +2048,7 @@ def check_a_dead_proxy_is_reported_as_a_proxy_failure(): # The first version of this check read that block in all three # engines and failed on code that was correct. Reading the wrong # block is how a check like this passes, or fails, for the wrong - # reason (CLAUDE.md §22). + # reason. anchor = "def _fetch_with_policy(" if anchor not in source: check("%s has a give-up branch to check" % module, False) @@ -1988,7 +2099,7 @@ class _E(Exception): def check_engines_do_not_evaluate_a_string_in_the_browser(): - """§18: a site whose CSP omits `unsafe-eval` kills wait_for_function with + """A site whose CSP omits `unsafe-eval` kills wait_for_function with an EvalError and takes the run down with exit 1, on the site's most obvious URL. Snapchat's is such a site: measured 2026-09-24, its `script-src` carries `'wasm-unsafe-eval'` and NOT `'unsafe-eval'`, so @@ -2007,7 +2118,7 @@ def check_engines_do_not_evaluate_a_string_in_the_browser(): def check_credentials_never_reach_a_log(): - """§8: an EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. + """An EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. A Playwright connection error repeats the endpoint five times (the message plus a four-line call log), so a masker handling only the first @@ -2048,7 +2159,7 @@ def check_sample_output_matches_the_schema(): not any("lorem" in (r.get("title") or "").lower() or "example.com" in (r.get("url") or "").lower() for r in rows)) - # The sample is cut from a real run and NOT anonymised. CLAUDE.md §10's + # The sample is cut from a real run and NOT anonymised. The anonymising # rule is about a private individual's words; a row here is a public # profile's catalogue entry — the name, bio and counts Snapchat shows # any logged-out visitor. What the checks pin is that it is REAL, @@ -2125,7 +2236,7 @@ def fetch(box, pw, args, pool, url, label): def run(rotate, handles): # Assembled from pieces, not written out: the credential scan reads # this file, and a literal `user:pass@host` here would make the scan - # fail on its own test data (CLAUDE.md §22). + # fail on its own test data. exits = ["http:" + "//" + "u" + ":" + "p" + "@one.example:1", "http:" + "//" + "u" + ":" + "p" + "@two.example:2"] pool = proxy_pool.ProxyPool(exits, rotate=rotate) @@ -2222,12 +2333,12 @@ def check_a_site_that_answered_is_not_a_run_that_failed(): def check_a_fingerprint_is_applied_whole_or_not_at_all(): - """CLAUDE.md §24: a HALF identity is measured worse than none. + """A HALF identity is measured worse than none. Four defects lived on this path, every one of them a SILENT success — the call was accepted, the log said nothing, and the page disagreed with the fingerprint. All four were found by reading the values back - out of a live page on 2026-09-21, which is what §24 tells you to do + out of a live page on 2026-09-21, which is what to do and which no amount of reading this code would have produced: 1. `navigator.languages` reported `["en-US"]` against the @@ -2315,7 +2426,7 @@ def check_a_fingerprint_is_applied_whole_or_not_at_all(): check("%s applies the client hints beside the user agent" % module, '"userAgent": user_agent, "userAgentMetadata": metadata' in source, - "a bare override is the thing §24 measured being refused") + "a bare override is the thing a sibling site refused") check("%s refuses to apply a partial identity" % module, "HALF identity is worse than" in source, "no brand list must mean no override at all") @@ -2356,7 +2467,7 @@ def check_the_credential_scan_covers_the_files_it_most_needs_to(): """The repo's own guard was blind to its biggest files, twice over. Both were found by PLANTING a real-shaped key and running the scan - rather than by reading it (CLAUDE.md §23), and both reported + rather than by reading it, and both reported "nothing credential-shaped" over a file that held one: 1. `.json` and `.csv` were not in `SCANNED_SUFFIXES` at all, so @@ -2373,7 +2484,7 @@ def check_the_credential_scan_covers_the_files_it_most_needs_to(): Added with no new allowlist entries, which is the point: the real fixtures and sample carry zero 32-hex strings and zero credentialled URLs, so the strictest rule now covers the largest files instead of - acquiring an exception a real key could hide behind (CLAUDE.md §24). + acquiring an exception a real key could hide behind. """ sys.path.insert(0, os.path.join(HERE, ".github")) import ci_checks @@ -2412,7 +2523,7 @@ def check_the_credential_scan_covers_the_files_it_most_needs_to(): # The sample is ASSEMBLED from pieces rather than written out, and that # is not fussiness — the scan now reads this file, and a literal # credentialled URL here would make the check fail on its own test - # data. CLAUDE.md §22: a note about a banned string is a use of it, and + # data. A note about a banned string is a use of it, and # assembling is what lets the scan cover the suite instead of # exempting the one file most likely to acquire a pasted secret. sample = "ws:" + "//" + "acct7" + ":" + "s3cr3tpw" + "@" + "host:9222" @@ -2433,7 +2544,7 @@ def check_the_credential_scan_covers_the_files_it_most_needs_to(): def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): """The `engine-smoke` job must be ABLE to fail. - That job exists for one reason (CLAUDE.md §10): "skipped, engine absent" + That job exists for one reason: "skipped, engine absent" reads identically to a real import error, so CI installs each engine and fails if THAT engine's group still reports a skip. It works by grepping the suite's own output for a sentinel. @@ -2442,7 +2553,7 @@ def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): imported"` — a string no suite in this family emits. Checked 2026-09-18, seven sibling repos carry the same dead grep, so in none of them could the job ever have failed. It passed for the wrong reason, which is - §22's "a check that swallows the loudest failure it could report". + "a check that swallows the loudest failure it could report". This check is the guard against that coming back: whatever sentinel the workflow looks for, this suite has to be capable of printing it. It @@ -2452,7 +2563,7 @@ def check_ci_greps_for_a_sentinel_this_suite_can_actually_emit(): """ workflow = os.path.join(HERE, ".github", "workflows", "tests.yml") if not os.path.isdir(os.path.join(HERE, ".github")): - # §22: trigger on the WHOLE .github directory being absent — which + # Trigger on the WHOLE .github directory being absent — which # is the Docker image, where it is deliberately not COPYed — and # never on a file inside it going missing, because a check that # quietly starts passing once its input disappears is the failure @@ -2546,7 +2657,7 @@ def check_no_module_defines_a_name_twice(): That is the dangerous shape: a duplicate that is currently harmless and becomes a silent wrong answer the moment somebody edits one copy. - CLAUDE.md §22's statement-after-return check is the same family of + The statement-after-return check is the same family of defect — something the parser accepts and a reader never sees. """ import collections @@ -2571,7 +2682,7 @@ def check_no_module_defines_a_name_twice(): def check_no_statement_follows_a_return_in_the_same_block(): - """CLAUDE.md §22: found the same fifteen dead lines in six repos. + """Found the same fifteen dead lines in six repos. A function whose `def` line had been lost, leaving its docstring and body absorbed into the end of the function above it. It parses, it @@ -2605,7 +2716,7 @@ def check_no_statement_follows_a_return_in_the_same_block(): def check_the_scraping_browsers_own_extension_does_not_read_as_a_challenge(): """The marker check, against a page fetched the way a PAID run fetches. - CLAUDE.md §21 records a guard that passed for the WRONG REASON: it ran + A sibling recorded a guard that passed for the WRONG REASON: it ran only against captures taken with a plain HTTP client, which carry no extension injection at all. Every other fixture in this repo is such a capture. This one is the material a real `--cdp-endpoint` fetch adds. @@ -2631,8 +2742,7 @@ def check_the_scraping_browsers_own_extension_does_not_read_as_a_challenge(): product_parser.STATE_CONTENT) # And the set must score zero WITHOUT an extension strip, or the strip - # becomes load-bearing and the next marker added inherits the hole - # (CLAUDE.md §24). + # becomes load-bearing and the next marker added inherits the hole. for marker in snapchat_payload.BOT_CHALLENGE_MARKERS: check("marker %r is absent from the injected material" % marker, marker not in CDP_INJECTED, diff --git a/snapchat_payload.py b/snapchat_payload.py index 3d4116e..39f8f8a 100644 --- a/snapchat_payload.py +++ b/snapchat_payload.py @@ -24,7 +24,7 @@ (against FOUR Spotlight videos in the payload) @kyliejenner 0 blocks the page opens on a live story -CLAUDE.md §24 records a sibling whose JSON-LD was a decoy beside the real +A sibling repo's JSON-LD was a decoy beside the real grid; here it is a partial, sometimes-absent view of the same data. So `__NEXT_DATA__` is primary, and JSON-LD is read for the two facts only it states — `dateCreated` and `dateModified` — and to tell a HIDDEN @@ -39,7 +39,7 @@ # Anchored on the element id Next.js itself looks the payload up by, not on # any class in the rendered page: the id is part of the framework's -# contract, classes are build artefacts (CLAUDE.md §4). +# contract, classes are build artefacts. _NEXT_DATA_RE = re.compile( r']*>(.*?)', re.S) _LD_JSON_RE = re.compile( @@ -51,15 +51,14 @@ class PayloadError(ValueError): Distinct from "the payload said there is no such account", which is a fact about the account. Confusing the two is how a scraper reports a - parsing bug as an empty result, or an empty result as a parsing bug - (CLAUDE.md §20). + parsing bug as an empty result, or an empty result as a parsing bug. """ def decode_page(raw: Any, content_type: Optional[str] = None) -> str: """Bytes (or str) to text, honouring a declared charset. - CLAUDE.md §24: one site in this family serves its detail pages as + One site in this family serves its detail pages as EUC-JP and its listing pages as UTF-8, and a blind `bytes.decode("utf-8", "replace")` turned every title into replacement characters while the numbers still parsed. Snapchat is UTF-8 @@ -220,22 +219,17 @@ def to_int(raw: Any) -> Optional[int]: # profiles, Spotlight, Discover, Lenses, a topic page, the not-found page — # every candidate below scored ZERO, and so did the bare word `captcha`. # -# So this set is not a list of what Snapchat has been seen to render. No -# challenge has been observed. But one IS configured: the profile page's -# own Content-Security-Policy permits scripts from `hcaptcha.com`, -# `*.hcaptcha.com` and `www.google.com/recaptcha` (measured 2026-09-24). -# CLAUDE.md §18's point exactly — "no challenge rendered" is not "no -# captcha configured" — and the reason the first two loaders below are the -# site's own candidates rather than a vendor list copied from a sibling. It is the loader paths of the widgets a -# solver in this family can handle, chosen to be SPECIFIC: CLAUDE.md §24 -# counted 21 occurrences of the bare word `captcha` on a good page fetched +# No challenge has been observed, but the profile page's CSP permits +# third-party captcha script hosts, so "no challenge rendered" is not "no +# captcha configured". The set is loader paths, chosen to be SPECIFIC: a +# sibling counted 21 occurrences of the bare word `captcha` on a good page fetched # over the 2Captcha Scraping Browser, because its auto-solve extension # injects hunter scripts into every page. A loader path is injected by # nothing. # -# `cf-turnstile` is NOT here, deliberately: that same extension injects it -# (CLAUDE.md §8, §19). `challenges.cloudflare.com` is, and CLAUDE.md §23 -# records a site where it was INVERTED because the site loaded Turnstile +# `cf-turnstile` is NOT here, deliberately: that same extension injects it. +# `challenges.cloudflare.com` is, and a sibling +# recorded a site where it was INVERTED because the site loaded Turnstile # itself — which Snapchat does not, 0 on 26 served pages. Count it again on # any new route before trusting it. BOT_CHALLENGE_MARKERS = ( @@ -243,8 +237,7 @@ def to_int(raw: Any) -> Optional[int]: "recaptcha/enterprise.js", "recaptcha/api2/anchor", "hcaptcha.com/1/api.js", - # A RENDERED hCaptcha widget's own iframe path — what a browser DOM holds - # once the loader has run, when the loader tag itself may be gone. + # Rendered-widget iframe path. "hcaptcha.com/captcha/", "challenges.cloudflare.com", "_cf_chl_opt", @@ -264,13 +257,13 @@ def challenge_markers_present(html: Any) -> List[str]: return [m for m in BOT_CHALLENGE_MARKERS if m in text] -# Positive-asset detection (CLAUDE.md §8, §18): a page Snapchat actually +# Positive-asset detection: a page Snapchat actually # serves is built out of its own static host; an interstitial, a proxy's # error page or Chromium's own network-error page is not. Counted # 2026-09-24: 5 to 39 references on every served page, the not-found page # included. The threshold is 2 rather than 1 so a minimal real page is -# never read as somebody else's (CLAUDE.md §17's classification-order -# trap). +# never read as somebody else's (order signals by how much they +# prove). SITE_ASSET_MARKERS = ("static.snapchat.com",) MIN_ASSET_REFERENCES = 2