From 00a1c7ba9f27b411517862cb50a0d7bee77d8738 Mon Sep 17 00:00:00 2001 From: xevxx Date: Fri, 23 Oct 2026 13:28:54 +0100 Subject: [PATCH] Fix anonymous thread loading via logged-out comments API --- Cargo.lock | 311 ++++++++++++++++ Cargo.toml | 1 + INVESTIGATION.md | 217 +++++++++++ investigation-results.json | 93 +++++ src/bin/redlib-probe.rs | 93 +++++ src/client.rs | 34 +- src/lib.rs | 1 + src/oauth.rs | 34 +- src/post.rs | 25 +- src/thread.rs | 740 +++++++++++++++++++++++++++++++++++++ src/utils.rs | 1 + static/style.css | 8 + templates/comment.html | 2 + 13 files changed, 1519 insertions(+), 41 deletions(-) create mode 100644 INVESTIGATION.md create mode 100644 investigation-results.json create mode 100644 src/bin/redlib-probe.rs create mode 100644 src/thread.rs diff --git a/Cargo.lock b/Cargo.lock index 0a957535e..298774999 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -551,6 +557,29 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "cssparser" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e901edd733a1472f944a45116df3f846f54d37e67e68640ac8bb69689aca2aa" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "darling" version = "0.20.11" @@ -629,6 +658,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.11.3" @@ -666,6 +716,27 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + [[package]] name = "either" version = "1.19.0" @@ -811,6 +882,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.34" @@ -885,6 +966,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -977,6 +1067,17 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +[[package]] +name = "html5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4" +dependencies = [ + "log", + "markup5ever", + "match_token", +] + [[package]] name = "htmlescape" version = "0.3.1" @@ -1376,6 +1477,34 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "311fe69c934650f8f19652b3946075f0fc41ad8757dbb68f1ca14e7900ecc1c3" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "match_token" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1431,6 +1560,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "no_std_io2" version = "0.9.4" @@ -1526,6 +1661,58 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1562,6 +1749,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -1693,6 +1886,7 @@ dependencies = [ "route-recognizer", "rss", "rust-embed", + "scraper", "sealed_test", "serde", "serde_json", @@ -1832,6 +2026,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.5" @@ -1890,6 +2093,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f3a24d916e78954af99281a455168d4a9515d65eca99a18da1b813689c4ad9" +dependencies = [ + "cssparser", + "ego-tree", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + [[package]] name = "sealed_test" version = "1.1.0" @@ -1912,6 +2129,31 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "selectors" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5685b6ae43bfcf7d2e7dfcfb5d8e8f61b46442c902531e41a32a9a8bf0ee0fb6" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -2039,6 +2281,15 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha2" version = "0.11.1" @@ -2084,6 +2335,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -2122,6 +2379,31 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2192,6 +2474,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -2516,6 +2809,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2628,6 +2927,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.10" diff --git a/Cargo.toml b/Cargo.toml index 78583e701..307dda345 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ async-recursion = "1.1.1" pulldown-cmark = { version = "0.13.3", features = ["simd", "html"], default-features = false } tegen = "0.1.4" serde_urlencoded = "0.7.1" +scraper = { version = "0.24.0", default-features = false } chrono = { version = "0.4.39", default-features = false, features = ["std"] } htmlescape = "0.3.1" bincode = "1.3.3" diff --git a/INVESTIGATION.md b/INVESTIGATION.md new file mode 100644 index 000000000..9a3bbfd74 --- /dev/null +++ b/INVESTIGATION.md @@ -0,0 +1,217 @@ +# Anonymous Reddit thread investigation + +Status: verified source fix and separately built artifact. Deployment remains a +later, user-controlled operation. + +## Scope and source state + +- Investigation began 2026-10-22 on Debian 13 from Redlib commit + `ba116c1186f6b0f9d45f0ae50c827e13eb2f5ea2` (v0.40.0). +- Upstream was rechecked at 2026-10-22T23:02:00Z. The same commit was still + the head of `redlib-org/redlib:main`; no later upstream fix existed. +- Upstream was checked again immediately before handoff on 2026-10-23. Main + was still `ba116c11`; #598 remained open. Its new comments independently + reported both that the encoding workaround was blocked and that builds of + the polak14 Shreddit fork were loading comments successfully. +- The running Docker deployment was not inspected, restarted, rebuilt, or + modified. +- No account, login, authenticated user session, alternate server, proxy, VPN, + browser automation, or official Reddit application was used. + +Relevant upstream reports and changes: + +- [#591](https://github.com/redlib-org/redlib/issues/591) established the + route-specific failure: listings succeeded while comments JSON returned an + HTML 403 response. +- [PR #592](https://github.com/redlib-org/redlib/pull/592) and + [commit 0a3e9cde](https://github.com/redlib-org/redlib/commit/0a3e9cde456b9ce92008d0e3846301c5efcd6565) + introduced the encoded-separator transport workaround. +- [#598](https://github.com/redlib-org/redlib/issues/598) and the latest + comments in [#446](https://github.com/redlib-org/redlib/issues/446) showed + that Reddit blocked that workaround again on 2026-10-22. + +## Control gate and live observations + +Direct `curl` received 403 responses from both listing and thread candidates. +It therefore failed the listing control and was not used to reject any +comments-specific transport. Redlib's TLS/client emulation is materially part +of the request contract. + +At approximately 2026-10-22T22:52Z, an isolated public build containing the +Shreddit experiment was run on `127.0.0.1:18080`. It used Redlib's `wreq` +client and a freshly acquired anonymous Android OAuth identity. The process was +stopped after the checks. The installed Docker deployment was untouched. + +The same client and network produced: + +| Check | Result | +| --- | --- | +| OAuth listing control | Success; rate-limit headers were present | +| Legacy comments JSON | HTTP 403 HTML block response | +| Supplied Northern Ireland thread | Rendered successfully; legitimately reported zero comments | +| `r/mildlyinfuriating` thread from #598 | 25 parsed comments; post declared 5.4k | +| `r/politics` thread from #591 | 25 parsed comments; post declared 926 | +| `sort=new` | 25 parsed comments | +| Top-level continuation | Success; 10 additional comments | +| Focused-comment route | Success; two comments rendered and target highlighted | + +The machine-readable, sanitized record is in +[`investigation-results.json`](investigation-results.json). It contains no +headers, cookies, tokens, bodies, device identifiers, or tracking identifiers. + +## Candidate conclusions + +### Selected: logged-out Shreddit + +Reddit's logged-out page calls these public, Reddit-owned services: + +- `GET /svc/shreddit/comments/r/{subreddit}/t3_{post}?sort={SORT}` +- `GET /svc/shreddit/comment/t1_{comment}?depth=0&subredditName={subreddit}` +- `POST /svc/shreddit/more-comments/{subreddit}/t3_{post}` + +Post metadata remains available through anonymous OAuth at `/by_id/t3_{id}`. +The Shreddit responses contain stable structural elements for comment IDs, +parent IDs, bodies, scores, timestamps, and continuation cursors. This +candidate passed post identity, nested-parent, sort, focused-comment, empty +discussion, and continuation checks. + +The initial endpoint/parser evidence came from the AGPL-3.0-only fork commits +[`6aed00a7`](https://github.com/polak14/redlib/commit/6aed00a7cadba3fbf24979322936d92859968ee8), +[`0baefabf`](https://github.com/polak14/redlib/commit/0baefabf4e806006e62c30a3b8766fa458bf08d2), +and [`4055fda1`](https://github.com/polak14/redlib/commit/4055fda1a5355f1507b23108795d827a16794b58). +The local implementation adds bounded reads, fixed-host enforcement, explicit +error categories, opaque thread/sort-bound continuation state, cache control, +credential-log redaction, and stricter URL/media sanitization. + +### Rejected as the durable primary: encoded legacy comments JSON + +The `%2Fcomments` variant was a routing discrepancy, not a supported anonymous +interface. It worked on 2026-10-21 and was blocked on 2026-10-22. It remains +useful evidence that the OAuth identity itself was valid, but adding another +encoding mutation would repeat the same failure mode. + +### Not selected: anonymous GraphQL + +No known-valid current comments operation and persisted-query hash was found in +the reviewed public source before Shreddit passed every candidate gate. Empty +or guessed GraphQL requests were deliberately not treated as authentication +tests. Per the investigation plan, further alternatives were not probed after +the first candidate qualified. + +### Controls only: `/by_id` and `/duplicates` + +`/by_id` supplies post metadata but not a comment tree. `/duplicates` supplies +related posts, not comments. Neither can restore thread semantics alone. + +## Implementation contract + +- `src/thread.rs` is the only thread-specific upstream adapter. +- `src/post.rs` consumes a validated `ThreadData` normalized into Redlib's + existing post/comment representation. Generic listing requests are unchanged. +- Requests use hard-coded Reddit hosts, do not follow redirects, time out after + 20 seconds, and reject responses above 8 MiB. +- HTTP 401, 403, 404, 429, 451, transport failures, and malformed data are + distinct internal outcomes. A comments 403 never triggers token refresh. +- Comment HTML is structurally parsed and rebuilt from an allowlist. Scripts, + event attributes, embedded documents, unsafe URL schemes, third-party media + loads, and known tracking elements are removed before `safe` rendering. +- Successful logical thread requests are cached for 30 seconds. Failures are + not converted into or cached as empty discussions. +- Continuations are base64url-encoded local state containing a version, thread + ID, sort, and bounded upstream cursor. The thread and sort must match before a + fixed-endpoint POST is made; arbitrary upstream URLs are never accepted. +- Anonymous tokens, response session headers, device UUIDs, and cookies are no + longer emitted by OAuth info/trace logging. + +## Verification of this implementation + +Build environment: + +- Rust/Cargo 1.99.1, rustfmt 1.10.0, Clippy 0.1.99 +- GCC 14.2.0, CMake 3.31.6, libclang 19.1.7 +- Debian build-essential 12.12 + +Completed checks: + +- `cargo fmt --all -- --check`: pass +- `cargo test --locked --offline thread::tests`: pass, seven tests +- `cargo clippy --locked --offline --all-targets -- -D warnings`: pass +- `cargo build --locked --offline --release --bins`: pass +- Release artifacts: + - `target/release/redlib` (14,509,024 bytes), SHA-256 + `c12ca6f566c8a3261ec5187cc715be9ede0053f18e2bc51c83ad374cdaaa6fe6` + - `target/release/redlib-probe` (10,166,176 bytes), SHA-256 + `0d93931cefb905a1e91dad7cb2f6b6c4bddd576d6698e02fc488039e27ee7cc6` +- Four separate executions of the release probe, each creating a fresh + anonymous identity: all listing controls and all 12 thread checks passed. +- Every probe listing contained 25 children. On each identity, the supplied + thread validated as empty and both populated controls produced validated + comments. The first three identities received 10-comment fragments; the + final, exact checksummed artifact received 25-comment fragments. The large + thread exposed a continuation throughout. +- Isolated release server on `127.0.0.1:18081`: listing, empty thread, populated + thread, `sort=new`, continuation, focused-comment context, local comment + search, and user filtering all returned successful HTML without an error + page. The continuation produced nine further comments; the focused route + rendered 11 comments and exactly one highlighted target. +- Replaying a continuation with a different sort produced the expected local + malformed-request error rather than an upstream request. +- Rendered live pages contained no script tags or inline event attributes. + Image/video rewriting and unsafe media removal are additionally covered by + the structural sanitizer fixture. +- The loopback process was stopped after validation. The Docker deployment was + never changed. + +An explicit Rust 1.81 build could not be completed because the unmodified +v0.40.0 lockfile already contains `cpufeatures 0.3.1`, whose Edition 2024 +manifest cannot be parsed by Cargo 1.81. This is a pre-existing declared-MSRV +drift, not introduced by the `scraper` addition; the selected `scraper 0.24` +itself uses Edition 2021. Current stable Rust builds cleanly. + +## Reproduction and maintenance + +Opt-in probe, after building: + +```sh +cargo run --locked --bin redlib-probe +``` + +The probe first requires a successful listing control, uses uncached Redlib +requests, runs sequentially with at least two seconds between thread checks, +and emits sanitized JSON Lines. Pass explicit local thread paths as arguments +to replace the fixed set. Redirects remain disabled. A 429 is reported with its +`Retry-After` value and is not retried early. + +Offline checks: + +```sh +cargo fmt --all -- --check +cargo test --locked --offline thread::tests +cargo clippy --locked --offline --all-targets -- -D warnings +cargo build --locked --release +``` + +Maintenance procedure when Reddit changes Shreddit markup: + +1. Run the probe and require the listing control to pass. +2. Capture only a deliberately sanitized fragment; never commit cookies, + headers, tokens, cursors from a personal context, or tracking identifiers. +3. Update the selectors and normalization in `src/thread.rs` with a minimal + fixture reproducing the changed structure. +4. Re-run sanitizer, nesting, focus, sort, continuation, empty-discussion, and + malformed-input tests before any live check. +5. Confirm all outbound URLs remain compile-time-selected Reddit hosts. + +## Remaining uncertainty + +No six-hour, 30-minute-interval soak was completed after the final binary was +available. The four-identity validation and end-to-end run covered the request +contract but are not a time-based stability result. The earlier candidate test +and the final build both worked from the evening of 2026-10-22 through the +morning of 2026-10-23, but they were not periodic samples of the same artifact +and are not represented as a soak. + +Reddit can still deliberately remove or gate its logged-out web interface. The +fix is durable in the engineering sense—supported-client transport, bounded and +validated requests, localized adaptation, diagnostics, and regression tests— +not a guarantee against a future upstream policy change. diff --git a/investigation-results.json b/investigation-results.json new file mode 100644 index 000000000..ce6ac4e9d --- /dev/null +++ b/investigation-results.json @@ -0,0 +1,93 @@ +{ + "schema": 1, + "sanitized": true, + "checkout": "ba116c1186f6b0f9d45f0ae50c827e13eb2f5ea2", + "observed_utc_approx": "2026-10-22T22:52:00Z", + "network": "same host and residential connection as target deployment", + "client": "isolated Redlib wreq process on 127.0.0.1:18080", + "experiments": [ + { + "category": "listing-control", + "transport": "anonymous OAuth listing", + "outcome": "success", + "validated": ["JSON listing", "rate-limit headers"] + }, + { + "category": "legacy-comments-json", + "transport": "anonymous OAuth comments route", + "outcome": "failure", + "http_status": 403, + "content_category": "HTML policy response" + }, + { + "category": "shreddit-thread", + "logical_path": "/r/northernireland/comments/1xau6bp/teachers_and_youth_workers_find_out_how_you_can/", + "outcome": "success", + "parsed_comments": 0, + "discussion_state": "legitimately empty" + }, + { + "category": "shreddit-thread", + "logical_path": "/r/mildlyinfuriating/comments/1xcm2ou/the_guy_next_to_me_claimed_both_armrests_and/", + "outcome": "success", + "parsed_comments": 25, + "declared_comments": "5.4k" + }, + { + "category": "shreddit-thread", + "logical_path": "/r/politics/comments/1x9xx9e/", + "outcome": "success", + "parsed_comments": 25, + "declared_comments": 926 + }, + { + "category": "shreddit-sort", + "sort": "new", + "outcome": "success", + "parsed_comments": 25 + }, + { + "category": "shreddit-continuation", + "outcome": "success", + "parsed_comments": 10 + }, + { + "category": "shreddit-focused-comment", + "context": 3, + "outcome": "success", + "parsed_comments": 2, + "highlighted_targets": 1 + } + ], + "final_build_validation": { + "observed_utc": "2026-10-23T07:21:51Z", + "fresh_anonymous_identities": 4, + "listing_controls_passed": 4, + "thread_checks_passed": 12, + "format": "pass", + "targeted_tests": { + "passed": 7, + "failed": 0 + }, + "clippy_warnings_denied": "pass", + "locked_release_build": "pass", + "release_artifacts": { + "redlib_sha256": "c12ca6f566c8a3261ec5187cc715be9ede0053f18e2bc51c83ad374cdaaa6fe6", + "probe_sha256": "0d93931cefb905a1e91dad7cb2f6b6c4bddd576d6698e02fc488039e27ee7cc6" + }, + "loopback_end_to_end": { + "address": "127.0.0.1:18081", + "listing": "pass", + "empty_thread": "pass", + "populated_thread": "pass", + "sort": "pass", + "focused_comment": "pass", + "continuation": "pass", + "comment_search": "pass", + "user_filter": "pass", + "cross_sort_continuation_rejected": true + }, + "six_hour_soak_completed": false + }, + "secrets_or_identifiers_recorded": false +} diff --git a/src/bin/redlib-probe.rs b/src/bin/redlib-probe.rs new file mode 100644 index 000000000..a7473f514 --- /dev/null +++ b/src/bin/redlib-probe.rs @@ -0,0 +1,93 @@ +//! Low-rate, opt-in anonymous thread diagnostic. +//! +//! Usage: +//! `cargo run --bin redlib-probe -- [thread-path ...]` +//! +//! Output is JSON Lines and intentionally contains no request headers, tokens, +//! cookies, response bodies, or Reddit tracking identifiers. + +use redlib::client::json_uncached; +use redlib::thread::fetch_uncached; +use serde_json::json; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const DEFAULT_THREADS: &[&str] = &[ + "/r/northernireland/comments/1xau6bp/teachers_and_youth_workers_find_out_how_you_can/", + "/r/mildlyinfuriating/comments/1xcm2ou/the_guy_next_to_me_claimed_both_armrests_and/", + "/r/politics/comments/1x9xx9e/", +]; + +fn timestamp() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} + +#[tokio::main] +async fn main() { + let paths: Vec = std::env::args().skip(1).collect(); + let mut threads: Vec = if paths.is_empty() { + DEFAULT_THREADS.iter().map(|path| (*path).to_owned()).collect() + } else { + paths + }; + // One listing control plus at most 299 thread probes. + threads.truncate(299); + + let started = Instant::now(); + let listing_path = format!("/r/reddit/hot.json?raw_json=1&probe={}", timestamp()); + let listing = json_uncached(listing_path.clone(), false).await; + let listing_succeeded = listing.is_ok(); + let listing_children = listing.as_ref().ok().and_then(|value| value["data"]["children"].as_array()).map_or(0, Vec::len); + let listing_error = listing.as_ref().err().cloned(); + println!( + "{}", + json!({ + "time_unix_utc": timestamp(), + "category": "listing-control", + "logical_path": listing_path, + "transport": "https://oauth.reddit.com/r/reddit/hot.json", + "status": if listing_succeeded { "success" } else { "failure" }, + "latency_ms": started.elapsed().as_millis(), + "validated_listing_children": listing_children, + "error": listing_error, + }) + ); + if !listing_succeeded { + std::process::exit(2); + } + + let mut consecutive_failures = 0_u8; + for path in threads { + tokio::time::sleep(Duration::from_secs(2)).await; + let logical_path = format!("{}.json?sort=confidence&raw_json=1", path.trim_end_matches('/')); + let started = Instant::now(); + let result = fetch_uncached(logical_path.clone(), false).await; + let rate_limited = matches!(&result, Err(redlib::thread::ThreadError::RateLimited(_))); + if result.is_ok() { + consecutive_failures = 0; + } else { + consecutive_failures += 1; + } + println!( + "{}", + json!({ + "time_unix_utc": timestamp(), + "category": "thread", + "logical_path": logical_path, + "transport": "https://oauth.reddit.com/by_id + https://www.reddit.com/svc/shreddit/comments", + "status": if result.is_ok() { "success" } else { "failure" }, + "content_type": "text/html (comments), application/json (post metadata)", + "redirects_followed": 0, + "latency_ms": started.elapsed().as_millis(), + "post_id": result.as_ref().ok().map(|data| data.post_id()), + "validated_comments": result.as_ref().ok().map_or(0, |data| data.rendered_comment_count()), + "continuations": result.as_ref().ok().map_or(0, |data| data.continuation_count()), + "error": result.err().map(|error| error.to_string()), + }) + ); + // Do not send another request before a Retry-After interval. This probe + // has no need to retry, so stopping is the safest way to honor it. + if rate_limited || consecutive_failures >= 3 { + break; + } + } +} diff --git a/src/client.rs b/src/client.rs index a68bafaab..baed9a165 100644 --- a/src/client.rs +++ b/src/client.rs @@ -405,6 +405,12 @@ fn comments_logical_path(path: &str) -> String { /// Make a request to a Reddit API and parse the JSON response #[cached(size = 100, time = 30, result = true)] pub async fn json(path: String, quarantine: bool) -> Result { + json_uncached(path, quarantine).await +} + +/// Uncached variant used by the opt-in diagnostic harness. Application code +/// should use [`json`] so successful responses retain the normal 30s cache. +pub async fn json_uncached(path: String, quarantine: bool) -> Result { // Closure to quickly build errors let err = |msg: &str, e: String, path: String| -> Result { // eprintln!("{} - {}: {}", url, msg, e); @@ -425,6 +431,12 @@ pub async fn json(path: String, quarantine: bool) -> Result { match reddit_get(path.clone(), quarantine).await { Ok(response) => { let status = response.status(); + let content_type = response + .headers() + .get(wreq_header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_ascii_lowercase(); let reset: Option = if let (Some(remaining), Some(reset), Some(used)) = ( response.headers().get("x-ratelimit-remaining").and_then(|val| val.to_str().ok().map(|s| s.to_string())), @@ -446,21 +458,25 @@ pub async fn json(path: String, quarantine: bool) -> Result { None }; + // Reddit sometimes returns an HTML policy page for blocked API routes. + // Report that accurately and never treat it as an expired OAuth token. + if status.as_u16() == 403 && content_type.starts_with("text/html") { + return Err("Reddit blocked anonymous access to this endpoint".to_string()); + } + if status.as_u16() == 429 { + return Err(reset.map_or_else( + || "Reddit rate limit exceeded".to_string(), + |seconds| format!("Reddit rate limit exceeded. Rate limit will reset in: {seconds}"), + )); + } + // asynchronously aggregate the chunks of the body match hyper::body::aggregate(response.into_hyper_response()).await { Ok(body) => { let has_remaining = body.has_remaining(); if !has_remaining { - // Rate limited, so spawn a force_refresh_token() - tokio::spawn(force_refresh_token()); - return match reset { - Some(val) => Err(format!( - "Reddit rate limit exceeded. Try refreshing in a few seconds.\ - Rate limit will reset in: {val}" - )), - None => Err("Reddit rate limit exceeded".to_string()), - }; + return Err(format!("Reddit returned an empty response with HTTP {status}")); } // Parse the response from Reddit as JSON diff --git a/src/lib.rs b/src/lib.rs index b8eb17e7b..a2fa12ca9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,5 +9,6 @@ pub mod search; pub mod server; pub mod settings; pub mod subreddit; +pub mod thread; pub mod user; pub mod utils; diff --git a/src/oauth.rs b/src/oauth.rs index 4fba584ab..3b2e18b46 100644 --- a/src/oauth.rs +++ b/src/oauth.rs @@ -88,7 +88,7 @@ impl Oauth { match e { AuthError::Wreq(error) => error.to_string(), AuthError::SerdeDeserialize(error) => error.to_string(), - AuthError::Field((value, error)) => format!("{error}\n{value}"), + AuthError::Field(error) => error.to_string(), } ); } @@ -142,7 +142,7 @@ impl Oauth { enum AuthError { Wreq(wreq::Error), SerdeDeserialize(serde_json::Error), - Field((serde_json::Value, &'static str)), + Field(&'static str), } impl From for AuthError { @@ -244,7 +244,7 @@ impl OauthBackend for MobileSpoofAuth { let resp = builder.json(&json).send().await?; trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length")); - trace!("OAuth headers: {:#?}", resp.headers()); + // Response headers can contain anonymous session identifiers. Do not log them. // Parse headers - loid header _should_ be saved sent on subsequent token refreshes. // Technically it's not needed, but it's easy for Reddit API to check for this. @@ -272,17 +272,17 @@ impl OauthBackend for MobileSpoofAuth { // Save token and expiry let token = json .get("access_token") - .ok_or_else(|| AuthError::Field((json.clone(), "access_token")))? + .ok_or(AuthError::Field("access_token"))? .as_str() - .ok_or_else(|| AuthError::Field((json.clone(), "access_token: as_str")))? + .ok_or(AuthError::Field("access_token: as_str"))? .to_string(); let expires_in = json .get("expires_in") - .ok_or_else(|| AuthError::Field((json.clone(), "expires_in")))? + .ok_or(AuthError::Field("expires_in"))? .as_u64() - .ok_or_else(|| AuthError::Field((json.clone(), "expires_in: as_u64")))?; + .ok_or(AuthError::Field("expires_in: as_u64"))?; - info!("[✅] Success - Retrieved token \"{}...\", expires in {}", &token[..32], expires_in); + info!("[✅] Success - Retrieved anonymous token, expires in {expires_in} seconds"); Ok(OauthResponse { token, @@ -357,7 +357,7 @@ impl OauthBackend for GenericWebAuth { let resp: wreq::Response = builder.body(body_str).send().await?; trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length")); - trace!("GenericWebAuth headers: {:#?}", resp.headers()); + // Response headers can contain anonymous session identifiers. Do not log them. // Parse headers - loid header _should_ be saved sent on subsequent token refreshes. // Technically it's not needed, but it's easy for Reddit API to check for this. @@ -385,21 +385,17 @@ impl OauthBackend for GenericWebAuth { // Parse response - access_token, token_type, device_id, expires_in, scope let token = json .get("access_token") - .ok_or_else(|| AuthError::Field((json.clone(), "access_token")))? + .ok_or(AuthError::Field("access_token"))? .as_str() - .ok_or_else(|| AuthError::Field((json.clone(), "access_token: as_str")))? + .ok_or(AuthError::Field("access_token: as_str"))? .to_string(); let expires_in = json .get("expires_in") - .ok_or_else(|| AuthError::Field((json.clone(), "expires_in")))? + .ok_or(AuthError::Field("expires_in"))? .as_u64() - .ok_or_else(|| AuthError::Field((json.clone(), "expires_in: as_u64")))?; + .ok_or(AuthError::Field("expires_in: as_u64"))?; - info!( - "[✅] GenericWebAuth success - Retrieved token \"{}...\", expires in {}", - &token[..32.min(token.len())], - expires_in - ); + info!("[✅] GenericWebAuth success - Retrieved anonymous token, expires in {expires_in} seconds"); // Insert a few necessary headers self.additional_headers.insert("Origin".to_owned(), "https://www.reddit.com".to_owned()); @@ -450,7 +446,7 @@ impl Device { ("X-Reddit-Device-Id".into(), uuid.clone()), ]); - info!("[🔄] Spoofing Android client with headers: {headers:?}, uuid: \"{uuid}\", and OAuth ID \"{REDDIT_ANDROID_OAUTH_CLIENT_ID}\""); + info!("[🔄] Preparing an ephemeral anonymous Android client identity"); Self { oauth_id: REDDIT_ANDROID_OAUTH_CLIENT_ID.to_string(), diff --git a/src/post.rs b/src/post.rs index 8dac5064e..8683e4c83 100644 --- a/src/post.rs +++ b/src/post.rs @@ -1,8 +1,8 @@ #![allow(clippy::cmp_owned)] -use crate::client::json; use crate::config::get_setting; use crate::server::RequestExt; use crate::subreddit::{can_access_quarantine, quarantine}; +use crate::thread::{fetch as fetch_thread, ThreadError}; use crate::utils::{ error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post, Preferences, }; @@ -57,9 +57,10 @@ pub async fn item(req: Request) -> Result, String> { let highlighted_comment = &req.param("comment_id").unwrap_or_default(); // Send a request to the url, receive JSON in response - match json(path, quarantined).await { + match fetch_thread(path, quarantined).await { // Otherwise, grab the JSON output from the request - Ok(response) => { + Ok(thread) => { + let response = thread.into_legacy_json(); // Parse the JSON into Post and Comment structs let post = parse_post(&response[0]["data"]["children"][0]).await; @@ -98,14 +99,11 @@ pub async fn item(req: Request) -> Result, String> { })) } // If the Reddit API returns an error, exit and send error page to user - Err(msg) => { - if msg == "quarantined" || msg == "gated" { - let sub = req.param("sub").unwrap_or_default(); - Ok(quarantine(&req, sub, &msg)) - } else { - error(req, &msg).await - } + Err(ThreadError::Restricted(reason)) if reason == "quarantined" || reason == "gated" => { + let sub = req.param("sub").unwrap_or_default(); + Ok(quarantine(&req, sub, &reason)) } + Err(error_message) => error(req, &error_message.to_string()).await, } } @@ -198,7 +196,7 @@ fn build_comment( let awards: Awards = Awards::parse(&data["all_awardings"]); let parent_kind_and_id = val(comment, "parent_id"); - let parent_info = parent_kind_and_id.split('_').collect::>(); + let (parent_kind, parent_id) = parent_kind_and_id.split_once('_').unwrap_or(("", "")); let highlighted = id == highlighted_comment; @@ -229,8 +227,8 @@ fn build_comment( Comment { id, kind, - parent_id: parent_info[1].to_string(), - parent_kind: parent_info[0].to_string(), + parent_id: parent_id.to_string(), + parent_kind: parent_kind.to_string(), post_link: post_link.to_string(), post_author: post_author.to_string(), body, @@ -249,6 +247,7 @@ fn build_comment( collapsed, is_filtered, more_count, + continuation_sort: val(comment, "continuation_sort"), prefs: Preferences::new(req), } } diff --git a/src/thread.rs b/src/thread.rs new file mode 100644 index 000000000..1e2ea1d33 --- /dev/null +++ b/src/thread.rs @@ -0,0 +1,740 @@ +//! Fetches public thread data without using Reddit's blocked comments JSON route. +//! +//! Post metadata still comes from `/by_id`. Comment trees come from the HTML +//! fragments used by Reddit's logged-out Shreddit client and are normalized to +//! the legacy listing shape consumed by `post.rs`. +//! +//! The endpoint discovery and initial parser were informed by polak14/redlib +//! commits 6aed00a7, 0baefabf, and 4055fda1 (AGPL-3.0-only). This version makes +//! the transport primary, validates and binds continuation state, bounds all +//! responses, and rebuilds comment HTML through an allowlist. + +use crate::client::{json, json_uncached, CLIENT}; +use crate::utils::{format_url, param}; +use base64::{engine::general_purpose, Engine as _}; +use cached::proc_macro::cached; +use chrono::DateTime; +use futures_lite::StreamExt; +use scraper::{ElementRef, Html, Node, Selector}; +use serde::{Deserialize, Serialize}; +use serde_json::{json as json_value, Value}; +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::sync::LazyLock; +use std::time::Duration; +use tokio::time::timeout; +use wreq::{Method, Response}; + +const SVC_BASE: &str = "https://www.reddit.com/svc/shreddit"; +const MAX_RESPONSE_BYTES: usize = 8 * 1024 * 1024; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); +const MAX_CURSOR_TOKEN_BYTES: usize = 8192; +const MAX_UPSTREAM_CURSOR_BYTES: usize = 4096; +const CONTINUATION_VERSION: u8 = 1; + +static COMMENT: LazyLock = LazyLock::new(|| Selector::parse("shreddit-comment").expect("valid static selector")); +static MORE: LazyLock = LazyLock::new(|| Selector::parse(r#"faceplate-partial[src*="/more-comments/"]"#).expect("valid static selector")); +static CURSOR: LazyLock = LazyLock::new(|| Selector::parse(r#"input[name="cursor"]"#).expect("valid static selector")); +static MORE_COUNT: LazyLock = LazyLock::new(|| Selector::parse("faceplate-number[number]").expect("valid static selector")); + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct ThreadRequest { + post_id: String, + comment_id: Option, + sort: String, + continuation: Option, +} + +/// A validated thread response. The fields remain private so callers cannot +/// accidentally bypass validation before converting to the existing parser's +/// input shape. +#[derive(Clone, Debug)] +pub struct ThreadData { + post: Value, + comments: Value, + continuations: usize, +} + +impl ThreadData { + pub fn into_legacy_json(self) -> Value { + json_value!([self.post, self.comments]) + } + + pub fn continuation_count(&self) -> usize { + self.continuations + } + + pub fn post_id(&self) -> &str { + self.post["data"]["children"][0]["data"]["id"].as_str().unwrap_or_default() + } + + pub fn rendered_comment_count(&self) -> usize { + self.comments["data"]["children"].as_array().map_or(0, |items| count_kind(items, "t1")) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ThreadError { + Restricted(String), + Missing, + InvalidAnonymousState, + RateLimited(Option), + UpstreamBlocked, + Transport(String), + Malformed(String), +} + +impl fmt::Display for ThreadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Restricted(reason) => write!(f, "{reason}"), + Self::Missing => write!(f, "Reddit did not return this thread or comment"), + Self::InvalidAnonymousState => write!(f, "Reddit rejected the anonymous session state"), + Self::RateLimited(Some(seconds)) => write!(f, "Reddit rate limited this request; retry after {seconds} seconds"), + Self::RateLimited(None) => write!(f, "Reddit rate limited this request"), + Self::UpstreamBlocked => write!(f, "Reddit blocked anonymous thread access from this connection"), + Self::Transport(detail) => write!(f, "Unable to retrieve the thread from Reddit: {detail}"), + Self::Malformed(detail) => write!(f, "Reddit returned malformed thread data: {detail}"), + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct ContinuationState { + v: u8, + thread: String, + sort: String, + cursor: String, +} + +/// Fetch a thread using Reddit's logged-out page transport. Successful results +/// are cached for 30 seconds; failures are never cached as empty discussions. +pub async fn fetch(path: String, quarantine: bool) -> Result { + let request = parse_request(&path)?; + fetch_cached(request, quarantine).await +} + +/// Uncached entry point for deliberate low-rate live diagnostics. +pub async fn fetch_uncached(path: String, quarantine: bool) -> Result { + let request = parse_request(&path)?; + fetch_upstream(request, quarantine, true).await +} + +#[cached(size = 100, time = 30, result = true)] +async fn fetch_cached(request: ThreadRequest, quarantine: bool) -> Result { + fetch_upstream(request, quarantine, false).await +} + +async fn fetch_upstream(request: ThreadRequest, quarantine: bool, uncached: bool) -> Result { + let metadata_path = format!("/by_id/t3_{}.json?raw_json=1", request.post_id); + let post = if uncached { + json_uncached(metadata_path, quarantine).await + } else { + json(metadata_path, quarantine).await + } + .map_err(classify_metadata_error)?; + validate_post(&post, &request.post_id)?; + + let post_data = &post["data"]["children"][0]["data"]; + let sub = post_data["subreddit"] + .as_str() + .filter(|sub| valid_subreddit(sub)) + .ok_or_else(|| ThreadError::Malformed("post subreddit was missing or invalid".into()))?; + + let html = if let Some(comment_id) = &request.comment_id { + if request.continuation.is_some() { + return Err(ThreadError::Malformed("continuations are not accepted on focused-comment routes".into())); + } + let url = format!("{SVC_BASE}/comment/t1_{comment_id}?depth=0&subredditName={sub}"); + let mut html = request_html(Method::GET, &url, quarantine, None).await?; + html.push_str(&more_comments(sub, &request.post_id, &cursor_for(comment_id), quarantine).await?); + html + } else if let Some(token) = &request.continuation { + let state = decode_continuation(token, &request.post_id, &request.sort)?; + more_comments(sub, &request.post_id, &state.cursor, quarantine).await? + } else { + let url = format!("{SVC_BASE}/comments/r/{sub}/t3_{}?sort={}", request.post_id, request.sort.to_uppercase()); + request_html(Method::GET, &url, quarantine, None).await? + }; + + let parsed = parse_comments(&html, &request.post_id, &request.sort)?; + let continuations = count_kind(&parsed, "more_comments"); + Ok(ThreadData { + post, + comments: listing(parsed), + continuations, + }) +} + +fn parse_request(path: &str) -> Result { + let route = path + .split('?') + .next() + .ok_or_else(|| ThreadError::Malformed("thread path was empty".into()))? + .trim_end_matches(".json") + .trim_end_matches('/'); + let segments: Vec<&str> = route.split('/').filter(|segment| !segment.is_empty()).collect(); + let comments_at = segments + .iter() + .position(|segment| *segment == "comments") + .ok_or_else(|| ThreadError::Malformed("not a comments route".into()))?; + let post_id = segments + .get(comments_at + 1) + .copied() + .filter(|id| valid_base36_id(id)) + .ok_or_else(|| ThreadError::Malformed("post ID was missing or invalid".into()))?; + let comment_id = segments.get(comments_at + 3).copied().filter(|id| !id.is_empty()).map(str::to_owned); + if comment_id.as_deref().is_some_and(|id| !valid_base36_id(id)) { + return Err(ThreadError::Malformed("comment ID was invalid".into())); + } + + let sort = param(path, "sort") + .filter(|sort| matches!(sort.as_str(), "confidence" | "top" | "new" | "controversial" | "old")) + .unwrap_or_else(|| "confidence".into()); + let continuation = param(path, "cursor"); + if continuation.as_ref().is_some_and(|value| value.len() > MAX_CURSOR_TOKEN_BYTES) { + return Err(ThreadError::Malformed("continuation token was too large".into())); + } + + Ok(ThreadRequest { + post_id: post_id.to_owned(), + comment_id, + sort, + continuation, + }) +} + +fn valid_base36_id(value: &str) -> bool { + !value.is_empty() && value.len() <= 16 && value.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()) +} + +fn valid_subreddit(value: &str) -> bool { + !value.is_empty() && value.len() <= 32 && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} + +fn validate_post(post: &Value, expected_id: &str) -> Result<(), ThreadError> { + let children = post["data"]["children"] + .as_array() + .ok_or_else(|| ThreadError::Malformed("post listing was missing children".into()))?; + let Some(first) = children.first() else { + return Err(ThreadError::Missing); + }; + if first["kind"] != "t3" || first["data"]["id"].as_str() != Some(expected_id) { + return Err(ThreadError::Malformed("post identity did not match the requested thread".into())); + } + Ok(()) +} + +fn classify_metadata_error(error: String) -> ThreadError { + match error.as_str() { + "quarantined" | "gated" | "private" | "banned" => ThreadError::Restricted(error), + _ if error.contains("rate limit") || error.contains("Too many requests") => ThreadError::RateLimited(None), + _ if error.contains("OAuth token has expired") => ThreadError::InvalidAnonymousState, + _ if error.contains("blocked anonymous access") => ThreadError::UpstreamBlocked, + _ => ThreadError::Transport(error), + } +} + +fn cursor_for(comment_id: &str) -> String { + general_purpose::STANDARD.encode(format!("commenttree:,::1:d1:t1_{comment_id}")) +} + +fn cookies(quarantine: bool) -> String { + if quarantine { + "over18=1; _options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D".into() + } else { + "over18=1".into() + } +} + +async fn more_comments(sub: &str, post_id: &str, cursor: &str, quarantine: bool) -> Result { + if cursor.is_empty() || cursor.len() > MAX_UPSTREAM_CURSOR_BYTES { + return Err(ThreadError::Malformed("upstream continuation cursor was invalid".into())); + } + let url = format!("{SVC_BASE}/more-comments/{sub}/t3_{post_id}"); + let csrf = format!("{:032x}", fastrand::u128(..)); + let body = serde_urlencoded::to_string([("cursor", cursor), ("csrf_token", &csrf)]).map_err(|error| ThreadError::Malformed(error.to_string()))?; + request_html(Method::POST, &url, quarantine, Some((csrf, body))).await +} + +async fn request_html(method: Method, url: &str, quarantine: bool, form: Option<(String, String)>) -> Result { + // Every URL is constructed from SVC_BASE plus validated identifiers. Never + // accept an upstream URL from a request or response. + if !url.starts_with(SVC_BASE) { + return Err(ThreadError::Transport("refused a non-allowlisted upstream host".into())); + } + + let operation = async { + let mut builder = CLIENT.request(method, url).header("Cookie", cookies(quarantine)); + if let Some((csrf, body)) = form { + builder = builder + .header("Cookie", format!("{}; csrf_token={csrf}", cookies(quarantine))) + .header("Content-Type", "application/x-www-form-urlencoded") + .body(body); + } + let response = builder.send().await.map_err(|error| ThreadError::Transport(error.to_string()))?; + validate_response(response).await + }; + + timeout(REQUEST_TIMEOUT, operation) + .await + .map_err(|_| ThreadError::Transport("request exceeded 20 seconds".into()))? +} + +async fn validate_response(response: Response) -> Result { + let status = response.status(); + let retry_after = response + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + + if let Some(error) = classify_status(status.as_u16(), retry_after) { + return Err(error); + } + + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_ascii_lowercase(); + if !content_type.starts_with("text/html") { + return Err(ThreadError::Malformed(format!("unexpected content type {content_type:?}"))); + } + if response + .headers() + .get("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > MAX_RESPONSE_BYTES) + { + return Err(ThreadError::Malformed("response exceeded 8 MiB".into())); + } + + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| ThreadError::Transport(error.to_string()))?; + if bytes.len().saturating_add(chunk.len()) > MAX_RESPONSE_BYTES { + return Err(ThreadError::Malformed("response exceeded 8 MiB".into())); + } + bytes.extend_from_slice(&chunk); + } + String::from_utf8(bytes).map_err(|_| ThreadError::Malformed("response was not UTF-8".into())) +} + +fn classify_status(status: u16, retry_after: Option) -> Option { + match status { + 200..=299 => None, + 401 => Some(ThreadError::InvalidAnonymousState), + 403 => Some(ThreadError::UpstreamBlocked), + 404 => Some(ThreadError::Missing), + 429 => Some(ThreadError::RateLimited(retry_after)), + 451 => Some(ThreadError::Restricted("restricted".into())), + 300..=399 => Some(ThreadError::Transport("Reddit returned a redirect; redirects are disabled".into())), + _ => Some(ThreadError::Transport(format!("Reddit returned HTTP {status}"))), + } +} + +fn listing(children: Vec) -> Value { + json_value!({ "kind": "Listing", "data": { "children": children } }) +} + +fn parse_comments(html: &str, post_id: &str, sort: &str) -> Result, ThreadError> { + validate_fragment_structure(html)?; + let document = Html::parse_fragment(html); + let post_fullname = format!("t3_{post_id}"); + let mut items: Vec<(String, Value)> = Vec::new(); + let mut seen = HashSet::new(); + + for element in document.select(&COMMENT) { + let attr = |name: &str| element.value().attr(name).unwrap_or_default(); + let fullname = attr("thingid"); + let Some(id) = fullname.strip_prefix("t1_").filter(|id| valid_base36_id(id)) else { + continue; + }; + if !seen.insert(fullname.to_owned()) { + continue; + } + let parent = element.value().attr("parentid").unwrap_or(&post_fullname); + if parent != post_fullname && !parent.strip_prefix("t1_").is_some_and(valid_base36_id) { + continue; + } + + let deleted = element.value().attr("is-comment-deleted").is_some(); + let author = if deleted { + "[deleted]" + } else { + let author = attr("author"); + if author.is_empty() { + continue; + } + author + }; + let body = if deleted { + "

[deleted]

".to_owned() + } else { + let Some(body) = Selector::parse(&format!(r#"[id="{fullname}-comment-rtjson-content"]"#)) + .ok() + .and_then(|selector| element.select(&selector).next()) + else { + continue; + }; + clean_html(body) + }; + let created = DateTime::parse_from_str(attr("created"), "%Y-%m-%dT%H:%M:%S%.f%z").map_or(0, |time| time.timestamp()); + + items.push(( + parent.to_owned(), + json_value!({ + "kind": "t1", + "data": { + "id": id, + "name": fullname, + "author": author, + "body_html": format!("
{body}
"), + "created_utc": created, + "score": attr("score").parse::().unwrap_or_default(), + "parent_id": parent, + "replies": "", + } + }), + )); + } + + for element in document.select(&MORE) { + let source = element.value().attr("src").unwrap_or_default(); + let Some(cursor) = element.select(&CURSOR).next().and_then(|input| input.value().attr("value")) else { + continue; + }; + if cursor.is_empty() || cursor.len() > MAX_UPSTREAM_CURSOR_BYTES { + continue; + } + + if source.contains("top-level=1") { + let count = param(source, "comments-remaining").and_then(|count| count.parse::().ok()).unwrap_or_default(); + let token = encode_continuation(post_id, sort, cursor)?; + items.push(( + post_fullname.clone(), + json_value!({ + "kind": "more_comments", + "data": { "id": token, "count": count, "parent_id": post_fullname, "continuation_sort": sort } + }), + )); + continue; + } + + let Some(parent) = general_purpose::STANDARD + .decode(cursor) + .ok() + .and_then(|bytes| String::from_utf8(bytes).ok()) + .and_then(|decoded| { + decoded + .rsplit(':') + .next() + .filter(|parent| parent.strip_prefix("t1_").is_some_and(valid_base36_id)) + .map(str::to_owned) + }) + else { + continue; + }; + let count = element + .select(&MORE_COUNT) + .next() + .and_then(|number| number.value().attr("number")) + .and_then(|number| number.parse::().ok()) + .unwrap_or_default(); + items.push((parent.clone(), json_value!({ "kind": "more", "data": { "id": "", "count": count, "parent_id": parent } }))); + } + + let names: HashSet = items.iter().filter_map(|(_, item)| item["data"]["name"].as_str().map(str::to_owned)).collect(); + let mut children: HashMap> = HashMap::new(); + let mut top = Vec::new(); + for (parent, item) in items { + if names.contains(&parent) { + children.entry(parent).or_default().push(item); + } else { + top.push(item); + } + } + Ok(top.into_iter().map(|item| attach_replies(item, &mut children)).collect()) +} + +fn validate_fragment_structure(html: &str) -> Result<(), ThreadError> { + let normalized = html.to_ascii_lowercase(); + let has_comment_data = normalized.contains(">) -> Value { + let name = item["data"]["name"].as_str().unwrap_or_default().to_owned(); + if let Some(replies) = children.remove(&name) { + item["data"]["replies"] = listing(replies.into_iter().map(|reply| attach_replies(reply, children)).collect()); + } + item +} + +fn count_kind(items: &[Value], expected: &str) -> usize { + items + .iter() + .map(|item| usize::from(item["kind"] == expected) + item["data"]["replies"]["data"]["children"].as_array().map_or(0, |replies| count_kind(replies, expected))) + .sum() +} + +fn encode_continuation(thread: &str, sort: &str, cursor: &str) -> Result { + let state = ContinuationState { + v: CONTINUATION_VERSION, + thread: thread.to_owned(), + sort: sort.to_owned(), + cursor: cursor.to_owned(), + }; + let bytes = serde_json::to_vec(&state).map_err(|error| ThreadError::Malformed(error.to_string()))?; + Ok(general_purpose::URL_SAFE_NO_PAD.encode(bytes)) +} + +fn decode_continuation(token: &str, thread: &str, sort: &str) -> Result { + if token.is_empty() || token.len() > MAX_CURSOR_TOKEN_BYTES { + return Err(ThreadError::Malformed("continuation token was invalid".into())); + } + let bytes = general_purpose::URL_SAFE_NO_PAD + .decode(token) + .map_err(|_| ThreadError::Malformed("continuation token was not valid base64url".into()))?; + let state: ContinuationState = serde_json::from_slice(&bytes).map_err(|_| ThreadError::Malformed("continuation token was not valid JSON".into()))?; + if state.v != CONTINUATION_VERSION || state.thread != thread || state.sort != sort || state.cursor.is_empty() || state.cursor.len() > MAX_UPSTREAM_CURSOR_BYTES { + return Err(ThreadError::Malformed("continuation token did not match this thread and sort".into())); + } + Ok(state) +} + +const ALLOWED_TAGS: &[&str] = &[ + "p", + "br", + "hr", + "a", + "em", + "strong", + "b", + "i", + "del", + "s", + "sup", + "sub", + "code", + "pre", + "blockquote", + "ul", + "ol", + "li", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "table", + "thead", + "tbody", + "tr", + "th", + "td", + "span", + "img", + "figure", +]; +const VOID_TAGS: &[&str] = &["br", "hr", "img"]; + +/// Rebuild comment markup rather than trusting upstream HTML. Event handlers, +/// scripts, tracking wrappers, unsafe URL schemes, and unknown attributes are +/// never copied. +fn clean_html(element: ElementRef<'_>) -> String { + let mut output = String::new(); + for child in element.children() { + match child.value() { + Node::Text(text) => output.push_str(&htmlescape::encode_minimal(text)), + Node::Element(node) => { + let tag = node.name(); + if matches!(tag, "script" | "style" | "svg" | "iframe" | "object" | "embed" | "faceplate-tracker" | "shreddit-analytics") { + continue; + } + let inner = ElementRef::wrap(child).map(clean_html).unwrap_or_default(); + match tag { + "shreddit-spoiler" => output.push_str(&format!("{inner}")), + "shreddit-player" => { + let source = safe_media_url(node.attr("src")); + if !source.is_empty() { + let controls = if node.attr("gif").is_some() { "autoplay loop muted playsinline" } else { "controls" }; + output.push_str(&format!("")); + } + } + _ if !ALLOWED_TAGS.contains(&tag) => output.push_str(&inner), + _ => { + output.push('<'); + output.push_str(tag); + for name in ["href", "src", "alt", "title", "align"] { + let Some(raw_value) = node.attr(name) else { continue }; + let value = match name { + "href" => safe_link_url(Some(raw_value)), + "src" if tag == "img" => safe_media_url(Some(raw_value)), + "src" => String::new(), + _ => htmlescape::encode_minimal(raw_value), + }; + if !value.is_empty() { + output.push_str(&format!(" {name}=\"{value}\"")); + } + } + if tag == "img" { + output.push_str(" loading=\"lazy\""); + } + output.push('>'); + if !VOID_TAGS.contains(&tag) { + output.push_str(&inner); + output.push_str("'); + } + } + } + } + _ => {} + } + } + output +} + +fn safe_link_url(value: Option<&str>) -> String { + let value = value.unwrap_or_default().trim(); + if value.starts_with('/') && !value.starts_with("//") { + return htmlescape::encode_minimal(&format_url(value)); + } + let Ok(parsed) = url::Url::parse(value) else { + return String::new(); + }; + if !matches!(parsed.scheme(), "http" | "https") { + return String::new(); + } + htmlescape::encode_minimal(&format_url(value)) +} + +fn safe_media_url(value: Option<&str>) -> String { + let safe = safe_link_url(value); + // Reddit-owned media is rewritten by format_url to a local proxy route. + // Refuse an external src so viewing a comment cannot contact a third party. + if safe.starts_with('/') && !safe.starts_with("//") { + safe + } else { + String::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE: &str = r#" + +

Hi r/rust & bad

+ +

reply

+ + + +

must not leak

 +

drop me

 +

secret

 + +"#; + + #[test] + fn parses_thread_paths_and_supported_sorts() { + for sort in ["confidence", "top", "new", "controversial", "old"] { + let request = parse_request(&format!("/r/rust/comments/1xaxcyl/title/.json?sort={sort}&raw_json=1")).unwrap(); + assert_eq!(request.post_id, "1xaxcyl"); + assert_eq!(request.comment_id, None); + assert_eq!(request.sort, sort); + } + + let request = parse_request("/r/rust/comments/1xaxcyl/title/19re6td.json?sort=bogus&context=3").unwrap(); + assert_eq!(request.comment_id.as_deref(), Some("19re6td")); + assert_eq!(request.sort, "confidence"); + assert!(parse_request("/r/rust/hot.json").is_err()); + assert!(parse_request("/r/rust/comments/../title.json").is_err()); + } + + #[test] + fn parses_nested_deleted_more_and_sanitized_html() { + let comments = parse_comments(SAMPLE, "p", "new").unwrap(); + assert_eq!(comments.len(), 4); + let first = &comments[0]["data"]; + assert_eq!(first["author"], "alice"); + assert_eq!(first["created_utc"], 1_792_515_653_i64); + assert_eq!(first["body_html"], r#"

Hi r/rust & bad

"#); + let replies = first["replies"]["data"]["children"].as_array().unwrap(); + assert_eq!(replies.len(), 2); + assert_eq!(replies[0]["data"]["author"], "bob"); + assert_eq!(replies[1]["kind"], "more"); + assert_eq!(comments[1]["data"]["body_html"], r#"

[deleted]

"#); + assert!(!serde_json::to_string(&comments).unwrap().contains("drop me")); + let media = comments[2]["data"]["body_html"].as_str().unwrap(); + assert!(media.contains("md-spoiler-text")); + assert!(media.contains("/preview/pre/x.jpeg")); + assert!(!media.contains("tracker.example")); + assert!(!media.contains("iframe")); + assert_eq!(comments[3]["kind"], "more_comments"); + } + + #[test] + fn continuation_is_opaque_and_bound_to_thread_and_sort() { + let token = encode_continuation("abc", "new", "upstream-cursor").unwrap(); + assert!(!token.contains("upstream-cursor")); + assert_eq!(decode_continuation(&token, "abc", "new").unwrap().cursor, "upstream-cursor"); + assert!(decode_continuation(&token, "other", "new").is_err()); + assert!(decode_continuation(&token, "abc", "top").is_err()); + } + + #[test] + fn empty_discussion_is_valid_and_has_no_continuation() { + let comments = parse_comments("
No comments yet
", "abc", "confidence").unwrap(); + assert!(comments.is_empty()); + let data = ThreadData { + post: json_value!({}), + comments: listing(comments), + continuations: 0, + }; + assert_eq!(data.continuation_count(), 0); + } + + #[test] + fn rejects_html_without_a_comment_payload() { + assert!(parse_comments("", "abc", "confidence").is_err()); + assert!(parse_comments("

Access denied

", "abc", "confidence").is_err()); + } + + #[test] + fn rejects_unsafe_url_schemes_and_external_media() { + for value in ["javascript:alert(1)", "data:text/html,bad", "vbscript:bad", "//evil.example/x"] { + assert!(safe_link_url(Some(value)).is_empty()); + } + assert_eq!(safe_link_url(Some("https://example.com/path?a=1&b=2")), "https://example.com/path?a=1&b=2"); + assert!(safe_media_url(Some("https://example.com/image.png")).is_empty()); + } + + #[test] + fn classifies_redirects_expiry_blocks_missing_and_rate_limits() { + assert_eq!(classify_status(200, None), None); + assert_eq!(classify_status(401, None), Some(ThreadError::InvalidAnonymousState)); + assert_eq!(classify_status(403, None), Some(ThreadError::UpstreamBlocked)); + assert_eq!(classify_status(404, None), Some(ThreadError::Missing)); + assert_eq!(classify_status(429, Some(17)), Some(ThreadError::RateLimited(Some(17)))); + assert!(matches!(classify_status(302, None), Some(ThreadError::Transport(message)) if message.contains("redirect"))); + } +} diff --git a/src/utils.rs b/src/utils.rs index b7e1801cb..1832915ed 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -485,6 +485,7 @@ pub struct Comment { pub collapsed: bool, pub is_filtered: bool, pub more_count: i64, + pub continuation_sort: String, pub prefs: Preferences, } diff --git a/static/style.css b/static/style.css index c6dada03e..51c3b7930 100644 --- a/static/style.css +++ b/static/style.css @@ -1131,6 +1131,14 @@ a.search_subreddit:hover { margin: auto; } +.comment_body video { + display: block; + width: auto; + height: auto; + max-width: 100%; + max-height: 512px; +} + .post_media_image.short svg, .post_media_image.short img { width: auto; diff --git a/templates/comment.html b/templates/comment.html index 3c008c78a..3c7ec383e 100644 --- a/templates/comment.html +++ b/templates/comment.html @@ -2,6 +2,8 @@ {% if kind == "more" && parent_kind == "t1" %} → More replies ({{ more_count }}) +{% else if kind == "more_comments" %} +→ More comments{% if more_count > 0 %} ({{ more_count }}){% endif %} {% else if kind == "t1" %}