Skip to content

Post the login form from the sandbox and replay every collection body #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
OmerFarukOruc wants to merge 1 commit into from
Closed

Post the login form from the sandbox and replay every collection body #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion clients/akamai/src/discover.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ static CHALLENGE: LazyLock<Regex> = LazyLock::new(|| {
});

static SEGMENT: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^[A-Za-z0-9_-]{1,24}$").expect("segment pattern"));
LazyLock::new(|| Regex::new(r"^[A-Za-z0-9_-]{1,64}$").expect("segment pattern"));

const MARK: &str = "aeiouy13579";

Expand Down Expand Up @@ -355,4 +355,31 @@ mod tests {
assert!(surface.challenge_page);
assert!(!surface.is_protected());
}

#[test]
fn eventim_obfuscated_path_with_a_27_char_segment_is_the_sensor() {
let html = r#"
<html><head>
<script src="/akam/13/334dc7ea"></script>
<script src="/3PNuxeTv-7Er-OkJZQeZ2anaH-E/DYf3rVYzubX34L3b/SAsPOw/J0/MAZlIVMUAB"></script>
<script src="/3PNuxeTv-7Er-OkJZQeZ2anaH-E/Qwf3rVYzub/ZTkTOw/LX/cIdnhuansX"></script>
</head></html>
"#;
let surface = discover(html, "https://www.eventim.de/");
let sensor = surface.sensor.expect("sensor");

assert_eq!(sensor.kind, Kind::Obfuscated);
assert!(
sensor
.url
.ends_with("/3PNuxeTv-7Er-OkJZQeZ2anaH-E/DYf3rVYzubX34L3b/SAsPOw/J0/MAZlIVMUAB")
);

let obfuscated: Vec<_> = surface
.scripts
.iter()
.filter(|script| script.kind == Kind::Obfuscated)
.collect();
assert_eq!(obfuscated.len(), 2, "{obfuscated:?}");
}
}
140 changes: 110 additions & 30 deletions clients/akamai/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ use serde::Deserialize;
use serde_json::{Map, Value, json};

use wre_client::client::{Client, Registration};
use wre_client::context::{Call, Ctx, FetchRequest, HttpOptions, Jar};
use wre_client::context::{
Call, Ctx, FetchRequest, HttpOptions, Jar, SetCookie, parse_set_cookie,
};
use wre_client::error::{ClientError, ClientResult};
use wre_client::shape::{Shape, field};
use wre_client::spec::{Capabilities, ClientDescriptor, Concurrency, OpSpec};
Expand Down Expand Up @@ -227,6 +229,18 @@ pub fn describe() -> ClientDescriptor {
field("kind", Shape::optional(Shape::Str)).summary(
"form sends it the way the page submits a form, otherwise it goes as an XHR",
),
field("redirects", Shape::optional(Shape::Int)).summary(
"Follow this many redirects. 0 returns the first hop. Default is the session client policy",
),
field("omit_referer", Shape::optional(Shape::Bool)).summary(
"Do not attach Referer from the session page url",
),
field("omit_origin", Shape::optional(Shape::Bool)).summary(
"Do not attach Origin, which the session otherwise derives from the page url",
),
field("origin", Shape::optional(Shape::Str)).summary(
"Origin header sent as given. Pass the literal string null for an opaque origin",
),
],
),
Shape::object(
Expand All @@ -236,6 +250,9 @@ pub fn describe() -> ClientDescriptor {
field("url", Shape::Str).summary("Url after redirects"),
field("headers", Shape::Json)
.summary("Response headers as name and value pairs, without set-cookie"),
field("set_cookies", Shape::Json).summary(
"One entry per set-cookie line, with its name, domain, path, same_site, secure and http_only",
),
field("body", Shape::Str),
field("cookies", Shape::Json).summary("Jar after the request"),
field("refused", Shape::Bool)
Expand Down Expand Up @@ -298,8 +315,12 @@ pub fn describe() -> ClientDescriptor {
"cookies",
Shape::object(
"CookiesInput",
[field("set", Shape::optional(Shape::Str))
.summary("A Cookie header to seed the jar with before reading it back")],
[
field("set", Shape::optional(Shape::Str))
.summary("A Cookie header to seed the jar with before reading it back"),
field("url", Shape::optional(Shape::Str))
.summary("Match or seed cookies for this url, defaults to page_url"),
],
),
Shape::object(
"Cookies",
Expand Down Expand Up @@ -478,6 +499,9 @@ fn config_shape() -> Shape {
.summary("Text typed into the page's first text field. Empty picks a short word \
from the session seed")
.with_default(json!("")),
field("typed_password", Shape::Str)
.summary("Value written into the page's password field before the submit click")
.with_default(json!("")),
field("live_xhr", Shape::Bool)
.summary("Let the sensor's own requests leave the sandbox, which is what the \
script does in a browser. Turning it off has the host post what the \
Expand Down Expand Up @@ -548,6 +572,8 @@ struct Config {
warp: f64,
#[serde(default = "default_typed")]
typed: String,
#[serde(default = "default_typed")]
typed_password: String,
#[serde(default)]
keep_payloads: bool,
#[serde(default = "default_rounds")]
Expand Down Expand Up @@ -697,6 +723,7 @@ impl Akamai {
load_posts_ms: self.config.load_posts_ms,
warp: self.config.warp,
typed: self.config.typed.clone(),
typed_password: self.config.typed_password.clone(),
}
}

Expand Down Expand Up @@ -828,6 +855,34 @@ fn escape(text: &str) -> String {
out
}

/// A boolean request parameter, off unless the caller asked for it.
fn flag(params: &Value, name: &str) -> bool {
params.get(name).and_then(Value::as_bool).unwrap_or(false)
}

/// The Origin header to send, or `None` to leave it off.
///
/// A caller can name one, suppress it, or let it fall out of the page url the
/// way a browser would for a request that carries a body.
fn origin_for(params: &Value, method: &str, page_url: &str) -> Option<String> {
if flag(params, "omit_origin") {
return None;
}

if let Some(given) = params.get("origin").and_then(Value::as_str) {
return Some(given.to_string());
}

if method == "GET" {
return None;
}

let parsed = url::Url::parse(page_url).ok()?;
let host = parsed.host_str()?;

Some(format!("{}://{}", parsed.scheme(), host))
}

fn refused(status: u16, body: &str) -> bool {
if status == 403 || status == 429 {
return true;
Expand Down Expand Up @@ -925,19 +980,31 @@ impl Client for Akamai {
.collect();

if post {
for round in 0..rounds {
// Replay what the script itself posted first, keeping each body's
// own framing, then top up with fresh payloads until the run has
// made the number of posts it was asked for.
let captured = session.collection_requests();
let total = rounds.max(1).max(captured.len());

for round in 0..total {
call.check()?;

let payload = if round == 0 {
first.clone()
} else {
session.nudge(1500.0)?;
session.payload()?.unwrap_or_else(|| first.clone())
let sent = match captured.get(round) {
Some(request) => session.replay(request)?,
None => {
let payload = if round == 0 {
first.clone()
} else {
session.nudge(1500.0)?;
session.payload()?.unwrap_or_else(|| first.clone())
};

session.post_payload(&payload, endpoint.as_deref())?
}
};

let sent = session.post_payload(&payload, endpoint.as_deref())?;
posts.push(serde_json::to_value(&sent).unwrap_or(Value::Null));
call.progress(round as u64 + 1, rounds as u64, "posted");
call.progress(round as u64 + 1, total as u64, "posted");
}
}

Expand Down Expand Up @@ -1051,41 +1118,41 @@ impl Client for Akamai {
let header = if telemetry { session.telemetry()? } else { None };

let submitted = params.get("kind").and_then(Value::as_str) == Some("form");
let plan = session::plan(&user_agent, submitted);

let order = if submitted { &session::FORM_ORDER[..] } else { &session::XHR_ORDER[..] };
let redirects = params
.get("redirects")
.and_then(Value::as_u64)
.map(|value| value as usize);

let mut request = FetchRequest {
url: url.clone(),
method,
headers: Vec::new(),
body,
fingerprint: None,
order: order.iter().map(|name| name.to_string()).collect(),
order: plan.order.iter().map(|name| name.to_string()).collect(),
redirects,
};

request = request
.header("accept-language", languages)
.header("user-agent", user_agent);

if submitted {
for (name, value) in session::FORM {
request = request.header(name, value);
}
} else {
for (name, value) in plan.rows {
request = request.header(*name, *value);
}

if !submitted {
request = request.header("accept", "*/*");
}

if !session.page_url().is_empty() {
if !flag(&params, "omit_referer") && !session.page_url().is_empty() {
request = request.header("referer", session.page_url().to_string());
}

if request.method != "GET" {
if let Ok(parsed) = url::Url::parse(session.page_url()) {
if let Some(host) = parsed.host_str() {
let origin = format!("{}://{}", parsed.scheme(), host);
request = request.header("origin", origin);
}
}
}
if let Some(origin) = origin_for(&params, &request.method, session.page_url()) {
request = request.header("origin", origin);
}

if !submitted && params.get("form").and_then(Value::as_object).is_some() {
Expand Down Expand Up @@ -1117,6 +1184,8 @@ impl Client for Akamai {

let response = session.fetch(request)?;
let text = response.text();
let set_cookies: Vec<SetCookie> =
response.set_cookies().into_iter().filter_map(parse_set_cookie).collect();

Ok(json!({
"status": response.status,
Expand All @@ -1127,6 +1196,7 @@ impl Client for Akamai {
.filter(|(name, _)| !name.eq_ignore_ascii_case("set-cookie"))
.map(|(name, value)| json!([name, value]))
.collect::<Vec<_>>(),
"set_cookies": set_cookies,
"body": text,
"cookies": session.cookies(),
"refused": refused(response.status, &text),
Expand Down Expand Up @@ -1178,6 +1248,7 @@ impl Client for Akamai {
body,
fingerprint: None,
order,
redirects: None,
})?;

Ok(json!({
Expand Down Expand Up @@ -1206,9 +1277,13 @@ impl Client for Akamai {
}

"cookies" => {
if let Some(header) = params.get("set").and_then(Value::as_str) {
let url = self.config.page_url.clone().unwrap_or_default();
let scope = params.get("url").and_then(Value::as_str).map(str::to_string);
let url = scope
.clone()
.or_else(|| self.config.page_url.clone())
.unwrap_or_default();

if let Some(header) = params.get("set").and_then(Value::as_str) {
for pair in header.split(';') {
let trimmed = pair.trim();
if trimmed.is_empty() {
Expand All @@ -1219,7 +1294,12 @@ impl Client for Akamai {
}

let session = self.session()?;
let pairs = session.cookie_pairs();
let pairs = match &scope {
// A named url reads the jar the way that url would see it, with
// no url the session's own page decides what a browser sends.
Some(url) => session.cookie_pairs_at(url),
None => session.cookie_pairs(),
};

Ok(json!({
"header": pairs
Expand Down
Loading