From a199bcbc6985fb79bacfc0f5e9d8521e4671be08 Mon Sep 17 00:00:00 2001 From: ImBaphomet Date: Tue, 10 Nov 2026 01:10:54 +0100 Subject: [PATCH 01/12] fix(fxserver): take a start for hung once no resource starts for the delay, however long it lasts --- README.md | 13 ++- crates/fake-fxserver/src/main.rs | 6 ++ crates/fake-fxserver/tests/supervisor.rs | 76 +++++++++++++ crates/observer-app/src/changes/job.rs | 9 +- crates/observer-fxserver/src/health.rs | 109 ++++++++++++++++++- crates/observer-fxserver/src/supervisor.rs | 119 +++++++++++++++++++-- web/src/i18n/en.ts | 6 +- 7 files changed, 316 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 28b94ec1..aedf0c12 100644 --- a/README.md +++ b/README.md @@ -162,11 +162,14 @@ hang detection off, only a crash fails it, and a silent server still waits to an Observer watches two signals while FXServer runs: the heartbeat of its bridge resource, which only comes while the main loop turns, and FXServer's HTTP endpoints. Once the server is up, one of them silent for 10 seconds marks it degraded. Silent for the delay under **Server → Hang detection**, 6 -minutes by default and counted from the start until the first answer, the server is hung: Observer -writes a dump of it (a report of its threads on Linux), kills its process tree and starts it again. - -A big server's first start can keep its main loop busy for minutes while resources build their caches of -clothing and vehicles, which reads as a hang. Raise the delay, from 1 to 60 minutes, or turn hang detection +minutes by default, the server is hung: Observer writes a dump of it (a report of its threads on Linux), +kills its process tree and starts it again. + +A start gives no heartbeat: FXServer runs its cfg before any thread of a resource, the bridge's included. +Until the cfg has run, each resource FXServer starts counts in its place, so a start is hung once it has +started no resource for the delay, however long it takes in all, and the console names the last resource +it started. One resource alone can take minutes on a big server's first start, such as a clothing pack +whose files are read for the first time. Raise the delay, from 1 to 60 minutes, or turn hang detection off: FXServer is then left running however long it stays silent, until someone restarts it. Either change applies at once, to a server already starting too, and the delay then counts from the change: turning detection back on never kills a server that was silent before. A crash is another matter: whenever diff --git a/crates/fake-fxserver/src/main.rs b/crates/fake-fxserver/src/main.rs index 09af6683..74a3b30c 100644 --- a/crates/fake-fxserver/src/main.rs +++ b/crates/fake-fxserver/src/main.rs @@ -13,6 +13,9 @@ //! - `fake_quit_delay_ms` (500): time between `quit` and the end, which is what FXServer 35245 does: //! exit code 1 on Windows, SIGSEGV on Linux; //! - `fake_ignore_quit 1`: `quit` does nothing; +//! - `fake_start_ms ` (0): how long each resource takes to start. FXServer 38435 prints +//! `Started resource` once a resource is up and nothing while it reads its stream files, which +//! took a clothing pack minutes on a first start; //! - `fake_heartbeat_ms` (3000): the bridge heartbeat period; //! - `fake_heap_ms` (10000): how often the bridge reports the JS heap; //! - `fake_players `: `/dynamic.json` counts `n` clients and `/players.json` lists ids 1 to `n` @@ -456,6 +459,9 @@ impl Fake { fn start(&mut self, resource: &str) { console::line(SCRIPTING, &format!("Creating script environments for {resource}")); + if let Some(ms) = self.server.number("fake_start_ms") { + thread::sleep(Duration::from_millis(ms)); + } console::line(RESOURCES, &format!("Started resource {resource}")); if !self.started.iter().any(|started| started == resource) { self.started.push(resource.to_owned()); diff --git a/crates/fake-fxserver/tests/supervisor.rs b/crates/fake-fxserver/tests/supervisor.rs index 5a187fb7..252dc512 100644 --- a/crates/fake-fxserver/tests/supervisor.rs +++ b/crates/fake-fxserver/tests/supervisor.rs @@ -856,6 +856,82 @@ async fn a_hung_main_loop_is_detected_and_the_server_restarted() { lab.shut_down(false).await; } +/// Adds `lines` to the lab's cfg, with a folder for each resource they start. +fn extend_cfg(lab: &Lab, resources: &[&str], lines: &[&str]) { + let data = lab.dir.path().join("server-data"); + for resource in resources { + let folder = data.join("resources").join(resource); + fs::create_dir_all(&folder).unwrap(); + fs::write( + folder.join("fxmanifest.lua"), + "fx_version 'cerulean'\ngame 'gta5'\n", + ) + .unwrap(); + } + let cfg = data.join("server.cfg"); + let added: String = resources + .iter() + .map(|resource| format!("ensure {resource}\n")) + .chain(lines.iter().map(|line| format!("{line}\n"))) + .collect(); + fs::write(&cfg, fs::read_to_string(&cfg).unwrap() + &added).unwrap(); +} + +#[tokio::test] +async fn a_start_longer_than_the_hang_delay_goes_online_while_it_keeps_starting_resources() { + // Eight resources of 600 ms each: a start of five seconds, under limits that take a server + // silent for two for hung. + let mut lab = Lab::new(&[("fake_start_ms", "600")]).await; + extend_cfg( + &lab, + &[ + "maps", "clothes", "vehicles", "weapons", "props", "peds", "sounds", "core", + ], + &[], + ); + + lab.supervisor.start("test").await.unwrap(); + + assert_eq!(lab.state(State::Online).await, Cause::Ready); + assert!( + !lab.seen.iter().any(hung_or_exited), + "the start was taken for a hang: {:#?}", + lab.seen + ); + assert_eq!(lab.generation(), 1); + lab.console_until("Started resource core").await; + lab.shut_down(false).await; +} + +#[tokio::test] +async fn a_start_that_stops_starting_resources_is_hung_and_says_where_it_stopped() { + let mut lab = Lab::new(&[("fake_start_ms", "300")]).await; + // The cfg sticks after its second resource, before any thread of the bridge runs. + extend_cfg(&lab, &["maps", "clothes"], &["fake_hang"]); + + lab.supervisor.start("test").await.unwrap(); + + assert_eq!( + lab.state(State::Hung).await, + Cause::Hang { + cause: HangCause::NoHeartbeat + } + ); + let stopped = lab.console_until("Started resource clothes").await; + let note = lab + .console_until("its start is stuck: no resource started since clothes, ") + .await; + assert!(stopped < note); + assert_eq!( + lab.state(State::Crashed).await.clone(), + Cause::Hang { + cause: HangCause::NoHeartbeat + } + ); + lab.supervisor.stop("test").await.ok(); + lab.shut_down(false).await; +} + /// Launch settings with hang detection off; the lab's own launch keeps it on. fn without_hang_detection(settings: &mut Settings) { if let Some(launch) = &mut settings.launch { diff --git a/crates/observer-app/src/changes/job.rs b/crates/observer-app/src/changes/job.rs index 5e0fe08e..5d44e2d0 100644 --- a/crates/observer-app/src/changes/job.rs +++ b/crates/observer-app/src/changes/job.rs @@ -31,8 +31,9 @@ use super::plan::{self, Chain, ListingInput, ListingPass, Planned, Read, Slot}; use super::{SERVER, Shared}; use crate::Error; -/// Hashing waits for the server to be online, or this long after the spawn. -const ONLINE_WAIT: Duration = Duration::from_mins(3); +/// Hashing waits for the server to be online, or this long after the spawn: a start reads the same +/// files, and a big server's first one takes minutes that hashing beside it only adds to. +const ONLINE_WAIT: Duration = Duration::from_mins(30); /// Below this much free space on the data folder's volume, digests only are kept. const DISK_FLOOR: u64 = 2_000_000_000; /// Files read in one go of the hashing pass, between which progress is written. @@ -60,7 +61,7 @@ pub(crate) enum Ended { } /// Takes the snapshot of a start: the listing at once, the hashing once `status` says the run is -/// online, or three minutes later. +/// online, or half an hour later. pub(crate) async fn start( shared: Arc, settings: ChangesSettings, @@ -517,7 +518,7 @@ impl Work { Ok(()) } - /// Waits for the run to be online, for three minutes at most. A run that ended frees the disk, + /// Waits for the run to be online, for half an hour at most. A run that ended frees the disk, /// so hashing starts at once. async fn wait_online( &self, diff --git a/crates/observer-fxserver/src/health.rs b/crates/observer-fxserver/src/health.rs index c45fde8f..3f322334 100644 --- a/crates/observer-fxserver/src/health.rs +++ b/crates/observer-fxserver/src/health.rs @@ -1,7 +1,8 @@ //! How Observer tells a running FXServer from a stuck one: its HTTP endpoints, answered off the //! main thread, the main thread's tick count in `/perf/`, and the bridge's heartbeat, which only -//! comes while the main loop runs. The same probes feed the statistics: `/perf/` and -//! `/players.json` are read in the middle of each five-second slot, once for both. +//! comes while the main loop runs. Until the cfg has run, the resources FXServer starts stand in for +//! the heartbeat. The same probes feed the statistics: `/perf/` and `/players.json` are read in the +//! middle of each five-second slot, once for both. use std::net::SocketAddr; use std::time::{Duration, Instant}; @@ -127,6 +128,10 @@ pub enum Verdict { pub struct Health { spawned: Instant, heartbeat: Option, + /// When the bridge first reported in, which it does once the cfg has run. + cfg_ran: Option, + /// When FXServer last started a resource of its cfg. + progress: Option, http: Option, /// The last `svMain` tick count, and whether it moved since the read before. ticks: Option, @@ -145,6 +150,8 @@ impl Health { Self { spawned, heartbeat: None, + cfg_ran: None, + progress: None, http: None, ticks: None, stalled: false, @@ -165,10 +172,23 @@ impl Health { } pub fn heartbeat(&mut self, now: Instant) { + self.cfg_ran.get_or_insert(now); self.heartbeat = Some(now); self.bark = false; } + /// FXServer started a resource at `at`. It runs its cfg before any thread of the bridge, so no + /// heartbeat comes during a start however long it takes: a resource started is what shows the + /// start going on. + pub fn progressed(&mut self, at: Instant) { + self.progress = self.progress.max(Some(at)); + } + + /// Whether the bridge has reported in since the spawn: the cfg has run. + pub fn cfg_ran(&self) -> bool { + self.cfg_ran.is_some() + } + pub fn bark(&mut self) { self.bark = true; } @@ -210,10 +230,19 @@ impl Health { .armed .map_or(Duration::MAX, |armed| now.saturating_duration_since(armed)); if now.saturating_duration_since(self.spawned) >= limits.boot_grace { - if heartbeat.min(armed) >= limits.heartbeat_fatal { + // A start is stuck once it starts no resource for that long, not once it took that long. + let stalled = match self.cfg_ran { + Some(_) => heartbeat, + None => silent(self.progress), + }; + if stalled.min(armed) >= limits.heartbeat_fatal { return Verdict::Hung(HangCause::NoHeartbeat); } - if http.min(armed) >= limits.http_fatal { + // The cfg opens the HTTP endpoint when it comes to that line: HTTP is owed once it ran. + let unanswered = self + .cfg_ran + .map(|ran| silent(Some(self.http.map_or(ran, |http| http.max(ran))))); + if unanswered.is_some_and(|unanswered| unanswered.min(armed) >= limits.http_fatal) { return Verdict::Hung(HangCause::NoHttp); } } @@ -699,6 +728,78 @@ mod tests { ); } + #[test] + fn a_start_is_hung_once_it_starts_no_resource_for_the_delay_however_long_it_takes() { + let limits = HealthLimits::default().under(&launch(true, 360)); + let start = Instant::now(); + let mut health = Health::new(start); + // Twenty minutes of resources, the slowest one five minutes on a cold cache. + for second in [15, 120, 420, 700, 1000, 1200] { + health.progressed(start + secs(second)); + assert_eq!( + health.verdict(start + secs(second + 299), &limits), + Verdict::Starting, + "{second}" + ); + } + + // Told late, an earlier one takes nothing back: the latest counts. + health.progressed(start + secs(900)); + assert_eq!(health.verdict(start + secs(1559), &limits), Verdict::Starting); + assert_eq!( + health.verdict(start + secs(1560), &limits), + Verdict::Hung(HangCause::NoHeartbeat) + ); + } + + #[test] + fn once_the_cfg_has_run_only_the_heartbeat_tells_the_main_loop_runs() { + let limits = HealthLimits::default().under(&launch(true, 360)); + let start = Instant::now(); + let mut health = Health::new(start); + health.probed(&http_ok(), start + secs(10)); + health.progressed(start + secs(300)); + health.heartbeat(start + secs(600)); + assert!(health.cfg_ran()); + for second in (600..1200).step_by(3) { + health.probed(&http_ok(), start + secs(second)); + } + // A resource restarted from the console, which the main loop runs before it sticks. + health.progressed(start + secs(900)); + + assert_eq!( + health.verdict(start + secs(959), &limits), + Verdict::Degraded(vec![Problem::HeartbeatLate]) + ); + assert_eq!( + health.verdict(start + secs(960), &limits), + Verdict::Hung(HangCause::NoHeartbeat) + ); + } + + #[test] + fn http_is_owed_from_the_end_of_the_cfg_which_opens_its_endpoint() { + let limits = HealthLimits::default().under(&launch(true, 360)); + let start = Instant::now(); + let mut health = Health::new(start); + // A long start whose cfg adds the endpoint last: HTTP never answered meanwhile. + for second in (60..=1200).step_by(60) { + health.progressed(start + secs(second)); + assert_eq!(health.verdict(start + secs(second), &limits), Verdict::Starting); + } + health.heartbeat(start + secs(1200)); + for second in (1200..1600).step_by(3) { + health.heartbeat(start + secs(second)); + } + + assert_eq!(health.verdict(start + secs(1201), &limits), Verdict::Starting); + assert_eq!(health.verdict(start + secs(1559), &limits), Verdict::Starting); + assert_eq!( + health.verdict(start + secs(1560), &limits), + Verdict::Hung(HangCause::NoHttp) + ); + } + #[test] fn a_silent_heartbeat_degrades_then_hangs_the_server() { let start = Instant::now(); diff --git a/crates/observer-fxserver/src/supervisor.rs b/crates/observer-fxserver/src/supervisor.rs index 2f6b8b79..e87e68d0 100644 --- a/crates/observer-fxserver/src/supervisor.rs +++ b/crates/observer-fxserver/src/supervisor.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; use std::net::{Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, Mutex, PoisonError, Weak}; use std::time::{Duration, Instant}; use observer_domain::diagnostics::ProcessKind; @@ -769,6 +769,32 @@ fn on_channel<'a>(line: &'a str, channel: &str) -> Option<&'a str> { (name.trim_start() == channel).then_some(text) } +/// The resource a line of FXServer's console says it started, as its `resources` channel prints it +/// once the resource is up, with its warnings counted after the name. +fn started_resource(line: &str) -> Option<&str> { + on_channel(line, "resources")? + .strip_prefix("Started resource ")? + .split_whitespace() + .next() +} + +/// The last resource FXServer said it started in one run, and when: what shows a start going on +/// before the bridge's first report. +#[derive(Debug, Default)] +struct StartProgress { + last: Mutex>, +} + +impl StartProgress { + fn started(&self, resource: &str, at: Instant) { + *self.last.lock().unwrap_or_else(PoisonError::into_inner) = Some((at, resource.to_owned())); + } + + fn last(&self) -> Option<(Instant, String)> { + self.last.lock().unwrap_or_else(PoisonError::into_inner).clone() + } +} + /// One spawn of FXServer, from its start until its tree is empty. struct Run { generation: u32, @@ -787,6 +813,7 @@ struct Run { cancel: CancellationToken, tasks: Vec>, health: Health, + progress: Arc, bridge_ready: bool, ready_at: Option, after: Option, @@ -1501,6 +1528,11 @@ impl Actor { let Some(run) = &mut self.run else { return; }; + if !run.health.cfg_ran() + && let Some((at, _)) = run.progress.last() + { + run.health.progressed(at); + } let verdict = run.health.verdict(now, &limits); let newly_ready = !matches!(verdict, Verdict::Starting | Verdict::Hung(_)) && run.ready_at.is_none(); if newly_ready { @@ -1538,6 +1570,10 @@ impl Actor { if dumping { return; } + let why = match &self.run { + Some(run) if !run.health.cfg_ran() => stalled_text(run.progress.last(), now), + _ => hang_text(cause).to_owned(), + }; if let Some(run) = &mut self.run { run.end(Ending::Hang { cause }); } @@ -1545,15 +1581,13 @@ impl Actor { match self.settings.hang_snapshot { Some(limits) => { self.note(&format!( - "FXServer is not responding ({}); writing {SNAPSHOT} of the hung server before killing it", - hang_text(cause) + "FXServer is not responding ({why}); writing {SNAPSHOT} of the hung server before killing it" )); self.snapshot(limits).await; self.note("killing FXServer's process tree"); } None => self.note(&format!( - "FXServer is not responding ({}); killing its process tree", - hang_text(cause) + "FXServer is not responding ({why}); killing its process tree" )), } self.kill(); @@ -1919,6 +1953,7 @@ impl Actor { .unwrap_or_else(|| SocketAddr::from((Ipv4Addr::LOCALHOST, DEFAULT_PORT))); let cancel = CancellationToken::new(); let (stdin, stdin_task) = StdinWriter::start(spawned.stdin, generation, Arc::clone(&self.console)); + let progress = Arc::new(StartProgress::default()); let mut tasks = vec![ tokio::spawn(read_console( spawned.stdout, @@ -1926,6 +1961,7 @@ impl Actor { generation, Arc::clone(&self.console), self.signals_tx.clone(), + Arc::clone(&progress), )), tokio::spawn(read_console( spawned.stderr, @@ -1933,6 +1969,7 @@ impl Actor { generation, Arc::clone(&self.console), self.signals_tx.clone(), + Arc::clone(&progress), )), tokio::spawn(trace::read( spawned.traces, @@ -1977,6 +2014,7 @@ impl Actor { cancel, tasks, health: Health::new(Instant::now()), + progress, bridge_ready: false, ready_at: None, after: None, @@ -2284,6 +2322,7 @@ async fn read_console( generation: u32, console: Arc, signals: mpsc::Sender<(u32, Signal)>, + progress: Arc, ) { let mut lines = LineAssembler::default(); let publish = |assembled: crate::console::Assembled| { @@ -2293,7 +2332,13 @@ async fn read_console( masked: false, }; let line = console.push(generation, kind, None, sanitize(&assembled.text), flags); - if let Some(signal) = Signal::read(kind, &plain(&line.text)) { + let text = plain(&line.text); + if kind == LineKind::Stdout + && let Some(resource) = started_resource(&text) + { + progress.started(resource, Instant::now()); + } + if let Some(signal) = Signal::read(kind, &text) { let _ = signals.try_send((generation, signal)); } }; @@ -2431,6 +2476,23 @@ fn hang_text(cause: HangCause) -> &'static str { } } +/// Why a start whose cfg never finished is taken for stuck: the last resource it started, and how +/// long ago. +fn stalled_text(last: Option<(Instant, String)>, now: Instant) -> String { + match last { + Some((at, resource)) => { + let seconds = now.saturating_duration_since(at).as_secs(); + let ago = if seconds >= 120 { + format!("{} min", seconds / 60) + } else { + format!("{seconds} s") + }; + format!("its start is stuck: no resource started since {resource}, {ago} ago") + } + None => "its start is stuck: it started no resource".to_owned(), + } +} + fn now_ms() -> i64 { jiff::Timestamp::now().as_millisecond() } @@ -2439,6 +2501,51 @@ fn now_ms() -> i64 { mod tests { use super::*; + #[test] + fn a_started_resource_is_read_from_fxservers_own_channel_only() { + // As FXServer 38435 prints them. + assert_eq!( + started_resource("[ resources] Started resource Pata_BathroomProps"), + Some("Pata_BathroomProps") + ); + assert_eq!( + started_resource("[ resources] Started resource Pata_Christmas2 (3 warnings)"), + Some("Pata_Christmas2") + ); + for line in [ + "[ resources] Scanning resources.", + "[ resources] Found 391 resources.", + "[ c-scripting-core] Creating script environments for prompt_vfd_4bays", + "[ citizen-server-impl] Couldn't start resource Pata_toddler.", + "[resources:Pata_Jap:s] Asset Pata_Jap/pata_jap81.ydr uses 32.0 MiB of physical memory.", + "[ script:chat] [ resources] Started resource fake", + "[ script:chat] Started resource fake", + "Started resource fake", + ] { + assert_eq!(started_resource(line), None, "{line}"); + } + } + + #[test] + fn a_stuck_start_names_the_last_resource_started_and_how_long_ago() { + let started = Instant::now(); + let last = || Some((started, "clothes".to_owned())); + let after = |seconds| started + Duration::from_secs(seconds); + + assert_eq!( + stalled_text(last(), after(371)), + "its start is stuck: no resource started since clothes, 6 min ago" + ); + assert_eq!( + stalled_text(last(), after(45)), + "its start is stuck: no resource started since clothes, 45 s ago" + ); + assert_eq!( + stalled_text(None, after(45)), + "its start is stuck: it started no resource" + ); + } + #[test] fn licence_rejections_are_recognised_on_either_stream() { let lines = [ diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 97ecb259..2412ebac 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -3472,10 +3472,10 @@ export const en = { unhealthy: (problems: string[]) => `Degraded: ${list(problems)}.`, problems: healthProblems, hangNoHeartbeat: - 'Not responding: the observer bridge stopped reporting in, so its main loop is stuck.', + 'Not responding: the observer bridge does not report in. Its main loop is stuck, or its start stopped starting resources.', hangNoHttp: 'Not responding: its HTTP endpoint stopped answering.', hangAdvice: (minutes: number) => - `FXServer stayed silent for ${count(minutes, { one: 'minute', other: 'minutes' })}, the delay set under Server → Hang detection. A first start that builds its caches of clothing and vehicles can take longer: raise the delay there, or turn hang detection off for that start.`, + `FXServer stayed silent for ${count(minutes, { one: 'minute', other: 'minutes' })}, the delay set under Server → Hang detection. While it starts, each resource it starts counts as a sign of life, and the console names the last one. A resource that takes longer than the delay, as a clothing pack can on a first start, needs it raised there, or hang detection turned off for that start.`, openHang: 'Open hang detection', exited: (description: string) => `FXServer exited: ${description}.`, fatalError: (message: string) => `FXServer reported a fatal error: ${message}`, @@ -4379,7 +4379,7 @@ export const en = { }, hangTimeout: { label: 'Wait before restarting', - about: 'Minutes of silence, from 1 to 60, 6 by default. Raise it if a first start is slow while resources build their caches of clothing and vehicles.', + about: 'Minutes of silence, from 1 to 60, 6 by default. While FXServer starts, each resource it starts counts as a sign of life: raise the delay if one resource alone takes longer, as a clothing pack can on a first start.', }, extraArguments: { label: 'Convars', From 2528d060e9ae6776645c576e0a095a55bd6bd701 Mon Sep 17 00:00:00 2001 From: ImBaphomet Date: Tue, 10 Nov 2026 01:10:54 +0100 Subject: [PATCH 02/12] feat: draw Rockstar's map of GTA V under the live map's players, as tiles each browser loads --- README.md | 28 +++++-- crates/observer-app/src/map/tests.rs | 3 +- crates/observer-domain/src/map.rs | 44 +++++++++- crates/observer-store/src/map.rs | 46 +++++++++- crates/observer-web/src/security.rs | 10 ++- crates/observer-web/src/tests.rs | 3 + crates/observer-web/src/tests/map.rs | 2 + web/src/api/generated/MapImagery.ts | 6 ++ web/src/api/generated/MapSettings.ts | 6 +- web/src/i18n/en.ts | 21 ++++- web/src/map/MapCanvas.test.tsx | 47 +++++++++++ web/src/map/MapCanvas.tsx | 120 ++++++++++++++++++++++----- web/src/map/tiles.test.ts | 74 +++++++++++++++++ web/src/map/tiles.ts | 109 ++++++++++++++++++++++++ web/src/pages/MapPage.tsx | 45 ++++++++-- web/src/settings/LiveMap.tsx | 20 ++++- web/src/settings/catalog.ts | 2 + 17 files changed, 534 insertions(+), 52 deletions(-) create mode 100644 web/src/api/generated/MapImagery.ts create mode 100644 web/src/map/tiles.test.ts create mode 100644 web/src/map/tiles.ts diff --git a/README.md b/README.md index aedf0c12..dfbe7e13 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ what to do with them. probation, and a return to the last healthy build. - **Statistics and diagnostics.** Players, tick times, CPU and memory of FXServer and the host, FXServer's threads and connections, crash dumps and hangs. -- **Live map.** Where the connected players stand, over a map image you add, coloured by play time, - a click away from their profile and a ban. +- **Live map.** Where the connected players stand, over Rockstar's map of GTA V or an image you add, + coloured by play time, a click away from their profile and a ban. - **Bans.** A ban holds a player's licence, other identifiers and hardware tokens, for a time or for good. The server refuses a banned player at connection with your message, and drops one banned while playing. @@ -322,6 +322,10 @@ nothing else. | `api.github.com` | Observer's latest release, without an account: 60 checks an hour per address | | `github.com`, `release-assets.githubusercontent.com`, `objects.githubusercontent.com` | a release's archive, its checksums and their signature | +The panel, in the browser, loads everything from Observer, with one exception: the [live map](#live-map) +loads Rockstar's map of GTA V from `s.rsg.sc` unless its settings say **My image** or **Grid**. Observer +itself never talks to that host. + Cfx.re's [Platform License Agreement](https://fivem.net/terms) covers the server builds you download. ### Command line @@ -815,12 +819,20 @@ shape too, and the settings' Live map category moves the thresholds. A click on play time, session, ping and position, a link to their profile and, with **Ban players**, a ban. Seeing the map takes **See the live map**. -Observer ships no map: GTA V's belongs to Rockstar Games. The owner gives, in the settings, the path of -an image on the host they have the right to use, such as GTA V's atlas from their own copy of the game: -PNG, JPEG or WebP, up to 16,384 pixels a side and 64 MB. Observer keeps a copy in its data folder -(`map/`) and serves it only to accounts that see the map. To place it in the game, stand at two spots -you know on the image, far apart, pick yourself in the list and click where you stand, or type the game -coordinates of the image's four edges. Without an image, the page shows a grid of game coordinates. +Under the players, the page draws Rockstar's own map of GTA V, as the game draws it (**Atlas**), seen +from the sky (**Satellite**) or as a road map (**Roads**): the settings' Live map category chooses. +Observer ships no map and keeps none, GTA V's belongs to Rockstar Games: each browser that opens the +page loads the map's tiles from Rockstar's servers (`s.rsg.sc`), which then see that browser's address, +and nothing else of it: no referrer, no cookie of Observer's. It is the one thing the panel loads from +elsewhere than Observer; the two other choices load nothing from outside. + +**My image** is for another map, a custom island or RedM's for one: the owner gives the path of an image +on the host they have the right to use, PNG, JPEG or WebP, up to 16,384 pixels a side and 64 MB. +Observer keeps a copy in its data folder (`map/`) and serves it only to accounts that see the map. To +place it in the game, stand at two spots you know on the image, far apart, pick yourself in the list and +click where you stand, or type the game coordinates of the image's four edges. **Grid** draws a grid of +game coordinates, which also stands in for an image not placed yet and for tiles Rockstar's servers did +not send. Positions come from the bridge, which reads them from OneSync: without OneSync there are none. The server sends them every 1, 2 or 5 seconds, only while someone has the page open, and stops within 30 diff --git a/crates/observer-app/src/map/tests.rs b/crates/observer-app/src/map/tests.rs index 3f09e8f0..1fe77de4 100644 --- a/crates/observer-app/src/map/tests.rs +++ b/crates/observer-app/src/map/tests.rs @@ -1,6 +1,6 @@ use std::net::{IpAddr, SocketAddr}; -use observer_domain::map::MapBounds; +use observer_domain::map::{MapBounds, MapImagery}; use observer_domain::permissions::Permissions; use observer_domain::schedule::{ScheduleSettings, ScheduleStatus}; use tempfile::TempDir; @@ -194,6 +194,7 @@ async fn settings_are_checked_before_they_are_kept() { let lab = Lab::new().await; let (map, task) = lab.map(); let placed = MapSettings { + imagery: MapImagery::Image, bounds: Some(MapBounds { west: -5640.0, east: 6470.0, diff --git a/crates/observer-domain/src/map.rs b/crates/observer-domain/src/map.rs index c709ff5c..0835e738 100644 --- a/crates/observer-domain/src/map.rs +++ b/crates/observer-domain/src/map.rs @@ -1,5 +1,6 @@ -//! The live map: where connected players stand, as the server sees them, drawn over a map image -//! the owner adds. Observer ships no map: GTA V's belongs to Rockstar Games. +//! The live map: where connected players stand, as the server sees them, drawn over Rockstar's map +//! of GTA V, which each browser loads from Rockstar's servers, or over an image the owner adds. +//! Observer ships and keeps no map of its own: GTA V's belongs to Rockstar Games. use jiff::Timestamp; use serde::{Deserialize, Serialize}; @@ -38,11 +39,33 @@ impl MapBounds { } } +/// What the map draws under the players. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub enum MapImagery { + /// Rockstar's map of GTA V as the game draws it. Like the two after it, each browser that + /// opens the map loads it from Rockstar's servers. + #[default] + Atlas, + /// The same map, seen from the sky. + Satellite, + /// The same map, as a road map. + Roads, + /// The image the owner added, once it is placed in the game. + Image, + /// A grid of game coordinates. + Grid, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] #[serde(rename_all = "camelCase", deny_unknown_fields)] #[ts(export)] pub struct MapSettings { - /// `None` until the image is calibrated: the map then shows a grid of game coordinates. + #[serde(default)] + pub imagery: MapImagery, + /// Where the owner's image sits; `None` until it is calibrated: the map then shows a grid of + /// game coordinates in its place. pub bounds: Option, /// A player is new under this many hours of play time. pub new_hours: u32, @@ -55,6 +78,7 @@ pub struct MapSettings { impl Default for MapSettings { fn default() -> Self { Self { + imagery: MapImagery::default(), bounds: None, new_hours: 10, veteran_hours: 100, @@ -157,6 +181,20 @@ mod tests { assert!(MapSettings::default().is_valid()); assert!(valid.is_valid()); + assert_eq!(MapSettings::default().imagery, MapImagery::Atlas); + // Settings sent without it, as a panel loaded before it existed sends them. + let sent: MapSettings = + serde_json::from_str(r#"{"bounds":null,"newHours":10,"veteranHours":100,"everySeconds":2}"#) + .unwrap(); + assert_eq!(sent, MapSettings::default()); + assert_eq!( + serde_json::to_value(MapSettings { + imagery: MapImagery::Satellite, + ..MapSettings::default() + }) + .unwrap()["imagery"], + "satellite" + ); for invalid in [ MapSettings { new_hours: 100, diff --git a/crates/observer-store/src/map.rs b/crates/observer-store/src/map.rs index 2fb95719..dae046f6 100644 --- a/crates/observer-store/src/map.rs +++ b/crates/observer-store/src/map.rs @@ -11,12 +11,23 @@ const IMAGE: &str = "map.image"; pub fn settings(conn: &Connection) -> Result { Ok(settings::get(conn, SETTINGS)? - .and_then(|text| serde_json::from_str::(&text).ok()) + .and_then(|text| read(&text)) // A value edited by hand that the panel could not have saved falls back to the defaults. .filter(MapSettings::is_valid) .unwrap_or_default()) } +/// Settings saved before the map could draw Rockstar's name no imagery: an image their owner +/// placed stays what the map draws. +fn read(text: &str) -> Option { + let mut value: serde_json::Value = serde_json::from_str(text).ok()?; + let fields = value.as_object_mut()?; + if !fields.contains_key("imagery") && fields.get("bounds").is_some_and(|bounds| !bounds.is_null()) { + fields.insert("imagery".to_owned(), "image".into()); + } + serde_json::from_value(value).ok() +} + pub fn save_settings(conn: &Connection, saved: &MapSettings) -> Result<(), StoreError> { let text = serde_json::to_string(saved).expect("numbers always serialise"); settings::set(conn, SETTINGS, &text) @@ -52,15 +63,46 @@ struct Stored { #[cfg(test)] mod tests { use jiff::Timestamp; - use observer_domain::map::MapBounds; + use observer_domain::map::{MapBounds, MapImagery}; use super::*; use crate::database::memory; + #[test] + fn settings_saved_before_the_imagery_keep_a_placed_image_and_take_the_default_without_one() { + let conn = memory(); + settings::set( + &conn, + SETTINGS, + r#"{"bounds":{"west":-5640.0,"east":6470.0,"south":-4140.0,"north":7970.0},"newHours":5,"veteranHours":50,"everySeconds":1}"#, + ) + .unwrap(); + let placed = settings(&conn).unwrap(); + assert_eq!(placed.imagery, MapImagery::Image); + assert_eq!(placed.new_hours, 5); + + settings::set( + &conn, + SETTINGS, + r#"{"bounds":null,"newHours":5,"veteranHours":50,"everySeconds":1}"#, + ) + .unwrap(); + assert_eq!(settings(&conn).unwrap().imagery, MapImagery::Atlas); + + // What the owner chose since is kept as chosen, bounds or not. + let chosen = MapSettings { + imagery: MapImagery::Roads, + ..placed + }; + save_settings(&conn, &chosen).unwrap(); + assert_eq!(settings(&conn).unwrap(), chosen); + } + #[test] fn settings_and_the_image_round_trip_and_bad_ones_fall_back() { let conn = memory(); let saved = MapSettings { + imagery: MapImagery::Image, bounds: Some(MapBounds { west: -5640.0, east: 6470.0, diff --git a/crates/observer-web/src/security.rs b/crates/observer-web/src/security.rs index 83c1a976..6956699d 100644 --- a/crates/observer-web/src/security.rs +++ b/crates/observer-web/src/security.rs @@ -12,11 +12,13 @@ use crate::AppState; use crate::error::Failure; use crate::proxy::Remote; -/// Everything the panel loads comes from Observer itself. `data:` images carry the grain texture; -/// `blob:` ones the live map's image, which the panel fetches with the session key. +/// Everything the panel loads comes from Observer itself, but the tiles of Rockstar's map of +/// GTA V, which the live map loads from Rockstar's servers when the owner picks it: images only, +/// from that one host, without a referrer. `data:` images carry the grain texture; `blob:` ones +/// the owner's map image, which the panel fetches with the session key. const CONTENT_SECURITY_POLICY: &str = "default-src 'self'; script-src 'self'; style-src 'self'; \ - img-src 'self' data: blob:; font-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'; \ - form-action 'self'; frame-ancestors 'none'"; + img-src 'self' data: blob: https://s.rsg.sc; font-src 'self'; connect-src 'self'; object-src 'none'; \ + base-uri 'none'; form-action 'self'; frame-ancestors 'none'"; const PERMISSIONS_POLICY: &str = "camera=(), geolocation=(), microphone=(), payment=(), usb=()"; diff --git a/crates/observer-web/src/tests.rs b/crates/observer-web/src/tests.rs index b356698c..b5a4a9e1 100644 --- a/crates/observer-web/src/tests.rs +++ b/crates/observer-web/src/tests.rs @@ -645,6 +645,9 @@ async fn every_response_carries_the_security_headers() { let csp = reply.headers[header::CONTENT_SECURITY_POLICY].to_str().unwrap(); assert!(csp.contains("script-src 'self'")); assert!(csp.contains("frame-ancestors 'none'")); + // Images alone come from elsewhere, and from one host: the live map's tiles. + assert!(csp.contains("img-src 'self' data: blob: https://s.rsg.sc;")); + assert_eq!(csp.matches("https://").count(), 1); assert_eq!(reply.headers[header::X_CONTENT_TYPE_OPTIONS], "nosniff"); assert_eq!(reply.headers[header::REFERRER_POLICY], "no-referrer"); assert_eq!(reply.headers[header::CACHE_CONTROL], "no-store"); diff --git a/crates/observer-web/src/tests/map.rs b/crates/observer-web/src/tests/map.rs index 49504869..a7605ee7 100644 --- a/crates/observer-web/src/tests/map.rs +++ b/crates/observer-web/src/tests/map.rs @@ -64,6 +64,7 @@ async fn the_map_opens_to_its_own_permission_and_its_image_to_the_owner() { assert_eq!(seen.status, StatusCode::OK, "{}", seen.text); assert_eq!(seen.json()["image"], json!(null)); assert_eq!(seen.json()["settings"]["everySeconds"], 2); + assert_eq!(seen.json()["settings"]["imagery"], "atlas"); assert_eq!(not_owner.status, StatusCode::FORBIDDEN); assert_eq!(reauth.status, StatusCode::NO_CONTENT, "{}", reauth.text); assert_eq!(added.status, StatusCode::OK, "{}", added.text); @@ -89,6 +90,7 @@ async fn the_settings_take_the_settings_permission_and_are_checked() { let watcher = server.member(&owner, "hopper", PLAYERS_MAP).await; let manager = server.member(&owner, "sarah", SETTINGS_MANAGE).await; let settings = json!({ + "imagery": "satellite", "bounds": { "west": -5640.0, "east": 6470.0, "south": -4140.0, "north": 7970.0 }, "newHours": 5, "veteranHours": 50, diff --git a/web/src/api/generated/MapImagery.ts b/web/src/api/generated/MapImagery.ts new file mode 100644 index 00000000..4dfac9d2 --- /dev/null +++ b/web/src/api/generated/MapImagery.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * What the map draws under the players. + */ +export type MapImagery = "atlas" | "satellite" | "roads" | "image" | "grid"; diff --git a/web/src/api/generated/MapSettings.ts b/web/src/api/generated/MapSettings.ts index cd01c20d..22a6f477 100644 --- a/web/src/api/generated/MapSettings.ts +++ b/web/src/api/generated/MapSettings.ts @@ -1,9 +1,11 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { MapBounds } from "./MapBounds"; +import type { MapImagery } from "./MapImagery"; -export type MapSettings = { +export type MapSettings = { imagery: MapImagery, /** - * `None` until the image is calibrated: the map then shows a grid of game coordinates. + * Where the owner's image sits; `None` until it is calibrated: the map then shows a grid of + * game coordinates in its place. */ bounds: MapBounds | null, /** diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 2412ebac..80778a18 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -56,6 +56,7 @@ import type { LeaveReason } from '../api/generated/LeaveReason'; import type { LicenceState } from '../api/generated/LicenceState'; import type { LineKind } from '../api/generated/LineKind'; import type { LiveChangeKind } from '../api/generated/LiveChangeKind'; +import type { MapImagery } from '../api/generated/MapImagery'; import type { ModelInfoKind } from '../api/generated/ModelInfoKind'; import type { NoticeReason } from '../api/generated/NoticeReason'; import type { NotWritable } from '../api/generated/NotWritable'; @@ -2055,6 +2056,9 @@ export const en = { `Positions every ${count(seconds, { one: 'second', other: 'seconds' })}, sent only while someone has this page open.`, updated: (ago: string) => `Updated ${ago} ago`, waitingFirst: 'Waiting for the first positions…', + tilesUnsent: "Rockstar's servers did not send the map: the grid shows game coordinates.", + retry: 'Try again', + credit: 'Map © Rockstar Games, loaded from its servers', noImage: 'No map image yet: the grid shows game coordinates.', notCalibrated: 'The map image is not placed in the game yet: the grid shows game coordinates.', addImage: 'Add one in Settings', @@ -4305,7 +4309,7 @@ export const en = { }, map: { name: 'Live map', - about: 'The map image of the Live map page, where it sits in the game, and the colours of the players on it.', + about: 'What the Live map draws under the players, your own image and where it sits in the game, and the colours of the players on it.', }, restarts: { name: 'Restarts', @@ -4601,9 +4605,13 @@ export const en = { label: 'Refusal message', about: 'What a banned player reads when the server refuses them, or drops them: the only text Observer ever shows players. {reason}, {ends}, {id}, {name} and {by} are filled in.', }, + mapImagery: { + label: 'Map', + about: "What the Live map draws under the players. Atlas, Satellite and Roads are Rockstar's own maps of GTA V: each browser that opens the Live map loads them from Rockstar's servers, which then see its address. Observer neither ships nor keeps them. My image and Grid load nothing from elsewhere.", + }, mapImage: { - label: 'Map image', - about: "Observer ships no map: GTA V's belongs to Rockstar Games. Give the path of an image on this host that you have the right to use, such as GTA V's atlas from your own copy of the game: PNG, JPEG or WebP, up to 16,384 px a side and 64 MB. Observer keeps a copy in its data folder; only accounts that see the live map load it.", + label: 'My image', + about: "For another map than Rockstar's, a custom island or RedM's for one: the path of an image on this host that you have the right to use, PNG, JPEG or WebP, up to 16,384 px a side and 64 MB. Observer keeps a copy in its data folder; only accounts that see the live map load it. It shows once Map is set to My image and the image is placed.", }, mapBounds: { label: 'Where the image sits in the game', @@ -4733,6 +4741,13 @@ export const en = { }, map: { openMap: 'Open the Live map', + imagery: { + atlas: 'Atlas', + satellite: 'Satellite', + roads: 'Roads', + image: 'My image', + grid: 'Grid', + } satisfies Record, path: 'Path on this host', use: 'Use this image', remove: 'Remove the image', diff --git a/web/src/map/MapCanvas.test.tsx b/web/src/map/MapCanvas.test.tsx index 61954a4a..a839f6ea 100644 --- a/web/src/map/MapCanvas.test.tsx +++ b/web/src/map/MapCanvas.test.tsx @@ -19,6 +19,13 @@ class FixedSize { disconnect() {} } +/** Whether a mark placed there lies out of the 800 by 600 map. */ +const outside = (style: CSSStyleDeclaration) => + parseFloat(style.left) < 0 || + parseFloat(style.left) > 800 || + parseFloat(style.top) < 0 || + parseFloat(style.top) > 600; + const draw = (props: Partial[0]> = {}) => { const onSelect = vi.fn(); render( @@ -76,6 +83,46 @@ describe('MapCanvas', () => { expect(screen.getByText('kaito')).not.toBeNull(); }); + it('should draw Rockstar’s map as tiles in place of the grid, and tell when its servers do not send it', async () => { + const onTilesFailed = vi.fn(); + draw({ tiles: 'atlas', onTilesFailed }); + await act(async () => {}); + const tiles = [...document.querySelectorAll('img')]; + + expect(tiles.length).toBeGreaterThan(1); + expect(tiles[0]!.src).toBe('https://s.rsg.sc/sc/images/games/GTAV/map/game/0/0/0.jpg'); + for (const tile of tiles) { + expect(tile.src).toMatch( + /^https:\/\/s\.rsg\.sc\/sc\/images\/games\/GTAV\/map\/game\/\d\/\d+\/\d+\.jpg$/, + ); + expect(tile.getAttribute('referrerpolicy')).toBe('no-referrer'); + } + expect(screen.queryAllByText('0')).toHaveLength(0); + expect(screen.getByRole('button', { name: DOTS[0]!.label })).not.toBeNull(); + + // A tile that does not come leaves the whole map under it; the whole map missing is told. + fireEvent.error(tiles[1]!); + expect(tiles[1]!.style.visibility).toBe('hidden'); + expect(onTilesFailed).not.toHaveBeenCalled(); + fireEvent.error(tiles[0]!); + expect(onTilesFailed).toHaveBeenCalledTimes(1); + }); + + it('should bring a mark out of view into it when the keyboard reaches it', async () => { + draw({ tiles: 'atlas' }); + await act(async () => {}); + const mark = screen.getByRole('button', { name: DOTS[0]!.label }); + for (let click = 0; click < 8; click += 1) { + fireEvent.click(screen.getByRole('button', { name: en.map.zoomIn })); + } + expect(outside(mark.parentElement!.style)).toBe(true); + + fireEvent.focus(mark); + + expect(parseFloat(mark.parentElement!.style.left)).toBeCloseTo(400, 0); + expect(parseFloat(mark.parentElement!.style.top)).toBeCloseTo(300, 0); + }); + it('should zoom from its buttons and draw a grid of game coordinates without an image', async () => { draw(); await act(async () => {}); diff --git a/web/src/map/MapCanvas.tsx b/web/src/map/MapCanvas.tsx index 105550da..73e061b4 100644 --- a/web/src/map/MapCanvas.tsx +++ b/web/src/map/MapCanvas.tsx @@ -6,6 +6,7 @@ import { Button } from '../components/Button'; import { useI18n } from '../i18n/I18nProvider'; import { cn } from '../lib/cn'; import { GRID_BOUNDS, niceStep, toFraction, type Tier } from './geometry'; +import { SEA, TILES_BOUNDS, TILES_WORLD, tilesIn, type TileStyle } from './tiles'; /** A player as the map draws them: where, in which colour, and what a reader hears. */ export type Dot = { id: number; x: number; y: number; tier: Tier; name: string; label: string }; @@ -29,10 +30,12 @@ const GRID_WIDTH = 1000; type View = { scale: number; x: number; y: number }; /** - * The map: the owner's image placed in the game, or a grid of game coordinates, dragged and zoomed, - * with a mark for each player that keeps its size whatever the zoom. + * The map: Rockstar's own as tiles, the owner's image placed in the game, or a grid of game + * coordinates, dragged and zoomed, with a mark for each player that keeps its size whatever the zoom. */ export function MapCanvas({ + tiles = null, + onTilesFailed, image, bounds, dots, @@ -43,6 +46,10 @@ export function MapCanvas({ children, className, }: { + /** The style of Rockstar's map to draw, which comes before the image; null for the image or the grid. */ + tiles?: TileStyle | null; + /** Rockstar's servers did not send the map. */ + onTilesFailed?: () => void; /** A `blob:` address and the image's size in pixels, or null for the grid. */ image: { url: string; width: number; height: number } | null; /** Where the image's edges lie in the game; null draws the grid. */ @@ -61,16 +68,19 @@ export function MapCanvas({ const words = t.map; const box = useRef(null); const [size, setSize] = useState({ width: 0, height: 0 }); - const placed = image !== null && bounds !== null; - const area = placed ? bounds : GRID_BOUNDS; - const world = placed - ? { width: image.width, height: image.height } - : { - width: GRID_WIDTH, - height: - (GRID_WIDTH * (GRID_BOUNDS.north - GRID_BOUNDS.south)) / - (GRID_BOUNDS.east - GRID_BOUNDS.west), - }; + const placed = tiles === null && image !== null && bounds !== null; + const area = tiles !== null ? TILES_BOUNDS : placed ? bounds : GRID_BOUNDS; + const world = + tiles !== null + ? TILES_WORLD + : placed + ? { width: image.width, height: image.height } + : { + width: GRID_WIDTH, + height: + (GRID_WIDTH * (GRID_BOUNDS.north - GRID_BOUNDS.south)) / + (GRID_BOUNDS.east - GRID_BOUNDS.west), + }; const [view, setView] = useState(null); const drag = useRef<{ pointer: number; x: number; y: number; moved: boolean } | null>(null); @@ -138,15 +148,16 @@ export function MapCanvas({ ); }, [dots, framing, size.width, world.width, world.height, worldOf]); - // The first view shows every player, or the whole map; a new image starts over. + // The first view shows every player, or the whole map; another map under them starts over, but + // another style of the same one. const framedFor = useRef(null); + const base = tiles !== null ? 'tiles' : placed ? image.url : 'grid'; useEffect(() => { - const key = `${image?.url ?? 'grid'}:${placed}`; - if (size.width > 0 && framedFor.current !== key) { - framedFor.current = key; + if (size.width > 0 && framedFor.current !== base) { + framedFor.current = base; fit(); } - }, [fit, image?.url, placed, size.width]); + }, [fit, base, size.width]); // Only a new ask moves the map, not each new position: what it reads stays out of the effect's // dependencies. @@ -263,11 +274,17 @@ export function MapCanvas({