diff --git a/.gitattributes b/.gitattributes index 95c000177..5e663bd0b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,3 +19,7 @@ mobile/android/gradlew text eol=lf *.bat text eol=crlf *.cmd text eol=crlf + +# The PS5 build applies these patches in WSL against LF upstream sources; a +# CRLF checkout of a patch no longer matches a single line of them. +ports/ps5/**/*.patch text eol=lf diff --git a/README.md b/README.md index 4cd31e7c0..f4e196fcc 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,7 @@ payload in the same run. Grab the row that matches your device. | Android | `Gen2Recomped--android.apk` | Allow unknown sources, install | | Nintendo Switch | `Gen2Recomped--switch.zip` | See [Switch](#nintendo-switch) below | | Xbox (Dev Mode) | `Gen2Recomped--xbox-uwp.zip` | See [Xbox](#xbox-dev-mode) below | +| PlayStation 5 (experimental) | _in progress — no release asset yet_ | See [PlayStation 5](#playstation-5-experimental) below | | Anbernic RG34XXSP | `Gen2Recomped--rg34xxsp-stockos64-mod.zip` | See [Handhelds](#handhelds) below | | iOS | `Gen2Recomped--ios.ipa` | Sideload — see [iOS](#ios) below | | Already have LÖVE 11.5 | `Gen2Recomped-.love` | `love Gen2Recomped-.love` | @@ -305,6 +306,36 @@ Needs Developer Mode on the console — a retail Xbox cannot install this. Xbox Device Portal in your browser, or `Add-AppxPackage` it from a paired PC. Build notes are in [ports/uwp/BUILD.md](ports/uwp/BUILD.md). +### PlayStation 5 (experimental) + +**In progress.** The PS5 port is a homebrew title for a jailbroken console, +loaded with ShadowMountPlus (title ID `PPSA42002`). It needs a native LÖVE +runtime for the console that is still being brought up, so expect rough edges +and no published release asset yet. The game's own code treats it like the +other TV consoles: a DualSense only, no mouse, keyboard or file picker, and no +self-update. + +Your dump goes in the game's save folder, which you reach over FTP: + +``` +/data/gen2recomp/Gen2Recomp/imports/ put your cartridge dump here +/data/gen2recomp/Gen2Recomp/exports/ battery saves come back out here +``` + +Then: + +1. Copy a legal `.gb` / `.gbc` / `.gba` / `.nds` into `/data/gen2recomp/Gen2Recomp/imports/`. + Dumps are matched by SHA-1, so the filename does not matter. +2. Launch the title from the home screen. +3. Press **Import** in the launcher — it scans that folder. + +Launcher controls: left stick or D-pad moves the pointer, **Cross** clicks and +confirms, **Circle** cancels, **L1**/**R1** change tabs. + +Updating: copy the new release over the old one and keep the `Gen2Recomp/` +folder — that is where your saves, imported ROMs, mods and options live. The +launcher tells you when a newer release exists but does not replace itself. + ### Handhelds `Gen2Recomped--rg34xxsp-stockos64-mod.zip` targets the **Anbernic @@ -336,8 +367,8 @@ from source on a Mac, see [docs/ios-install.md](docs/ios-install.md). Desktop and mobile builds check GitHub Releases and can update themselves. Console builds tell you a release exists but will not replace themselves — a packaged console app cannot rewrite its own files — so update those the same -way you installed them. The Switch is the exception: its OTA launcher replaces -both NROs in place. See [docs/updater.md](docs/updater.md). +way you installed them (on the PS5, copy the new release over FTP). The Switch +is the exception: its OTA launcher replaces both NROs in place. See [docs/updater.md](docs/updater.md). ## Controls diff --git a/conf.lua b/conf.lua index 52db30f4a..50acd93bf 100644 --- a/conf.lua +++ b/conf.lua @@ -90,7 +90,21 @@ function love.conf(t) local osName = love._os local mobile = osName == "Android" or osName == "iOS" local nx = osName == "NX" - if nx then + local ps5 = osName == "PS5" + if ps5 then + -- PlayStation 5 homebrew (native LOVE runtime). The output is the TV: + -- 1920x1080, or 3840x2160 where the runtime selects it. It is always a + -- full-screen, non-resizable window -- nothing will ever resize it -- and + -- the 160x144 viewport letterboxes into whatever the display is, so the + -- size here is only a starting point. No highdpi. + t.window.width = 1920 + t.window.height = 1080 + t.window.fullscreen = true + t.window.fullscreentype = "desktop" + t.window.resizable = false + t.window.borderless = true + t.window.highdpi = false + elseif nx then -- Switch (love-nx). Handheld is 720p and docked is 1080p, and SDL only -- follows the dock/undock transition when the window is resizable and NOT -- exclusive fullscreen -- a fullscreen window keeps the boot mode and the diff --git a/data/localization/launcher_sources.lua b/data/localization/launcher_sources.lua index efeb3f3f0..22bbf423a 100644 --- a/data/localization/launcher_sources.lua +++ b/data/localization/launcher_sources.lua @@ -146,6 +146,7 @@ return { "An update is available", "Update from the OTA launcher (gen2recomp.nro)", "Install the newer package to update", +"Copy the new release to your PS5 to update", "Cannot check for updates on this build", "A new version needs a fresh download", "Open releases", diff --git a/ports/ps5/BUILD.md b/ports/ps5/BUILD.md new file mode 100644 index 000000000..eda385a18 --- /dev/null +++ b/ports/ps5/BUILD.md @@ -0,0 +1,196 @@ +# Gen2Recomped on PS5 (native title) + +The game runs on a jailbroken PS5 as a native **title**: an `eboot.bin` in a +folder under `/data/homebrew`, registered on the home screen by +[ShadowMountPlus](https://github.com/drakmor/ShadowMountPlus). It is not an +elfldr payload. The eboot is LOVE 11.5 plus a small launcher; the game itself +is the same `game.love` every other port ships (`scripts/pack_love.sh`). + +## Build + +On Linux or in WSL (Ubuntu 26.04 tested): + +```sh +sudo apt install clang-18 lld-18 clang cmake ninja-build make autoconf automake libtool \ + pkg-config python3 curl git zip unzip patch +scripts/build_ps5.sh # title folder in ~/.cache/gen2recomp-ps5/out/title/PPSA42002 +scripts/build_ps5.sh --deploy 10.0.0.101 # ... and upload it, then ask ShadowMountPlus to scan +scripts/build_ps5.sh --test-game # the runtime bring-up game instead of Gen2Recomped +scripts/build_ps5.sh --version 1.2.3 # stamp the packed game.love (pack_love.sh --version) +``` + +From Windows: `wsl bash scripts/build_ps5.sh --deploy 10.0.0.101`. + +The first run fetches and builds the runtime (about 5 minutes, ~1 GB in +`$G2R_PS5_WORK`, default `~/.cache/gen2recomp-ps5`); later runs only repack +`game.love` and relink. `--clean` rebuilds the runtime. Nothing large is +written into the repository. + +`--deploy` uploads with FTP to `ftpsrv` on port 2121 (`PS5_FTP_PORT`), +`eboot.bin` and `param.json` last, then `POST http://:10101/api/v1/scan`. +ShadowMountPlus defers that scan while any homebrew title runs or is +suspended: close it and scan again (or use its menu). It copies the icon and +backgrounds only when it first registers a title. + +## Title folder + +``` +/data/homebrew/PPSA42002/ + eboot.bin launcher + LOVE + SDL2 + ps5-opengl, development FSELF + sce_module/libc.prx the boilerplate's clean-room loader shim + sce_sys/param.json ports/ps5/sce_sys/param.json + sce_sys/icon0.png ports/ps5/sce_sys/icon0.png (512x512) + sce_sys/pic0.dds ports/ps5/sce_sys/pic0.png as 3840x2160 BC7 DX10 DDS + sce_sys/pic1.dds ports/ps5/sce_sys/pic1.png, same + game.love scripts/pack_love.sh +``` + +`sce_sys/` in the repository keeps PNGs; `ports/ps5/tools/png2dds.c` (built +on the fly, a BC7 mode-6 encoder over LOVE's stb_image) makes the DDS files, +so no texconv is needed. + +## On the console + +| What | Where | +|---|---| +| Game | `/app0/game.love`, mounted fused (the title folder is read-only) | +| Data root | `/data/gen2recomp` (`/download0/gen2recomp` if the title cannot write `/data`) | +| Save directory | `/data/gen2recomp/Gen2Recomp` (`love.filesystem.getSaveDirectory()`, identity `Gen2Recomp`) | +| ROM imports | `/data/gen2recomp/Gen2Recomp/imports` (the save directory's `imports`, as on Switch) | +| Log | `/data/gen2recomp/love.log` (stdout/stderr: `print`, LOVE errors), plus `[gen2recomp]` lines in the kernel log | +| JIT override | an empty `/data/gen2recomp/jit-off` or `jit-on` | +| Frame dump | an empty `/data/gen2recomp/screenshot`: every 600th frame goes to `/data/gen2recomp/screenshot.png` | + +`love.system.getOS()` and `love._os` are `"PS5"` (from the start: `conf.lua` +already sees it). `love.system.openURL` returns false and does nothing. + +Leaving the game (`love.event.quit()`) closes the title through +`sceSystemServiceLoadExec("exit")`; a title must never `exit()`. The PS +button/"Close application" works as for any title. + +## Verified on the console + +PS5 firmware 13.00, etaHEN, ShadowMountPlus 1.7beta7 (title registered as +`PPSA42002` "Gen2Recomped", icon and backgrounds shown), 2026-11-14: + +- **Video**: `OpenGL 4.6 (Core Profile) Mesa 26.2.0 | PS5 AGC` (LOVE asks for + 3.3 core and gets 4.6 core), 1920x1080, max texture 16384, MSAA 4, shaders + and canvases work, steady 60 FPS (vsync) in the bring-up game. +- **Pad**: the DualSense of the signed-in user appears as a gamepad + (`joystickadded`, GUID above); every mapped button was checked (Cross a, + Circle b, Square x, Triangle y, Options start, touchpad back, L1/R1, D-pad, + L3). Analog sticks and L2/R2 axes were not exercised. +- **Audio**: LOVE's OpenAL source plays; the system log shows the title's + process opening a shared AudioOut port. Nobody listened to it. +- **JIT**: the probe succeeds, LuaJIT runs with the JIT on (a 20M-iteration + loop: 0.19 s); `jit-off` was not needed. +- **FFI**: `ffi.C.PHYSFS_getSearchPath`, `ffi.C.getpid` work; `ffi.load` + fails with a clean error. +- **C++ exceptions** become Lua errors; `love.thread` workers run. +- **Save directory** `/data/gen2recomp/` is created and written. +- **Gen2Recomped** boots to its hub (News page, "0 of 18 games ready") without + ROMs, with no Lua errors. + +Not checked: leaving through `love.event.quit()` (the title was always closed +from the system), suspend/resume, a second user's pad, HDR, sound output by ear. + +Console captures of a running title (the screenshot API this was tested with) +were often black even while the title rendered at 60 FPS; the frame dump +above reads the frame back through LOVE instead. + +## The stack + +| Part | Version / pin | Role | +|---|---|---| +| [ps5-native-app-boilerplate](https://github.com/drakmor/ps5-native-app-boilerplate) | `dd44bbd` | title CRT, `libc.prx`, ELF to FSELF converter, payload SDK v0.42 (sha256-pinned by its setup script) | +| [ps5-opengl SDK](https://github.com/darkxex/ps5-opengl/releases/tag/v1.0.2) | 1.0.2, sha256 `30d8e51d…` | EGL + OpenGL 4.6 core / compatibility: Mesa 26.2 on the PS5's AGC, the GL stack EglantinePS5 ships | +| [SDL (ps5-payload-dev fork)](https://github.com/ps5-payload-dev/SDL) | `8c56053` (2.30.12) | events, threads, PS5 pad (ScePad) and audio (SceAudioOut) backends; its OSMesa video driver is replaced by ps5-opengl's SDL2 "G19" driver (`integration/SDL2` in the SDK's sources) with `ports/ps5/sdl/g19-love.patch` | +| [LuaJIT](https://github.com/LuaJIT/LuaJIT) | `f9140a6` (PacBrew's) | FFI on (`__PROSPERO__` hidden, as PacBrew does), `LUAJIT_USE_SYSMALLOC` | +| [LOVE](https://github.com/love2d/love/releases/tag/11.5) | 11.5, sha256 `066e0843…` | autotools, static, `ports/ps5/love/0001-love-11.5-ps5.patch` | +| [PacBrew](https://github.com/ps5-payload-dev/pacbrew-repo/releases/tag/v0.40.2) | v0.40.2, sha256 `a85f65de…` | prebuilt freetype, openal-soft 1.24.3 (SDL2 backend), vorbis, ogg, theora, modplug, mpg123, libpng, zlib, bzip2 | + +All pins live in `scripts/ps5/deps.sh`. + +Why this stack: ps5-opengl is what the user's other titles (PS5SD, PS5LM via +ps5-homebrew-ui, EglantinePS5) draw with, and it is a real hardware GL inside +a title. PacBrew's own SDL2/LOVE packages use Mesa through zink or OSMesa +loaded with dlopen, which is a payload design. + +### What the runtime changes + +- **LOVE** (`ports/ps5/love/0001-love-11.5-ps5.patch`): `getOS()` and + `love._os` are `"PS5"`; `openURL` is a no-op; `getExecutablePath()` is + `/app0/game.love` (or `$LOVE_PS5_GAME`), which makes `boot.lua` mount the + game fused, so the save directory is `/` with no + `love/` folder; LuaJIT luasocket compat fix (PacBrew's). +- **SDL** (`ports/ps5/sdl/`): the G19 driver gives LOVE one EGL window/context + at the fixed 1920x1080 scanout size, whatever size `conf.lua` asks for + (patched; upstream rejects other sizes and resizable windows). LOVE gets an + OpenGL 3.3 core context first, like on desktop. SDL audio and the game + controller layer are on; the launcher adds a controller mapping for the PS5 + backend's GUID (`0300d0424c050000e60c000011810000`): Cross=a, Circle=b, + Square=x, Triangle=y, Options=start, **touchpad click=back**, L1/R1 + shoulders, L2/R2 triggers (analog), L3/R3, D-pad as buttons, both sticks. + Only pads of signed-in users are seen (ScePad per user). +- **Launcher** (`ports/ps5/launcher/main.cpp`): data root, log, environment, + JIT decision, splash, frame dump hook, LOVE's own run loop (with + restart), title exit. +- **Memory**: `malloc` and friends are wrapped (`--wrap`) into ps5-opengl's + `native-app/app_heap.c`, a 3 GiB heap in the title's direct memory. LuaJIT + uses `malloc` (`LUAJIT_USE_SYSMALLOC`): its own allocator would map flexible + memory, of which a title has only a few hundred MiB. +- **JIT**: LuaJIT exits the process if `mprotect(PROT_EXEC)` fails, so the + launcher tries the same mapping first (and runs a 6-byte test function); if + it fails the JIT engine is turned off before any Lua runs and LuaJIT + interprets. FFI works either way. +- **FFI symbols**: `ffi.C.X` is `dlsym(RTLD_DEFAULT, "X")`. The title exports + nothing, so `dlsym`/`dlopen`/`dlerror`/`dlclose` are wrapped + (`ports/ps5/launcher/ffi.c`) and answer from a generated table: every + `PHYSFS_*` function LOVE contains plus the libc names in + `ports/ps5/launcher/ffi_exports.txt`. `ffi.load()` fails with an error + (no dynamic libraries), which the game already `pcall`s. +- **C++ runtime**: `libSceLibcInternal` exports `__gxx_personality_v0`, + `__cxa_get_globals`, the `std::` exception classes and more. Bound there, + Sony's personality routine runs under the static libunwind and the first + `throw` crashes. The link extracts libc++abi, libc++ and libunwind first + (`-u` for each such symbol, archives before the Sony stubs), and + `check_imports.py` fails the build if any C++ runtime symbol is imported. +- **lstat**: fails with `EPERM` for every path in a title (`stat` works); + PhysFS uses it to create the save directory. `lstat` is wrapped + (`--wrap=lstat`, `compat.c`) to fall back to `stat`. +- **LuaJIT unwinding**: LLVM libunwind's `__register_frame` takes one FDE, as + on macOS; LuaJIT registers its JIT code that way here (`lj_err.c` patched + by `build_runtime.sh`), otherwise each trace logs "bad fde" and errors + cannot unwind through compiled code. +- **Imports**: a title has no `libScePosixForWebKit` or `libkernel_sys`; an + import bound there would be a null call. `scripts/ps5/check_imports.py` + runs after every link and fails the build if any import binds to them. + `ports/ps5/launcher/compat.c` supplies the few libc functions no Sony + module has (`getentropy`, `__cxa_thread_atexit_impl`, `isatty`, the + `lstat` wrapper); the rest + come from the payload SDK's `libc.a` (linked after the Sony stubs, so a + stub always wins). + +## Files + +| File | | +|---|---| +| `scripts/build_ps5.sh` | the one command | +| `scripts/ps5/deps.sh` | pins and fetchers | +| `scripts/ps5/build_runtime.sh` | LuaJIT, SDL2, LOVE, link, FSELF | +| `scripts/ps5/check_imports.py` | import check after linking | +| `ports/ps5/launcher/` | launcher, FFI table, libc compat | +| `ports/ps5/sdl/` | SDL2 CMake wrapper and the G19 patch | +| `ports/ps5/love/` | LOVE patch | +| `ports/ps5/tools/png2dds.c` | PNG to BC7 DDS for `pic0`/`pic1` | +| `ports/ps5/testgame/` | bring-up game (`--test-game`) | +| `ports/ps5/sce_sys/` | param.json and artwork (title ID `PPSA42002`) | + +## Licences + +ps5-opengl (and its SDL2 driver), the boilerplate's tooling and libc.prx are +GPL-3.0-or-later; ps5-opengl's Mesa is MIT; SDL, LOVE, LuaJIT are zlib/MIT; +openal-soft is LGPL. The eboot links ps5-opengl statically, so `eboot.bin` is +a GPL-3.0 work; `game.love` is a separate file under the game's own licence. +Distributing the eboot means offering the corresponding source (the pinned +inputs above plus this directory). diff --git a/ports/ps5/launcher/compat.c b/ports/ps5/launcher/compat.c new file mode 100644 index 000000000..526a76981 --- /dev/null +++ b/ports/ps5/launcher/compat.c @@ -0,0 +1,154 @@ +/* Gen2Recomped PS5: libc functions the title defines itself. + * + * A title gets no libScePosixForWebKit and no libkernel_sys, so an import the + * payload SDK only finds there is a null call at run time. + * scripts/ps5/check_imports.py lists such imports after linking and fails the + * build; each one is answered here instead. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void arc4random_buf(void *buffer, size_t length); + +/* libc++ (std::random_device) wants it and no Sony module exports it; the + * payload SDK's arc4random (sysctl KERN_ARND) does the work. */ +int getentropy(void *buffer, size_t length) +{ + if (length > 256) { + errno = EIO; + return -1; + } + arc4random_buf(buffer, length); + return 0; +} + +/* libc++abi's __cxa_thread_atexit (thread_local destructors) calls this when + * the C library has it; the converter wants every import resolved, and no + * Sony module has it. Destructors run at thread exit, newest first. */ +struct g2r_tls_dtor { + void (*dtor)(void *); + void *object; + struct g2r_tls_dtor *next; +}; +static pthread_key_t g2r_tls_key; +static pthread_once_t g2r_tls_once = PTHREAD_ONCE_INIT; + +static void g2r_tls_run(void *list) +{ + struct g2r_tls_dtor *node = list; + while (node != NULL) { + struct g2r_tls_dtor *next = node->next; + node->dtor(node->object); + free(node); + node = next; + } +} + +static void g2r_tls_init(void) +{ + pthread_key_create(&g2r_tls_key, g2r_tls_run); +} + +int __cxa_thread_atexit_impl(void (*dtor)(void *), void *object, void *dso) +{ + (void)dso; + pthread_once(&g2r_tls_once, g2r_tls_init); + struct g2r_tls_dtor *node = malloc(sizeof(*node)); + if (node == NULL) + return -1; + node->dtor = dtor; + node->object = object; + node->next = pthread_getspecific(g2r_tls_key); + pthread_setspecific(g2r_tls_key, node); + return 0; +} + +/* lstat() in a title fails with EPERM for every path, existing or not + * (measured on 13.00; stat() works). PhysFS stats with lstat when it creates + * directories, so LOVE could never set up its save directory. Linked with + * --wrap=lstat: fall back to stat(), which only differs for symbolic links, + * and a title's data folders have none. */ +int __real_lstat(const char *path, struct stat *st); + +int __wrap_lstat(const char *path, struct stat *st) +{ + int rc = __real_lstat(path, st); + if (rc != 0 && errno == EPERM) + rc = stat(path, st); + return rc; +} + +/* Directory listing. In a title, readdir() through the system C library + * returns no entries under /data (measured on 13.00: PhysFS saw an empty + * save directory and imports/ folder while stat() found the files in them), + * so love.filesystem.getDirectoryItems was always empty and the import inbox + * never found a ROM. The kernel answers getdents() on /data only with a + * 64 KiB buffer (PS5Homebrew's notes); smaller ones get EINVAL. Linked with + * --wrap=opendir/readdir/closedir, so PhysFS and everything else use this. */ +struct g2r_dir { + int fd; + int len; + int pos; + char buf[65536] __attribute__((aligned(8))); +}; + +DIR *__wrap_opendir(const char *path) +{ + int fd = open(path, O_RDONLY | O_DIRECTORY); + if (fd < 0) + return NULL; + struct g2r_dir *d = malloc(sizeof(*d)); + if (d == NULL) { + close(fd); + errno = ENOMEM; + return NULL; + } + d->fd = fd; + d->len = 0; + d->pos = 0; + return (DIR *)d; +} + +struct dirent *__wrap_readdir(DIR *dirp) +{ + struct g2r_dir *d = (struct g2r_dir *)dirp; + for (;;) { + if (d->pos >= d->len) { + int n = getdents(d->fd, d->buf, (int)sizeof(d->buf)); + if (n <= 0) + return NULL; /* end (errno untouched) or error (errno set) */ + d->len = n; + d->pos = 0; + } + struct dirent *e = (struct dirent *)(d->buf + d->pos); + if (e->d_reclen == 0) { + d->pos = d->len; + return NULL; + } + d->pos += e->d_reclen; + if (e->d_fileno != 0) + return e; + } +} + +int __wrap_closedir(DIR *dirp) +{ + struct g2r_dir *d = (struct g2r_dir *)dirp; + int rc = close(d->fd); + free(d); + return rc; +} + +/* stdout and stderr go to a log file, never a terminal. */ +int isatty(int fd) +{ + (void)fd; + errno = ENOTTY; + return 0; +} diff --git a/ports/ps5/launcher/ffi.c b/ports/ps5/launcher/ffi.c new file mode 100644 index 000000000..7a53a833d --- /dev/null +++ b/ports/ps5/launcher/ffi.c @@ -0,0 +1,57 @@ +/* Gen2Recomped PS5: LuaJIT FFI symbol lookup in a static title. + * + * ffi.C. resolves through dlsym(RTLD_DEFAULT, name). The title is one + * static executable that exports nothing, and a title's dlsym/dlopen are not + * worth trusting (an import the system cannot resolve is a null call), so the + * link wraps them (--wrap=dlsym,dlopen,dlerror,dlclose) and answers from a + * table instead: every PHYSFS_* function of LOVE's PhysFS plus the libc + * functions listed in ports/ps5/launcher/ffi_exports.txt. The table itself, + * g2r_ffi_exports[], is generated by scripts/ps5/build_runtime.sh. + * + * ffi.load() of anything fails cleanly with an error message. */ +#include +#include + +struct g2r_ffi_export { + const char *name; + void *address; +}; + +extern const struct g2r_ffi_export g2r_ffi_exports[]; +extern const size_t g2r_ffi_export_count; + +static const char *g2r_dlerror_message; + +void *__wrap_dlsym(void *handle, const char *name) +{ + (void)handle; + if (name != NULL) { + for (size_t i = 0; i < g2r_ffi_export_count; i++) { + if (strcmp(g2r_ffi_exports[i].name, name) == 0) + return g2r_ffi_exports[i].address; + } + } + g2r_dlerror_message = "symbol not exported by the PS5 title"; + return NULL; +} + +void *__wrap_dlopen(const char *path, int mode) +{ + (void)path; + (void)mode; + g2r_dlerror_message = "no dynamic libraries in the PS5 title"; + return NULL; +} + +char *__wrap_dlerror(void) +{ + const char *message = g2r_dlerror_message; + g2r_dlerror_message = NULL; + return (char *)message; +} + +int __wrap_dlclose(void *handle) +{ + (void)handle; + return 0; +} diff --git a/ports/ps5/launcher/ffi_exports.txt b/ports/ps5/launcher/ffi_exports.txt new file mode 100644 index 000000000..0d5c6e1dd --- /dev/null +++ b/ports/ps5/launcher/ffi_exports.txt @@ -0,0 +1,43 @@ +# libc functions that ffi.C can reach in the PS5 title, besides every PHYSFS_* +# function (added automatically). One name per line; '#' starts a comment. +# Used by the game today: mkdir, chmod, getpid, setenv, unsetenv, and the +# socket calls of src/core/DiscordPresence.lua (which fail harmlessly). +mkdir +rmdir +chmod +getpid +getenv +setenv +unsetenv +remove +rename +time +usleep +socket +connect +close +send +recv +setsockopt +strlen +memcpy +memmove +memset +memcmp +malloc +calloc +realloc +free +pow +atan2 +asin +acos +sin +cos +tan +sqrt +floor +ceil +fmod +exp +log diff --git a/ports/ps5/launcher/main.cpp b/ports/ps5/launcher/main.cpp new file mode 100644 index 000000000..923de6e26 --- /dev/null +++ b/ports/ps5/launcher/main.cpp @@ -0,0 +1,357 @@ +// Gen2Recomped PS5 launcher: runs LOVE 11.5 on the bundled game.love inside a +// native PS5 title (eboot.bin started by the shell, registered by +// ShadowMountPlus). +// +// What it does before LOVE starts: +// * picks the writable root, /data/gen2recomp (falls back to +// /download0/gen2recomp when the title cannot write /data), and points +// XDG_DATA_HOME and HOME there. LOVE runs the game fused (see +// ports/ps5/love/0001-love-11.5-ps5.patch), so the save directory is +// /, i.e. /data/gen2recomp/Gen2Recomp; +// * sends stdout/stderr (LOVE's print, errors) to /love.log; +// * decides whether LuaJIT may JIT: a title may or may not get executable +// memory, and LuaJIT exits the process when mprotect(PROT_EXEC) fails, so +// the launcher tries it first and turns the JIT engine off when it cannot. +// /jit-off or /jit-on override the probe; +// * hides the splash screen. +// Then it is LOVE's own love.cpp loop (restart included). A title must not +// exit(): when the game quits, the launcher asks the system to close it. +// +// Logged to the kernel log with the prefix "[gen2recomp]". + +#define SDL_MAIN_HANDLED 1 +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +} + +#include "common/runtime.h" +#include "common/version.h" +#include "modules/love/love.h" + +extern "C" { +int sceKernelDebugOutText(int channel, const char *text); +int sceKernelUsleep(unsigned int microseconds); +int sceSystemServiceHideSplashScreen(void); +int sceSystemServiceLoadExec(const char *path, char *const argv[]); +} + +// The title's direct-memory heap (ps5-opengl's native-app/app_heap.c, linked +// with --wrap=malloc and friends): every malloc, operator new and -- through +// LUAJIT_USE_SYSMALLOC -- the whole Lua heap. Flexible memory in a title is a +// few hundred MiB; 3 GiB of the title's direct memory leaves the rest of it +// to the GPU (ps5-opengl). +extern "C" const size_t ps5_opengl_heap_size = (size_t) 3 << 30; + +namespace +{ + +const char *const kGame = "/app0/game.love"; +char g_root[256] = "/data/gen2recomp"; + +void say(const char *fmt, ...) +{ + char line[512]; + int n = snprintf(line, sizeof(line), "[gen2recomp] "); + va_list ap; + va_start(ap, fmt); + vsnprintf(line + n, sizeof(line) - n - 1, fmt, ap); + va_end(ap); + size_t len = strlen(line); + if (len == 0 || line[len - 1] != '\n') + { + line[len] = '\n'; + line[len + 1] = 0; + } + sceKernelDebugOutText(0, line); + fputs(line, stdout); + fflush(stdout); +} + +bool exists(const char *path) +{ + struct stat st; + return stat(path, &st) == 0; +} + +bool writable_dir(const char *dir) +{ + mkdir(dir, 0777); + char probe[300]; + snprintf(probe, sizeof(probe), "%s/.probe", dir); + int fd = open(probe, O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd < 0) + return false; + bool ok = write(fd, "ok", 2) == 2; + close(fd); + unlink(probe); + return ok; +} + +void pick_root() +{ + const char *candidates[] = {"/data/gen2recomp", "/download0/gen2recomp"}; + for (const char *dir : candidates) + { + if (writable_dir(dir)) + { + snprintf(g_root, sizeof(g_root), "%s", dir); + return; + } + sceKernelDebugOutText(0, "[gen2recomp] cannot write a data root candidate\n"); + } +} + +// Players reach the save directory over FTP (ROMs into imports/, saves out of +// exports/), and ftpsrv runs as a different user from the title. Builds before +// ports/ps5/love/0002-physfs-shared-modes.patch made these folders 0700, which +// ftpsrv cannot enter; open the ones that already exist. Folders made from now +// on are created 0777 by the patched PhysFS. +void open_shared_dirs() +{ + const char *subdirs[] = {"", "/Gen2Recomp", "/Gen2Recomp/imports", "/Gen2Recomp/imports/mods", + "/Gen2Recomp/imports/saves", "/Gen2Recomp/exports"}; + for (const char *sub : subdirs) + { + char path[300]; + snprintf(path, sizeof(path), "%s%s", g_root, sub); + struct stat st; + if (stat(path, &st) == 0 && S_ISDIR(st.st_mode) && (st.st_mode & 0777) != 0777) + chmod(path, 0777); + } +} + +void open_log() +{ + char path[300]; + snprintf(path, sizeof(path), "%s/love.log", g_root); + if (freopen(path, "w", stdout) != nullptr) + setvbuf(stdout, nullptr, _IOLBF, 0); + if (freopen(path, "a", stderr) != nullptr) + setvbuf(stderr, nullptr, _IONBF, 0); +} + +// The same sequence LuaJIT's mcode allocator uses: map read/write, then flip +// to read/execute. Calls the page too, so a mapping that claims PROT_EXEC but +// faults shows up here, right after the "probing" line, not mid-game. +bool jit_memory_works() +{ + const size_t size = 4096; + void *page = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (page == MAP_FAILED) + { + say("JIT probe: mmap failed (errno %d)", errno); + return false; + } + static const unsigned char code[] = {0xB8, 0x2A, 0x00, 0x00, 0x00, 0xC3}; // mov eax, 42; ret + memcpy(page, code, sizeof(code)); + if (mprotect(page, size, PROT_READ | PROT_EXEC) != 0) + { + say("JIT probe: mprotect(PROT_EXEC) failed (errno %d)", errno); + munmap(page, size); + return false; + } + say("JIT probe: executing a test page"); + int result = ((int (*)(void)) page)(); + munmap(page, size); + return result == 42; +} + +bool decide_jit() +{ + char path[300]; + snprintf(path, sizeof(path), "%s/jit-off", g_root); + if (exists(path)) + { + say("JIT off (%s)", path); + return false; + } + snprintf(path, sizeof(path), "%s/jit-on", g_root); + if (exists(path)) + { + say("JIT on (%s)", path); + return true; + } + bool ok = jit_memory_works(); + say("JIT %s (probe)", ok ? "on" : "off: interpreter only"); + return ok; +} + +int love_preload(lua_State *L, lua_CFunction f, const char *name) +{ + lua_getglobal(L, "package"); + lua_getfield(L, -1, "preload"); + lua_pushcfunction(L, f); + lua_setfield(L, -2, name); + lua_pop(L, 2); + return 0; +} + +enum DoneAction +{ + DONE_QUIT, + DONE_RESTART, +}; + +// love.cpp's runlove(), with the JIT decision and the PS5 arguments. +DoneAction run_love(bool jit, int &retval) +{ + lua_State *L = luaL_newstate(); + if (L == nullptr) + { + say("luaL_newstate failed"); + retval = 1; + return DONE_QUIT; + } + luaL_openlibs(L); + + if (!jit) + luaJIT_setmode(L, 0, LUAJIT_MODE_ENGINE | LUAJIT_MODE_OFF); + + love_preload(L, luaopen_love_jitsetup, "love.jitsetup"); + lua_getglobal(L, "require"); + lua_pushstring(L, "love.jitsetup"); + lua_call(L, 1, 0); + + love_preload(L, luaopen_love, "love"); + + // arg[-2] is the "executable"; the game itself comes from + // Filesystem::getExecutablePath() (/app0/game.love, mounted fused). + lua_newtable(L); + lua_pushstring(L, "/app0/eboot.bin"); + lua_rawseti(L, -2, -2); + lua_pushstring(L, "embedded boot.lua"); + lua_rawseti(L, -2, -1); + lua_setglobal(L, "arg"); + + lua_getglobal(L, "require"); + lua_pushstring(L, "love"); + lua_call(L, 1, 1); + lua_pushboolean(L, 1); + lua_setfield(L, -2, "_exe"); + lua_pop(L, 1); + + // Debug aid: with an (empty) /screenshot file, every 600th frame + // (10 s at 60 Hz) is saved to /screenshot.png, read back by LOVE + // itself. Screen captures of the console (Remote Play, the capture API) + // do not always see a title's GL output. + static const char *const kScreenshotHook = + "local root = ...\n" + "local f = io.open(root .. '/screenshot', 'r')\n" + "if not f then return end\n" + "f:close()\n" + "local open = package.preload['love.graphics']\n" + "package.preload['love.graphics'] = function(...)\n" + " local g = open(...)\n" + " local present, frames = g.present, 0\n" + " g.present = function(...)\n" + " frames = frames + 1\n" + " if frames % 600 == 0 then\n" + " g.captureScreenshot(function(img)\n" + " local out = io.open(root .. '/screenshot.png', 'wb')\n" + " if out then out:write(img:encode('png'):getString()); out:close() end\n" + " end)\n" + " end\n" + " return present(...)\n" + " end\n" + " return g\n" + "end\n"; + if (luaL_loadstring(L, kScreenshotHook) == 0) + { + lua_pushstring(L, g_root); + if (lua_pcall(L, 1, 0, 0) != 0) + { + say("screenshot hook: %s", lua_tostring(L, -1)); + lua_pop(L, 1); + } + } + + lua_getglobal(L, "require"); + lua_pushstring(L, "love.boot"); + lua_call(L, 1, 1); + + lua_newthread(L); + lua_pushvalue(L, -2); + int stackpos = lua_gettop(L); + int nres; + while (love::luax_resume(L, 0, &nres) == LUA_YIELD) + lua_pop(L, lua_gettop(L) - stackpos); + + retval = 0; + DoneAction done = DONE_QUIT; + if (lua_type(L, -1) == LUA_TSTRING && strcmp(lua_tostring(L, -1), "restart") == 0) + done = DONE_RESTART; + if (lua_isnumber(L, -1)) + retval = (int) lua_tonumber(L, -1); + + lua_close(L); + return done; +} + +} // namespace + +int main(int argc, char **argv) +{ + (void) argc; + (void) argv; + + pick_root(); + open_shared_dirs(); + open_log(); + say("LOVE %s on PS5, data root %s", love_version(), g_root); + + setenv("XDG_DATA_HOME", g_root, 1); + setenv("HOME", g_root, 1); + if (getenv("LOVE_PS5_GAME") == nullptr) + setenv("LOVE_PS5_GAME", kGame, 1); + if (!exists(getenv("LOVE_PS5_GAME"))) + say("WARNING: %s is missing", getenv("LOVE_PS5_GAME")); + + const bool jit = decide_jit(); + + sceSystemServiceHideSplashScreen(); + SDL_SetMainReady(); + // SDL's controller database only knows the DualSense under Linux GUIDs; + // the PS5 joystick backend (ScePad) reports its own GUID and button + // order (src/joystick/ps5/SDL_ps5joystick.c btn_map). The D-pad is mapped + // to its buttons, not the hat, which that backend only half updates. The + // touchpad click is "back" (the DualSense has no Select; Create is taken + // by the system). + SDL_SetHint(SDL_HINT_GAMECONTROLLERCONFIG, + "0300d0424c050000e60c000011810000,PS5 Controller," + "a:b0,b:b1,x:b2,y:b3,back:b4,start:b6,leftstick:b7,rightstick:b8," + "leftshoulder:b9,rightshoulder:b10,dpup:b11,dpdown:b12,dpleft:b13,dpright:b14," + "leftx:a0,lefty:a1,rightx:a2,righty:a3,lefttrigger:a4,righttrigger:a5,"); + + int retval = 0; + DoneAction done; + do + { + done = run_love(jit, retval); + say("LOVE returned %d%s", retval, done == DONE_RESTART ? " (restart)" : ""); + } while (done == DONE_RESTART); + + SDL_Quit(); + fflush(nullptr); + + // A title that returns from main or calls exit() dies with SIGSYS; ask + // the system to close it the way PS5SD and PS5LM do. + say("closing the title"); + sceSystemServiceLoadExec("exit", nullptr); + for (;;) + sceKernelUsleep(100000); +} diff --git a/ports/ps5/love/0001-love-11.5-ps5.patch b/ports/ps5/love/0001-love-11.5-ps5.patch new file mode 100644 index 000000000..e8b4fa479 --- /dev/null +++ b/ports/ps5/love/0001-love-11.5-ps5.patch @@ -0,0 +1,81 @@ +diff -ru a/src/libraries/luasocket/libluasocket/compat.c b/src/libraries/luasocket/libluasocket/compat.c +--- a/src/libraries/luasocket/libluasocket/compat.c 2021-12-29 13:34:40.000000000 +0100 ++++ b/src/libraries/luasocket/libluasocket/compat.c 2026-11-14 14:29:42.311122885 +0100 +@@ -1,6 +1,6 @@ + #include "compat.h" + +-#if LUA_VERSION_NUM==501 ++#if 0 + /* + ** Adapted from Lua 5.2 + */ +diff -ru a/src/modules/filesystem/Filesystem.cpp b/src/modules/filesystem/Filesystem.cpp +--- a/src/modules/filesystem/Filesystem.cpp 2023-12-03 13:50:33.000000000 +0100 ++++ b/src/modules/filesystem/Filesystem.cpp 2026-11-14 14:29:42.309548501 +0100 +@@ -20,6 +20,8 @@ + + // LOVE + #include "Filesystem.h" ++ ++#include + #include "common/utf8.h" + + // Assume POSIX or Visual Studio. +@@ -92,7 +94,13 @@ + + std::string Filesystem::getExecutablePath() const + { +-#if defined(LOVE_MACOSX) ++#if defined(__PROSPERO__) ++ // The PS5 launcher runs a bundled game.love from the read-only title ++ // folder; reporting it as the executable makes boot.lua mount it as a ++ // fused game (save directory /, no "love" folder). ++ const char *game = getenv("LOVE_PS5_GAME"); ++ return (game != nullptr && *game != 0) ? game : "/app0/game.love"; ++#elif defined(LOVE_MACOSX) + return love::macosx::getExecutablePath(); + #elif defined(LOVE_IOS) + return love::ios::getExecutablePath(); +diff -ru a/src/modules/love/love.cpp b/src/modules/love/love.cpp +--- a/src/modules/love/love.cpp 2023-12-03 13:50:33.000000000 +0100 ++++ b/src/modules/love/love.cpp 2026-11-14 14:29:42.304244075 +0100 +@@ -468,7 +468,9 @@ + lua_pushcfunction(L, w_love_isVersionCompatible); + lua_setfield(L, -2, "isVersionCompatible"); + +-#ifdef LOVE_WINDOWS_UWP ++#if defined(__PROSPERO__) ++ lua_pushstring(L, "PS5"); ++#elif defined(LOVE_WINDOWS_UWP) + lua_pushstring(L, "UWP"); + #elif LOVE_WINDOWS + lua_pushstring(L, "Windows"); +diff -ru a/src/modules/system/System.cpp b/src/modules/system/System.cpp +--- a/src/modules/system/System.cpp 2023-12-03 13:50:33.000000000 +0100 ++++ b/src/modules/system/System.cpp 2026-11-14 14:29:42.301936179 +0100 +@@ -64,7 +64,9 @@ + + std::string System::getOS() const + { +-#if defined(LOVE_MACOSX) ++#if defined(__PROSPERO__) ++ return "PS5"; ++#elif defined(LOVE_MACOSX) + return "OS X"; + #elif defined(LOVE_IOS) + return "iOS"; +@@ -91,7 +93,13 @@ + bool System::openURL(const std::string &url) const + { + +-#if defined(LOVE_MACOSX) ++#if defined(__PROSPERO__) ++ ++ // A PS5 title has no browser to hand a URL to (and no posix_spawn). ++ (void) url; ++ return false; ++ ++#elif defined(LOVE_MACOSX) + + bool success = false; + CFURLRef cfurl = CFURLCreateWithBytes(nullptr, diff --git a/ports/ps5/love/0002-physfs-shared-modes.patch b/ports/ps5/love/0002-physfs-shared-modes.patch new file mode 100644 index 000000000..6d23369e7 --- /dev/null +++ b/ports/ps5/love/0002-physfs-shared-modes.patch @@ -0,0 +1,25 @@ +--- a/src/libraries/physfs/physfs_platform_posix.c ++++ b/src/libraries/physfs/physfs_platform_posix.c +@@ -149,8 +149,12 @@ + + int __PHYSFS_platformMkDir(const char *path) + { +- const int rc = mkdir(path, S_IRWXU); ++ /* PS5: the title runs as its own user and ftpsrv as another, and players ++ copy ROMs into (and saves out of) these folders over FTP, so they are ++ created open to everyone, whatever the umask. */ ++ const int rc = mkdir(path, 0777); + BAIL_IF(rc == -1, errcodeFromErrno(), 0); ++ chmod(path, 0777); + return 1; + } /* __PHYSFS_platformMkDir */ + +@@ -182,7 +186,7 @@ + #endif + + do { +- fd = open(filename, mode, S_IRUSR | S_IWUSR); ++ fd = open(filename, mode, 0666); /* PS5: see __PHYSFS_platformMkDir */ + } while ((fd < 0) && (errno == EINTR)); + BAIL_IF(fd < 0, errcodeFromErrno(), NULL); + diff --git a/ports/ps5/sce_sys/icon0.png b/ports/ps5/sce_sys/icon0.png new file mode 100644 index 000000000..82a48cc54 Binary files /dev/null and b/ports/ps5/sce_sys/icon0.png differ diff --git a/ports/ps5/sce_sys/param.json b/ports/ps5/sce_sys/param.json new file mode 100644 index 000000000..eccbf0827 --- /dev/null +++ b/ports/ps5/sce_sys/param.json @@ -0,0 +1,47 @@ +{ + "ageLevel": { + "default": 0 + }, + "amm": { + "multimapVaRangeInGib": 512, + "pagetableMemorySizeInMib": 256, + "vaRangeInGib": 512 + }, + "applicationCategoryType": 0, + "applicationDrmType": "free", + "attribute": 0, + "attribute2": 0, + "attribute3": 0, + "conceptId": "42002", + "contentBadgeType": 1, + "contentId": "UP9000-PPSA42002_00-GEN2RECOMPED0001", + "contentVersion": "01.000.000", + "downloadDataSize": 256, + "gameIntent": { + "permittedIntents": [ + { + "intentType": "launchActivity" + } + ] + }, + "kernel": { + "cpuPageTableSize": 268435456, + "gpuPageTableSize": 268435456 + }, + "localizedParameters": { + "defaultLanguage": "en-US", + "en-US": { + "titleName": "Gen2Recomped" + } + }, + "masterVersion": "01.00", + "pubtools": { + "creationDate": "2026-10-19 00:00:00", + "loudnessSnd0": "-28.00", + "toolVersion": "2.00" + }, + "requiredSystemSoftwareVersion": "0x0000000000000000", + "sdkVersion": "0x0000000000000000", + "titleId": "PPSA42002", + "versionFileUri": "" +} diff --git a/ports/ps5/sce_sys/pic0.png b/ports/ps5/sce_sys/pic0.png new file mode 100644 index 000000000..83270be92 Binary files /dev/null and b/ports/ps5/sce_sys/pic0.png differ diff --git a/ports/ps5/sce_sys/pic1.png b/ports/ps5/sce_sys/pic1.png new file mode 100644 index 000000000..83270be92 Binary files /dev/null and b/ports/ps5/sce_sys/pic1.png differ diff --git a/ports/ps5/sdl/CMakeLists.txt b/ports/ps5/sdl/CMakeLists.txt new file mode 100644 index 000000000..2511f8be5 --- /dev/null +++ b/ports/ps5/sdl/CMakeLists.txt @@ -0,0 +1,51 @@ +# Gen2Recomped PS5: a static SDL2 for LOVE in a native PS5 title. +# +# SDL 2.30.12 from ps5-payload-dev/SDL (scripts/ps5/deps.sh pins the commit), +# with its OSMesa video driver swapped for ps5-opengl's SDL2 video driver +# ("G19", integration/SDL2/SDL_ps5g19.c, patched by g19-love.patch so any +# window LOVE asks for becomes the fixed scanout size). Adapted from that +# integration's CMakeLists.txt, with two differences: SDL's PS5 audio driver +# (SceAudioOut) stays on, because OpenAL Soft plays through SDL audio here, and +# the game-controller layer stays on for love.joystick's gamepad API. +# +# Configured by scripts/ps5/build_runtime.sh with the payload SDK's +# prospero-cmake (toolchain/prospero.cmake sets PS5 and FreeBSD). +cmake_minimum_required(VERSION 3.16) +project(G2RSDL2 C) + +if(NOT SDL_SOURCE OR NOT G19_SOURCE OR NOT PS5_OPENGL_PREFIX) + message(FATAL_ERROR "set SDL_SOURCE, G19_SOURCE and PS5_OPENGL_PREFIX") +endif() + +set(SDL_SHARED OFF CACHE BOOL "" FORCE) +set(SDL_STATIC ON CACHE BOOL "" FORCE) +set(SDL_STATIC_PIC ON CACHE BOOL "" FORCE) +set(SDL_TEST OFF CACHE BOOL "" FORCE) +set(SDL_TESTS OFF CACHE BOOL "" FORCE) +set(SDL_VIRTUAL_JOYSTICK OFF CACHE BOOL "" FORCE) +set(SDL_HIDAPI OFF CACHE BOOL "" FORCE) +set(SDL_HIDAPI_JOYSTICK OFF CACHE BOOL "" FORCE) +set(SDL_PTHREADS ON CACHE BOOL "" FORCE) +set(SDL2_DISABLE_SDL2MAIN ON CACHE BOOL "" FORCE) +set(SDL_AUDIO ON CACHE BOOL "" FORCE) +set(SDL_JOYSTICK ON CACHE BOOL "" FORCE) +# OPENGL stays OFF at SDL's level: the G19 driver defines SDL_VIDEO_OPENGL for +# itself and talks to ps5-opengl's EGL directly (static, no dlopen). +foreach(feature RENDER HAPTIC SENSOR OPENGL OPENGLES VULKAN X11 WAYLAND KMSDRM + OFFSCREEN JACK PULSEAUDIO DBUS IBUS FCITX LIBUDEV LIBSAMPLERATE) + set(SDL_${feature} OFF CACHE BOOL "" FORCE) +endforeach() +# The title's libc does not export these; use SDL's own fallbacks. +foreach(symbol WCSLCPY WCSLCAT WCSCASECMP WCSNCASECMP) + set(HAVE_${symbol} 0 CACHE INTERNAL "native libc contract" FORCE) +endforeach() + +add_subdirectory("${SDL_SOURCE}" sdl) + +get_target_property(sources SDL2-static SOURCES) +list(FILTER sources EXCLUDE REGEX "/video/ps5/") +set_property(TARGET SDL2-static PROPERTY SOURCES "${sources}") +target_sources(SDL2-static PRIVATE "${G19_SOURCE}/SDL_ps5g19.c") +target_include_directories(SDL2-static PRIVATE "${SDL_SOURCE}/src" "${G19_SOURCE}" + "${PS5_OPENGL_PREFIX}/include") +target_compile_definitions(SDL2-static PRIVATE SDL_VIDEO_DRIVER_PS5=1 SDL_VIDEO_OPENGL=1) diff --git a/ports/ps5/sdl/g19-love.patch b/ports/ps5/sdl/g19-love.patch new file mode 100644 index 000000000..92ed16ced --- /dev/null +++ b/ports/ps5/sdl/g19-love.patch @@ -0,0 +1,30 @@ +Gen2Recomped PS5: let the G19 SDL2 video driver take LOVE's window. + +LOVE creates its window at the size conf.lua asks for (and resizable on +desktop builds); G19 refused anything but the fixed scanout size. On the PS5 +there is exactly one surface, so take any OpenGL window and make it the +scanout size, the way SDL's fullscreen-desktop mode reports the display size. +LOVE then reads the real size back with SDL_GetWindowSize. + +--- a/SDL_ps5g19.c ++++ b/SDL_ps5g19.c +@@ -80,11 +80,14 @@ + EGL_STENCIL_SIZE, _this->gl_config.stencil_size, EGL_NONE + }; + if (on_thread(g) < 0) return -1; +- if (g->window || g->surface || window->w != PS5_OPENGL_NATIVE_WIDTH || +- window->h != PS5_OPENGL_NATIVE_HEIGHT || +- !(window->flags & SDL_WINDOW_OPENGL) || (window->flags & SDL_WINDOW_RESIZABLE)) +- return SDL_SetError("G19 requires one fixed %dx%d OpenGL window", +- PS5_OPENGL_NATIVE_WIDTH, PS5_OPENGL_NATIVE_HEIGHT); ++ if (g->window || g->surface || !(window->flags & SDL_WINDOW_OPENGL)) ++ return SDL_SetError("G19 requires one OpenGL window"); ++ /* Gen2Recomped: whatever size was asked for, the window is the scanout. */ ++ window->x = window->windowed.x = 0; ++ window->y = window->windowed.y = 0; ++ window->w = window->windowed.w = PS5_OPENGL_NATIVE_WIDTH; ++ window->h = window->windowed.h = PS5_OPENGL_NATIVE_HEIGHT; ++ window->flags &= ~SDL_WINDOW_RESIZABLE; + if (_this->gl_config.stereo || _this->gl_config.multisamplebuffers || + _this->gl_config.multisamplesamples || _this->gl_config.floatbuffers || + _this->gl_config.framebuffer_srgb_capable || _this->gl_config.accum_red_size || diff --git a/ports/ps5/testgame/conf.lua b/ports/ps5/testgame/conf.lua new file mode 100644 index 000000000..cf7f1c3ca --- /dev/null +++ b/ports/ps5/testgame/conf.lua @@ -0,0 +1,10 @@ +-- PS5 runtime bring-up game (scripts/build_ps5.sh --test-game). Not shipped. +function love.conf(t) + t.identity = "Gen2RecompPS5Test" + t.version = "11.5" + t.window.title = "Gen2Recomped PS5 runtime test" + t.window.width = 1280 + t.window.height = 720 + t.window.resizable = true + t.window.vsync = 1 +end diff --git a/ports/ps5/testgame/main.lua b/ports/ps5/testgame/main.lua new file mode 100644 index 000000000..7764030b8 --- /dev/null +++ b/ports/ps5/testgame/main.lua @@ -0,0 +1,181 @@ +-- PS5 runtime bring-up: video, shaders, canvas, pad, audio, save directory, +-- LuaJIT (JIT + FFI, including ffi.C.PHYSFS_*). Everything it finds is drawn +-- and printed (the launcher sends print to /love.log). + +local lines = {} +local function note(fmt, ...) + local s = string.format(fmt, ...) + lines[#lines + 1] = s + print("[test] " .. s) +end + +local tone, shader, canvas, font +local pressed = {} +local frames, fpsShown = 0, 0 +local axes = "" + +function love.load() + font = love.graphics.newFont(28) + note("OS %s / love._os %s", love.system.getOS(), tostring(love._os)) + local name, version, vendor, device = love.graphics.getRendererInfo() + note("renderer %s | %s | %s | %s", name, version, vendor, device) + note("window %dx%d, dpi %.2f", love.graphics.getWidth(), love.graphics.getHeight(), love.graphics.getDPIScale()) + local lim = love.graphics.getSystemLimits() + note("texturesize %d, canvasmsaa %d", lim.texturesize, lim.canvasmsaa) + note("fused %s, source %s", tostring(love.filesystem.isFused()), love.filesystem.getSource()) + note("save dir %s", love.filesystem.getSaveDirectory()) + + -- save directory + note("user dir %s, appdata dir %s", love.filesystem.getUserDirectory(), love.filesystem.getAppdataDirectory()) + note("os.date %s, os.time %s", tostring(os.date()), tostring(os.time())) + local okW, ok, err = pcall(love.filesystem.write, "hello.txt", "written at " .. os.date() .. "\n") + note("write hello.txt: %s %s %s", tostring(okW), tostring(ok), tostring(err or "")) + pcall(function() + local ffi = require("ffi") + ffi.cdef("int PHYSFS_getLastErrorCode(void); const char *PHYSFS_getErrorByCode(int code); const char *PHYSFS_getWriteDir(void);") + local code = ffi.C.PHYSFS_getLastErrorCode() + local wd = ffi.C.PHYSFS_getWriteDir() + local function str(p) return p ~= nil and ffi.string(p) or "null" end + note("PHYSFS last error %d (%s), write dir %s", code, str(ffi.C.PHYSFS_getErrorByCode(code)), str(wd)) + end) + note("read back: %s", tostring(love.filesystem.read("hello.txt")):gsub("\n", "")) + love.filesystem.createDirectory("imports") + note("imports dir exists: %s", tostring(love.filesystem.getInfo("imports") ~= nil)) + + -- LuaJIT + if jit then + note("LuaJIT %s, jit.status %s, arch %s, os %s", jit.version, tostring((jit.status())), jit.arch, jit.os) + local t0 = love.timer.getTime() + local acc = 0 + for i = 1, 20000000 do acc = acc + (i % 7) * 0.5 end + note("20M-iteration loop: %.3f s (acc %.0f)", love.timer.getTime() - t0, acc) + end + local okffi, ffi = pcall(require, "ffi") + if okffi then + pcall(ffi.cdef, "int PHYSFS_mount(const char *newDir, const char *mountPoint, int appendToPath);") + pcall(ffi.cdef, "char **PHYSFS_getSearchPath(void); void PHYSFS_freeList(void *listVar);") + pcall(ffi.cdef, "int getpid(void);") + local okS, list = pcall(function() return ffi.C.PHYSFS_getSearchPath() end) + if okS and list ~= nil then + local i, dirs = 0, {} + while list[i] ~= nil do dirs[#dirs + 1] = ffi.string(list[i]); i = i + 1 end + ffi.C.PHYSFS_freeList(list) + note("ffi.C.PHYSFS_getSearchPath: %s", table.concat(dirs, ", ")) + else + note("ffi.C.PHYSFS_getSearchPath FAILED: %s", tostring(list)) + end + local okP, pid = pcall(function() return ffi.C.getpid() end) + note("ffi.C.getpid: %s %s", tostring(okP), tostring(pid)) + local okL, e = pcall(ffi.load, "love") + note("ffi.load('love') fails cleanly: %s (%s)", tostring(not okL), tostring(e)) + end + + -- C++ exception -> Lua error (love::Exception through luax_catchexcept, + -- then LuaJIT's unwinder): needs working eh_frame/libunwind in the title + -- and the static C++ runtime (scripts/ps5/build_runtime.sh, cxx_own). + local okE, errE = pcall(love.graphics.newImage, "does/not/exist.png") + note("C++ exception caught as Lua error: %s (%s)", tostring(not okE), tostring(errE):sub(1, 60)) + local okT, errT = pcall(function() error({code = 7}) end) + note("Lua error with table value: %s", tostring(not okT and type(errT) == "table" and errT.code == 7)) + + -- a love.thread worker + local th = love.thread.newThread([[ + local ch = love.thread.getChannel("ps5test") + local acc = 0 + for i = 1, 1000000 do acc = acc + i end + ch:push(acc) + ]]) + th:start() + note("thread result: %s", tostring(love.thread.getChannel("ps5test"):demand(5))) + + -- shader + canvas + local okSh, sh = pcall(love.graphics.newShader, [[ + extern number time; + vec4 effect(vec4 color, Image tex, vec2 tc, vec2 sc) { + return vec4(0.5 + 0.5 * sin(time + tc.x * 6.0), 0.5 + 0.5 * sin(time * 1.3 + tc.y * 6.0), 0.8, 1.0) * color; + } + ]]) + note("shader: %s %s", tostring(okSh), okSh and "" or tostring(sh)) + if okSh then shader = sh end + canvas = love.graphics.newCanvas(320, 180) + + -- audio: a 440 Hz tone, played on any pad button + local rate, len = 48000, 0.25 + local sd = love.sound.newSoundData(math.floor(rate * len), rate, 16, 1) + for i = 0, sd:getSampleCount() - 1 do + local env = math.min(1, (sd:getSampleCount() - i) / 2000) + sd:setSample(i, math.sin(2 * math.pi * 440 * i / rate) * 0.4 * env) + end + local okA, src = pcall(love.audio.newSource, sd) + note("audio source: %s, active devices %s", tostring(okA), tostring(love.audio.getActiveSourceCount and love.audio.getActiveSourceCount())) + if okA then tone = src; tone:play() end + + local js = love.joystick.getJoysticks() + note("joysticks: %d", #js) + for _, j in ipairs(js) do + note(" %s guid %s gamepad %s", j:getName(), j:getGUID(), tostring(j:isGamepad())) + end +end + +function love.joystickadded(j) + note("joystick added: %s gamepad %s", j:getName(), tostring(j:isGamepad())) +end + +function love.gamepadpressed(j, b) + pressed[#pressed + 1] = b + if #pressed > 12 then table.remove(pressed, 1) end + print("[test] gamepadpressed " .. b) + if tone then tone:stop(); tone:play() end +end + +function love.joystickpressed(j, b) + if not j:isGamepad() then + pressed[#pressed + 1] = "btn" .. b + print("[test] joystickpressed " .. b) + if tone then tone:stop(); tone:play() end + end +end + +local t = 0 +function love.update(dt) + t = t + dt + frames = frames + 1 + local js = love.joystick.getJoysticks()[1] + if js and js:isGamepad() then + axes = string.format("LX %.2f LY %.2f RX %.2f RY %.2f L2 %.2f R2 %.2f", + js:getGamepadAxis("leftx"), js:getGamepadAxis("lefty"), + js:getGamepadAxis("rightx"), js:getGamepadAxis("righty"), + js:getGamepadAxis("triggerleft"), js:getGamepadAxis("triggerright")) + end + if math.floor(t) ~= math.floor(t - dt) then + fpsShown = love.timer.getFPS() + if math.floor(t) % 10 == 0 then print(string.format("[test] t=%d fps=%d", math.floor(t), fpsShown)) end + end +end + +function love.draw() + local w, h = love.graphics.getDimensions() + love.graphics.clear(0.08, 0.1, 0.16) + love.graphics.setCanvas(canvas) + love.graphics.clear(0, 0, 0, 1) + if shader then shader:send("time", t); love.graphics.setShader(shader) end + love.graphics.rectangle("fill", 0, 0, 320, 180) + love.graphics.setShader() + love.graphics.setCanvas() + love.graphics.setColor(1, 1, 1) + love.graphics.draw(canvas, w - 340, 20) + for i = 0, 5 do + love.graphics.setColor(0.5 + 0.5 * math.sin(t + i), 0.4, 0.9 - i * 0.1) + love.graphics.circle("fill", w - 170 + math.cos(t * 2 + i) * 120, 360 + math.sin(t * 2 + i) * 120, 24) + end + love.graphics.setColor(1, 1, 1) + love.graphics.setFont(font) + local y = 20 + for _, s in ipairs(lines) do + love.graphics.print(s, 20, y) + y = y + 34 + end + love.graphics.print(string.format("FPS %d frames %d", fpsShown, frames), 20, h - 120) + love.graphics.print("pad: " .. table.concat(pressed, " "), 20, h - 85) + love.graphics.print(axes, 20, h - 50) +end diff --git a/ports/ps5/tools/png2dds.c b/ports/ps5/tools/png2dds.c new file mode 100644 index 000000000..b60ea98a8 --- /dev/null +++ b/ports/ps5/tools/png2dds.c @@ -0,0 +1,246 @@ +/* png2dds: PNG -> 3840x2160 BC7_UNORM DX10 DDS for a PS5 title's + * sce_sys/pic0.dds and pic1.dds (single surface, no mipmaps; the layout the + * ps5-native-app-boilerplate's validate-assets.sh accepts). + * + * png2dds + * + * Built for the host by scripts/build_ps5.sh against LOVE's bundled + * stb_image.h. The input is scaled to 3840x2160 (bilinear) if it has another + * size. Every 4x4 block is BC7 mode 6 (one subset, RGBA 7.7.7.7 endpoints + * with p-bits, 4-bit indices), endpoints along the block's principal colour + * axis: not texconv quality, but plenty for a home-screen background, and no + * Windows tool or extra download. */ +#include +#include +#include +#include +#include + +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_PNG +#include "stb_image.h" + +enum { W = 3840, H = 2160 }; + +static const int weights[16] = {0, 4, 9, 13, 17, 21, 26, 30, 34, 38, 43, 47, 51, 55, 60, 64}; + +typedef struct { + uint64_t lo, hi; + int pos; +} Bits; + +static void put(Bits *b, uint32_t value, int count) +{ + for (int i = 0; i < count; i++, b->pos++) { + uint64_t bit = (value >> i) & 1; + if (b->pos < 64) + b->lo |= bit << b->pos; + else + b->hi |= bit << (b->pos - 64); + } +} + +/* 8-bit endpoint from 7 bits + p-bit; the best (7-bit, p) pair for a target. */ +static void quantize(const float target[4], int p, int out7[4]) +{ + for (int c = 0; c < 4; c++) { + int v = (int)lrintf((target[c] - p) / 2.0f); + out7[c] = v < 0 ? 0 : v > 127 ? 127 : v; + } +} + +static int expand(int v7, int p) { return (v7 << 1) | p; } + +static float encode_block(const uint8_t px[16][4], Bits *out) +{ + float mean[4] = {0}, cov[4][4] = {{0}}; + for (int i = 0; i < 16; i++) + for (int c = 0; c < 4; c++) + mean[c] += px[i][c] / 16.0f; + for (int i = 0; i < 16; i++) + for (int a = 0; a < 4; a++) + for (int b = 0; b < 4; b++) + cov[a][b] += (px[i][a] - mean[a]) * (px[i][b] - mean[b]); + float axis[4] = {1, 1, 1, 0.5f}; + for (int it = 0; it < 8; it++) { + float next[4] = {0}, len = 0; + for (int a = 0; a < 4; a++) + for (int b = 0; b < 4; b++) + next[a] += cov[a][b] * axis[b]; + for (int a = 0; a < 4; a++) + len += next[a] * next[a]; + if (len < 1e-6f) + break; + len = sqrtf(len); + for (int a = 0; a < 4; a++) + axis[a] = next[a] / len; + } + float lo = 1e9f, hi = -1e9f; + for (int i = 0; i < 16; i++) { + float t = 0; + for (int c = 0; c < 4; c++) + t += (px[i][c] - mean[c]) * axis[c]; + lo = t < lo ? t : lo; + hi = t > hi ? t : hi; + } + float e[2][4]; + for (int c = 0; c < 4; c++) { + e[0][c] = fminf(255, fmaxf(0, mean[c] + axis[c] * lo)); + e[1][c] = fminf(255, fmaxf(0, mean[c] + axis[c] * hi)); + } + + float best_err = 1e30f; + int best_q[2][4] = {{0}}, best_p[2] = {0}, best_idx[16] = {0}; + for (int p0 = 0; p0 < 2; p0++) { + for (int p1 = 0; p1 < 2; p1++) { + int q[2][4], ep[2][4]; + quantize(e[0], p0, q[0]); + quantize(e[1], p1, q[1]); + for (int c = 0; c < 4; c++) { + ep[0][c] = expand(q[0][c], p0); + ep[1][c] = expand(q[1][c], p1); + } + int pal[16][4]; + for (int k = 0; k < 16; k++) + for (int c = 0; c < 4; c++) + pal[k][c] = ((64 - weights[k]) * ep[0][c] + weights[k] * ep[1][c] + 32) >> 6; + float err = 0; + int idx[16]; + for (int i = 0; i < 16; i++) { + int bk = 0, be = 1 << 30; + for (int k = 0; k < 16; k++) { + int d = 0; + for (int c = 0; c < 4; c++) { + int x = pal[k][c] - px[i][c]; + d += x * x; + } + if (d < be) { + be = d; + bk = k; + } + } + idx[i] = bk; + err += be; + } + if (err < best_err) { + best_err = err; + memcpy(best_q, q, sizeof(q)); + best_p[0] = p0; + best_p[1] = p1; + memcpy(best_idx, idx, sizeof(idx)); + } + } + } + /* The anchor (pixel 0) index is stored with 3 bits: its top bit must be 0. */ + if (best_idx[0] & 8) { + for (int c = 0; c < 4; c++) { + int t = best_q[0][c]; + best_q[0][c] = best_q[1][c]; + best_q[1][c] = t; + } + int t = best_p[0]; + best_p[0] = best_p[1]; + best_p[1] = t; + for (int i = 0; i < 16; i++) + best_idx[i] = 15 - best_idx[i]; + } + memset(out, 0, sizeof(*out)); + put(out, 1u << 6, 7); /* mode 6 */ + for (int c = 0; c < 4; c++) { + put(out, (uint32_t)best_q[0][c], 7); + put(out, (uint32_t)best_q[1][c], 7); + } + put(out, (uint32_t)best_p[0], 1); + put(out, (uint32_t)best_p[1], 1); + put(out, (uint32_t)best_idx[0], 3); + for (int i = 1; i < 16; i++) + put(out, (uint32_t)best_idx[i], 4); + return best_err; +} + +static void put32(uint8_t *p, uint32_t v) +{ + p[0] = v & 255; + p[1] = (v >> 8) & 255; + p[2] = (v >> 16) & 255; + p[3] = v >> 24; +} + +int main(int argc, char **argv) +{ + if (argc != 3) { + fprintf(stderr, "usage: png2dds \n"); + return 2; + } + int sw, sh, n; + uint8_t *src = stbi_load(argv[1], &sw, &sh, &n, 4); + if (!src) { + fprintf(stderr, "png2dds: cannot read %s: %s\n", argv[1], stbi_failure_reason()); + return 1; + } + uint8_t *img = malloc((size_t)W * H * 4); + for (int y = 0; y < H; y++) { + float fy = (y + 0.5f) * sh / H - 0.5f; + int y0 = (int)floorf(fy); + float ty = fy - y0; + int ya = y0 < 0 ? 0 : y0, yb = y0 + 1 >= sh ? sh - 1 : y0 + 1; + for (int x = 0; x < W; x++) { + float fx = (x + 0.5f) * sw / W - 0.5f; + int x0 = (int)floorf(fx); + float tx = fx - x0; + int xa = x0 < 0 ? 0 : x0, xb = x0 + 1 >= sw ? sw - 1 : x0 + 1; + for (int c = 0; c < 4; c++) { + float a = src[((size_t)ya * sw + xa) * 4 + c], b = src[((size_t)ya * sw + xb) * 4 + c]; + float d = src[((size_t)yb * sw + xa) * 4 + c], e = src[((size_t)yb * sw + xb) * 4 + c]; + float v = (a + (b - a) * tx) * (1 - ty) + (d + (e - d) * tx) * ty; + img[((size_t)y * W + x) * 4 + c] = (uint8_t)lrintf(v); + } + } + } + stbi_image_free(src); + + const size_t data_size = (size_t)W * H; /* 16 bytes per 4x4 block */ + uint8_t header[148] = {0}; + memcpy(header, "DDS ", 4); + put32(header + 4, 124); + put32(header + 8, 0x000A1007); /* caps, height, width, pixelformat, mipmapcount, linearsize */ + put32(header + 12, H); + put32(header + 16, W); + put32(header + 20, (uint32_t)data_size); + put32(header + 28, 1); + put32(header + 76, 32); + put32(header + 80, 4); /* DDPF_FOURCC */ + memcpy(header + 84, "DX10", 4); + put32(header + 108, 0x1000); /* DDSCAPS_TEXTURE */ + put32(header + 128, 98); /* DXGI_FORMAT_BC7_UNORM */ + put32(header + 132, 3); /* D3D10_RESOURCE_DIMENSION_TEXTURE2D */ + put32(header + 140, 1); /* array size */ + + FILE *f = fopen(argv[2], "wb"); + if (!f) { + fprintf(stderr, "png2dds: cannot write %s\n", argv[2]); + return 1; + } + fwrite(header, 1, sizeof(header), f); + double total = 0; + for (int by = 0; by < H; by += 4) { + for (int bx = 0; bx < W; bx += 4) { + uint8_t px[16][4]; + for (int i = 0; i < 16; i++) + memcpy(px[i], img + ((size_t)(by + i / 4) * W + bx + i % 4) * 4, 4); + Bits b; + total += encode_block(px, &b); + uint8_t block[16]; + for (int i = 0; i < 8; i++) { + block[i] = (uint8_t)(b.lo >> (8 * i)); + block[8 + i] = (uint8_t)(b.hi >> (8 * i)); + } + fwrite(block, 1, 16, f); + } + } + fclose(f); + free(img); + printf("png2dds: %s -> %s (%dx%d from %dx%d, RMSE %.2f)\n", argv[1], argv[2], W, H, sw, sh, + sqrt(total / ((double)W * H * 4))); + return 0; +} diff --git a/scripts/build_ps5.sh b/scripts/build_ps5.sh new file mode 100644 index 000000000..8c9fd3b95 --- /dev/null +++ b/scripts/build_ps5.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Gen2Recomped for PS5: build the native title folder (and optionally put it +# on a console). One command: +# +# scripts/build_ps5.sh [--deploy HOST] [--test-game] [--version X.Y.Z] +# [--out DIR] [--clean] +# +# --deploy HOST upload the folder to HOST's /data/homebrew/ over +# FTP (ftpsrv, port $PS5_FTP_PORT or 2121) and ask +# ShadowMountPlus (port 10101) to scan, which registers the +# title on the home screen the first time +# --test-game bundle ports/ps5/testgame (runtime bring-up: video, +# shaders, pad, audio, save dir, JIT/FFI) instead of the game +# --version stamp this release into the packed game.love +# (scripts/pack_love.sh --version) +# --out DIR where the title folder goes (default +# $G2R_PS5_WORK/out/title; the folder is DIR/) +# --clean rebuild the runtime from scratch +# +# Runs on Linux or WSL (ports/ps5/BUILD.md lists the host packages). The +# runtime (LOVE 11.5 + SDL2 + ps5-opengl + LuaJIT, scripts/ps5/build_runtime.sh) +# is built once into $G2R_PS5_WORK (default ~/.cache/gen2recomp-ps5) and reused. +# +# Title folder: +# /eboot.bin launcher + LOVE, development FSELF +# /sce_module/libc.prx the boilerplate's clean-room loader shim +# /sce_sys/param.json from ports/ps5/sce_sys +# /sce_sys/icon0.png from ports/ps5/sce_sys +# /sce_sys/pic0.dds ports/ps5/sce_sys/pic0.png as BC7 DDS +# /sce_sys/pic1.dds ports/ps5/sce_sys/pic1.png as BC7 DDS +# /game.love scripts/pack_love.sh's payload +set -euo pipefail + +repo=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +# shellcheck source=ps5/deps.sh +source "$repo/scripts/ps5/deps.sh" + +deploy="" +test_game=0 +version="" +out="" +runtime_args=() +while [ $# -gt 0 ]; do + case $1 in + --deploy) [ $# -ge 2 ] || ps5_die "--deploy needs a host"; deploy=$2; shift 2 ;; + --test-game) test_game=1; shift ;; + --version) [ $# -ge 2 ] || ps5_die "--version needs X.Y.Z"; version=$2; shift 2 ;; + --out) [ $# -ge 2 ] || ps5_die "--out needs a directory"; out=$2; shift 2 ;; + --clean) runtime_args+=(--clean); shift ;; + -h|--help) sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) ps5_die "unknown argument: $1" ;; + esac +done +[ "$(uname -s)" = Linux ] || ps5_die "run this on Linux or in WSL (wsl bash scripts/build_ps5.sh)" + +sce_sys=$repo/ports/ps5/sce_sys +[ -f "$sce_sys/param.json" ] || ps5_die "missing $sce_sys/param.json" +title_id=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["titleId"])' "$sce_sys/param.json") +[[ $title_id =~ ^PPSA[0-9]{5}$ ]] || ps5_die "param.json titleId '$title_id' is not PPSA#####" + +# ---------------------------------------------------------------- runtime +bash "$repo/scripts/ps5/build_runtime.sh" "${runtime_args[@]}" +runtime=$G2R_PS5_WORK/out/runtime + +# ---------------------------------------------------------------- game.love +stage=$G2R_PS5_WORK/out +mkdir -p "$stage" +love="$stage/game.love" +rm -f "$love" +if [ $test_game = 1 ]; then + ps5_say "packing ports/ps5/testgame" + (cd "$repo/ports/ps5/testgame" && zip -q -r -X "$love" . -x '.*') +else + pack=(bash "$repo/scripts/pack_love.sh" --output "$love" --listing "$stage/game.love.txt") + [ -n "$version" ] && pack+=(--version "$version") + "${pack[@]}" +fi + +# ---------------------------------------------------------------- folder +out=${out:-$G2R_PS5_WORK/out/title} +app=$out/$title_id +rm -rf "$app" +mkdir -p "$app/sce_sys" "$app/sce_module" +cp "$runtime/eboot.bin" "$app/eboot.bin" +cp "$runtime/libc.prx" "$app/sce_module/libc.prx" +cp "$sce_sys/param.json" "$app/sce_sys/param.json" +[ -f "$sce_sys/icon0.png" ] && cp "$sce_sys/icon0.png" "$app/sce_sys/icon0.png" +cp "$love" "$app/game.love" + +# pic0/pic1: the shell wants 3840x2160 BC7 DX10 DDS; sce_sys keeps PNGs. +if [ -f "$sce_sys/pic0.png" ] && [ -f "$sce_sys/pic1.png" ]; then + tool=$G2R_PS5_WORK/build/png2dds + src=$repo/ports/ps5/tools/png2dds.c + stb=$G2R_PS5_WORK/build/love/src/libraries/stb + if [ ! -x "$tool" ] || [ "$src" -nt "$tool" ]; then + cc -O2 -w -I"$stb" -o "$tool" "$src" -lm || ps5_die "png2dds build failed" + fi + for pic in pic0 pic1; do + "$tool" "$sce_sys/$pic.png" "$app/sce_sys/$pic.dds" + done +fi + +ps5_say "title folder: $app" +du -sh "$app" | sed 's/^/ps5: /' + +# ---------------------------------------------------------------- deploy +if [ -n "$deploy" ]; then + port=${PS5_FTP_PORT:-2121} + base="ftp://$deploy:$port/data/homebrew/$title_id" + ps5_say "uploading to $base" + # eboot.bin and param.json last, so a half-copied folder never looks like + # a title to ShadowMountPlus. + (cd "$app" && find . -type f ! -name eboot.bin ! -path ./sce_sys/param.json | sort) | + while read -r rel; do + rel=${rel#./} + curl -sS --ftp-create-dirs -T "$app/$rel" "$base/$rel" || ps5_die "upload failed: $rel" + done + for rel in eboot.bin sce_sys/param.json; do + curl -sS --ftp-create-dirs -T "$app/$rel" "$base/$rel" || ps5_die "upload failed: $rel" + done + ps5_say "asking ShadowMountPlus to scan" + curl -sS -m 10 -X POST -H 'Content-Type: application/json' -d '{"reset_attempts":true}' \ + "http://$deploy:10101/api/v1/scan" && echo || + ps5_say "WARNING: no answer from ShadowMountPlus on $deploy:10101; scan from its menu" + ps5_say "deployed. ShadowMountPlus defers the scan while a homebrew title runs: close it first." +fi diff --git a/scripts/ps5/build_runtime.sh b/scripts/ps5/build_runtime.sh new file mode 100644 index 000000000..d86180827 --- /dev/null +++ b/scripts/ps5/build_runtime.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# Builds the PS5 runtime of Gen2Recomped: LOVE 11.5 and its libraries, +# statically linked with the launcher into a native PS5 title executable +# (eboot.bin, a development FSELF) plus sce_module/libc.prx. +# +# scripts/ps5/build_runtime.sh [--clean] +# +# Linux or WSL (Ubuntu 26.04 tested) with clang-18, clang, cmake, ninja, make, +# autotools, python3, curl, git. Every input is pinned in scripts/ps5/deps.sh +# and fetched into $G2R_PS5_WORK (default ~/.cache/gen2recomp-ps5); the build +# trees are there too. Output: $G2R_PS5_WORK/out/runtime/{eboot.bin,libc.prx}. +# Each stage is skipped when its output is newer than its inputs; --clean +# rebuilds everything. +# +# The stack (ports/ps5/BUILD.md has the why): +# ps5-opengl SDK 1.0.2 EGL + OpenGL 4.6 core / compatibility (Mesa 26.2 on AGC) +# SDL 2.30.12 (PS5 fork) + ps5-opengl's G19 video driver, PS5 audio + pad +# LuaJIT 2.1 FFI on, system malloc, JIT decided at run time +# LOVE 11.5 + ports/ps5/love/*.patch +# PacBrew v0.40.2 freetype, openal-soft (SDL2 backend), vorbis, +# ogg, theora, modplug, mpg123, png, zlib, bzip2 +# boilerplate CRT, libc.prx, ELF -> FSELF converter +set -euo pipefail + +repo=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +# shellcheck source=deps.sh +source "$repo/scripts/ps5/deps.sh" + +clean=0 +for a in "$@"; do + case $a in + --clean) clean=1 ;; + -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) ps5_die "unknown argument: $a" ;; + esac +done + +for tool in git curl make cmake ninja python3 clang-18 autoconf patch; do + command -v "$tool" >/dev/null || ps5_die "missing host tool: $tool" +done + +ps5_fetch_all + +W=$G2R_PS5_WORK +S=$W/src +B=$W/build +O=$W/out/runtime +[ $clean = 1 ] && rm -rf "$B" "$O" +mkdir -p "$B" "$O" + +BP=$S/boilerplate +GLROOT=$S/ps5-opengl-sdk-1.0.2 +GL=$GLROOT/sdk +G19=$GLROOT/src/ps5-opengl/integration/SDL2 +NATIVE=$GLROOT/src/ps5-opengl/native-app +PB=$S/pacbrew-0.40.2/homebrew +JOBS=${JOBS:-$(nproc)} + +# A private copy of the payload SDK (hard links), with ps5-opengl's AGC import +# stubs added, so the converter sees them and the boilerplate stays untouched. +SDK=$B/sdk +if [ ! -f "$SDK/.ready" ]; then + rm -rf "$SDK" + cp -al "$BP/.deps/native/ps5-payload-sdk" "$SDK" + cp "$GL/lib/libSceAgc.so" "$GL/lib/libSceAgcDriver.so" "$SDK/target/lib/" + touch "$SDK/.ready" +fi +CC=$SDK/bin/prospero-clang +CXX=$SDK/bin/prospero-clang++ +AR=$SDK/bin/prospero-ar +CFLAGS_COMMON="-O2 -ffunction-sections -fdata-sections" +CXXFLAGS_COMMON="$CFLAGS_COMMON -fexceptions -fcxx-exceptions -frtti" + +newer() { # newer ...: is the output newer than every input? + local out=$1; shift + [ -e "$out" ] || return 1 + [ -z "$(find "$@" -newer "$out" -print -quit 2>/dev/null)" ] +} + +# ---------------------------------------------------------------- LuaJIT +LJ=$B/luajit +if ! newer "$LJ/src/libluajit.a" "$repo/scripts/ps5/deps.sh" "$repo/scripts/ps5/build_runtime.sh"; then + ps5_say "LuaJIT $LUAJIT_REV" + rm -rf "$LJ" && mkdir -p "$LJ" + git -C "$S/LuaJIT" archive "$LUAJIT_REV" | tar -x -C "$LJ" + # LuaJIT knows __PROSPERO__ and then turns FFI off; the payload SDK has + # what FFI needs (PacBrew's recipe does the same). + sed -i 's|__PROSPERO__|__JUNK__|g' "$LJ/src/lj_arch.h" + # The title unwinds with LLVM's libunwind, whose __register_frame takes + # one FDE (as on macOS), not a whole .eh_frame: register the JIT's + # machine code the macOS way, or every trace logs "bad fde: FDE is really + # a CIE" and errors cannot unwind through JIT-compiled code. + sed -i 's|^#if LJ_TARGET_OSX$|#if LJ_TARGET_OSX \|\| defined(LUAJIT_LLVM_LIBUNWIND)|' "$LJ/src/lj_err.c" + grep -q 'defined(LUAJIT_LLVM_LIBUNWIND)' "$LJ/src/lj_err.c" || ps5_die "LuaJIT lj_err.c patch failed" + # Sony's clock() counts microseconds, but the payload SDK's says + # CLOCKS_PER_SEC is 128 (FreeBSD's), so os.clock() ran 7812 times fast + # (measured on 13.00: a 0.35 s loop reported 2700 s). + sed -i 's|(1.0/(lua_Number)CLOCKS_PER_SEC)|(1.0/1000000.0)|' "$LJ/src/lib_os.c" + grep -q '(1.0/1000000.0)' "$LJ/src/lib_os.c" || ps5_die "LuaJIT lib_os.c clock patch failed" + make -C "$LJ" -j"$JOBS" amalg BUILDMODE=static CROSS="$SDK/bin/prospero-" \ + DEFAULT_CC=clang TARGET_LIBS=-lunwind \ + XCFLAGS="-DLUAJIT_USE_SYSMALLOC -DLUAJIT_LLVM_LIBUNWIND" Q= \ + > "$B/luajit.log" 2>&1 || { + # The luajit executable may not link as a payload; the library is all + # that is needed. + [ -f "$LJ/src/libluajit.a" ] || { tail -40 "$B/luajit.log" >&2; ps5_die "LuaJIT build failed"; } + } +fi + +# ---------------------------------------------------------------- SDL2 +SDLB=$B/sdl +PREFIX=$B/prefix +if ! newer "$PREFIX/lib/libSDL2.a" "$repo/ports/ps5/sdl" "$repo/scripts/ps5/deps.sh"; then + ps5_say "SDL2 $SDL_REV + ps5-opengl G19 video driver" + rm -rf "$SDLB" && mkdir -p "$SDLB/src" "$SDLB/g19" + git -C "$S/SDL" archive "$SDL_REV" | tar -x -C "$SDLB/src" + patch -d "$SDLB/src" -p1 -s < "$G19/static-ps5.patch" + patch -d "$SDLB/src" -p1 -s < "$G19/ps5-joystick.patch" + cp "$G19/SDL_ps5g19.c" "$G19/ps5g19_display.h" "$SDLB/g19/" + patch -d "$SDLB/g19" -p1 -s < "$repo/ports/ps5/sdl/g19-love.patch" + "$SDK/bin/prospero-cmake" -S "$repo/ports/ps5/sdl" -B "$SDLB/build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="$CFLAGS_COMMON" \ + -DSDL_SOURCE="$SDLB/src" -DG19_SOURCE="$SDLB/g19" -DPS5_OPENGL_PREFIX="$GL" \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_VERBOSE_MAKEFILE=OFF > "$B/sdl.log" 2>&1 || + { tail -40 "$B/sdl.log" >&2; ps5_die "SDL configure failed"; } + cmake --build "$SDLB/build" -j"$JOBS" >> "$B/sdl.log" 2>&1 || + { tail -40 "$B/sdl.log" >&2; ps5_die "SDL build failed"; } + rm -rf "$PREFIX/include/SDL2" + mkdir -p "$PREFIX/include/SDL2" "$PREFIX/lib" + cp "$SDLB/src/include/"*.h "$PREFIX/include/SDL2/" + cp "$SDLB/build/sdl/include/SDL2/"*.h "$PREFIX/include/SDL2/" 2>/dev/null || true + cp "$SDLB/build/sdl/include-config-release/SDL2/"*.h "$PREFIX/include/SDL2/" 2>/dev/null || true + cp "$(find "$SDLB/build" -name libSDL2.a | head -1)" "$PREFIX/lib/libSDL2.a" +fi + +# ---------------------------------------------------------------- pkg-config +# PacBrew's .pc files name /user/homebrew (their install prefix); point them +# at the unpacked tree. SDL2 and LuaJIT come from this build. +# Their libraries are seen through a directory of .a links only: the .la files +# next to them name paths of PacBrew's build machine and break libtool. +PC=$B/pkgconfig +PBLIB=$B/pacbrew-lib +rm -rf "$PC" "$PBLIB" && mkdir -p "$PC" "$PBLIB" +ln -s "$PB"/lib/*.a "$PBLIB/" +for f in "$PB"/lib/pkgconfig/*.pc; do + sed -e "s|/user/homebrew/lib|$PBLIB|g" -e "s|\${prefix}/lib|$PBLIB|g" \ + -e "s|\${exec_prefix}/lib|$PBLIB|g" -e "s|/user/homebrew|$PB|g" "$f" > "$PC/$(basename "$f")" +done +cat > "$PC/sdl2.pc" < "$PC/luajit.pc" < "$B/love.log" 2>&1 || { tail -40 "$B/love.log" >&2; ps5_die "LOVE configure failed"; } + make -C "$LV" -j"$JOBS" >> "$B/love.log" 2>&1 || + { tail -40 "$B/love.log" >&2; ps5_die "LOVE build failed"; } + lib=$(find "$LV/src" -name 'liblove*.a' -path '*.libs*' | head -1) + [ -n "$lib" ] || ps5_die "LOVE built no static library" + cp "$lib" "$LV/liblove.a" +fi + +# ---------------------------------------------------------------- host tool +# The boilerplate's converter, with the libc heap capped at 256 MiB the way +# ps5-opengl's native folder does (the title's heap is ours, app_heap.c). +HT=$B/hosttool +if [ ! -x "$HT/build/host/ps5-native-tool" ]; then + ps5_say "boilerplate host tool" + rm -rf "$HT" && mkdir -p "$HT" + cp -a "$BP/tooling" "$BP/tools" "$HT/" + mkdir -p "$HT/.deps" && ln -s "$BP/.deps/native" "$HT/.deps/native" + python3 - "$HT/tooling/native/sce_module_writer.cpp" <<'PY' +import sys +p = sys.argv[1] +s = open(p).read() +old = "write_u64(result.data, result.heap_size, std::numeric_limits::max());" +if s.count(old) != 1: + raise SystemExit("boilerplate sce_module_writer.cpp changed; update build_runtime.sh") +open(p, "w").write(s.replace(old, "write_u64(result.data, result.heap_size, 0x10000000ULL);")) +PY + bash "$HT/tools/build-host-tools.sh" > "$B/hosttool.log" 2>&1 || + { tail -20 "$B/hosttool.log" >&2; ps5_die "host tool build failed"; } +fi +TOOL=$HT/build/host/ps5-native-tool + +# ---------------------------------------------------------------- launcher +L=$B/launcher +rm -rf "$L" && mkdir -p "$L" +bp_cxx() { "$CXX" -std=c++20 -O2 -fno-exceptions -fno-rtti -ffunction-sections -fdata-sections "$@"; } +bp_cxx -c "$BP/tooling/native/app_crt.cpp" -o "$L/app_crt.o" +bp_cxx -c "$BP/tooling/native/app_cpp_runtime.cpp" -o "$L/app_cpp_runtime.o" +"$CXX" -std=c++17 $CXXFLAGS_COMMON -I"$LV/src" -I"$LV/src/modules" -I"$PREFIX/include/SDL2" \ + -I"$LJ/src" -DSDL_MAIN_HANDLED=1 -c "$repo/ports/ps5/launcher/main.cpp" -o "$L/main.o" +"$CC" -std=c11 $CFLAGS_COMMON -c "$repo/ports/ps5/launcher/ffi.c" -o "$L/ffi.o" +"$CC" -std=c11 $CFLAGS_COMMON -c "$repo/ports/ps5/launcher/compat.c" -o "$L/compat.o" +"$CC" -std=c11 $CFLAGS_COMMON -c "$NATIVE/app_heap.c" -o "$L/app_heap.o" + +# ffi.C's table: every PHYSFS_* function LOVE's library defines, plus the +# libc names of ports/ps5/launcher/ffi_exports.txt. +{ + "$SDK/bin/prospero-nm" -g --defined-only "$LV/liblove.a" 2>/dev/null | + awk '$2 == "T" && $3 ~ /^PHYSFS_/ { print $3 }' + sed 's/#.*//; s/[[:space:]]//g; /^$/d' "$repo/ports/ps5/launcher/ffi_exports.txt" +} | sort -u > "$L/ffi_exports.list" +grep -q '^PHYSFS_mount$' "$L/ffi_exports.list" || ps5_die "no PHYSFS_mount in liblove.a" +{ + echo '/* Generated by scripts/ps5/build_runtime.sh. */' + echo '#include ' + while read -r name; do echo "extern void $name(void);"; done < "$L/ffi_exports.list" + echo 'struct g2r_ffi_export { const char *name; void *address; };' + echo 'const struct g2r_ffi_export g2r_ffi_exports[] = {' + while read -r name; do echo " {\"$name\", (void *)$name},"; done < "$L/ffi_exports.list" + echo '};' + echo 'const size_t g2r_ffi_export_count = sizeof(g2r_ffi_exports) / sizeof(g2r_ffi_exports[0]);' +} > "$L/ffi_exports.c" +"$CC" -std=c11 -O2 -w -fno-builtin -c "$L/ffi_exports.c" -o "$L/ffi_exports.o" + +# ---------------------------------------------------------------- link +# ps5-opengl's linker layout: the boilerplate's, plus the .eh_frame bounds +# libunwind needs (C++ exceptions in LOVE, LuaJIT's error unwinding). +cat > "$L/ps5-pie.ld" < "$L/app-symbols.map" +builtins="$(clang-18 --print-resource-dir)/lib/linux/libclang_rt.builtins-x86_64.a" +[ -f "$builtins" ] || ps5_die "clang-18 compiler-rt builtins not found: $builtins" +pb() { echo "$PB/lib/$1"; } +"$SDK/bin/prospero-lld" -T "$L/ps5-pie.ld" --eh-frame-hdr \ + --version-script "$L/app-symbols.map" -e _start -o "$L/llvm-pie.elf" \ + --gc-sections \ + --wrap=malloc --wrap=calloc --wrap=realloc --wrap=free \ + --wrap=posix_memalign --wrap=malloc_usable_size \ + --wrap=dlsym --wrap=dlopen --wrap=dlerror --wrap=dlclose --wrap=lstat \ + --wrap=opendir --wrap=readdir --wrap=closedir \ + -u ps5_agc_gate2_run "${cxx_u[@]}" -L "$SDK/target/lib" -L "$GL/lib" \ + "$L/app_crt.o" "$L/app_cpp_runtime.o" "$L/main.o" "$L/ffi.o" "$L/ffi_exports.o" \ + "$L/compat.o" "$L/app_heap.o" \ + "$SDK/target/lib/libc++abi.a" "$SDK/target/lib/libc++.a" "$SDK/target/lib/libunwind.a" \ + --as-needed "$SDK"/target/lib/*.so --no-as-needed \ + --why-extract="$B/why-extract.txt" \ + --start-group \ + "$LV/liblove.a" "$PREFIX/lib/libSDL2.a" "$LJ/src/libluajit.a" \ + "$(pb libopenal.a)" "$(pb libfreetype.a)" "$(pb libpng16.a)" "$(pb libbz2.a)" "$(pb libz.a)" \ + "$(pb libvorbisfile.a)" "$(pb libvorbis.a)" "$(pb libtheoradec.a)" "$(pb libogg.a)" \ + "$(pb libmodplug.a)" "$(pb libmpg123.a)" \ + "$GL/lib/libPS5OpenGL.a" \ + "$builtins" "$SDK/target/lib/libc.a" \ + --end-group > "$B/link.log" 2>&1 || + { grep -m 60 -E 'error|undefined' "$B/link.log" >&2 || tail -40 "$B/link.log" >&2; ps5_die "link failed"; } + +# Imports a title cannot resolve are null calls: list what came from the +# modules PS5LM found missing in a title (libScePosixForWebKit, libkernel_sys). +python3 "$repo/scripts/ps5/check_imports.py" "$L/llvm-pie.elf" "$SDK/target/lib" "$SDK/bin/prospero-nm" \ + | tee "$B/imports.txt" + +"$TOOL" link --in "$L/llvm-pie.elf" --out "$L/eboot.elf" --stub-dir "$SDK/target/lib" \ + --module-sdk 0x02000009 --companion-sdk 0x08050001 --file-name eboot.elf > "$B/convert.log" 2>&1 || + { tail -20 "$B/convert.log" >&2; ps5_die "ELF conversion failed"; } +"$TOOL" self --sign --in "$L/eboot.elf" --out "$O/eboot.bin" --magic 0x1D3D154F >> "$B/convert.log" 2>&1 || + { tail -20 "$B/convert.log" >&2; ps5_die "FSELF signing failed"; } +cp "$BP/runtime/libc.prx" "$O/libc.prx" +ps5_say "runtime ready: $O/eboot.bin ($(stat -c %s "$O/eboot.bin") bytes)" diff --git a/scripts/ps5/check_imports.py b/scripts/ps5/check_imports.py new file mode 100644 index 000000000..cf1ac79aa --- /dev/null +++ b/scripts/ps5/check_imports.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Show which PS5 module each import of the title binds to, and fail on the +ones a title cannot resolve. + + check_imports.py + +The boilerplate's converter binds every undefined symbol to the first +DT_NEEDED stub module that exports it, in DT_NEEDED order. PS5LM found +(docs/CONSOLE.md, "Imports that resolve to nothing") that functions bound to +libScePosixForWebKit or libkernel_sys are null in a title: a call jumps to +address 0. Such a symbol has to be defined by the title itself +(ports/ps5/launcher/compat.c). Exit status 1 when any is left, so the build +stops before it produces a title that crashes. + +Also fails when a piece of the C++ runtime is imported (libSceLibcInternal +exports __gxx_personality_v0, __cxa_get_globals, std:: exception classes...): +mixing Sony's C++ ABI library with the static libc++abi/libunwind crashes on +the first throw, so all of it must come from the static libraries. +""" +import collections +import pathlib +import re +import subprocess +import sys + +RISKY = {"libScePosixForWebKit.so", "libScePosixForWebKit.sprx", "libScePosixForWebKit.prx", + "libkernel_sys.so", "libkernel_sys.sprx", "libkernel_sys.prx"} + + +# The C++ runtime: must never be imported. __cxa_atexit/__cxa_finalize are +# the C library's (the title CRT registers destructors through Sony's libc). +CXX_RUNTIME = re.compile(r"^(__gxx_personality|__cxa_|_Unwind_|_ZSt|_ZNSt|_ZNKSt|_ZTISt|_ZTVSt|_ZTSSt" + r"|_ZTVN10__cxxabiv|_Znw|_Zna|_Zdl|_Zda)") +CXX_ALLOWED = {"__cxa_atexit", "__cxa_finalize"} + + +def run(*cmd): + return subprocess.run(cmd, capture_output=True, text=True, check=True).stdout + + +def main(): + elf, libdir, nm = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3] + needed = re.findall(r"\(NEEDED\)\s+Shared library: \[([^\]]+)\]", run("readelf", "-d", elf)) + by_soname = {} + for stub in sorted(libdir.glob("*.so")): + found = re.findall(r"\(SONAME\)\s+Library soname: \[([^\]]+)\]", run("readelf", "-d", str(stub))) + by_soname.setdefault(found[0] if found else stub.name, stub) + exports = {} + for name in needed: + syms = set() + for line in run(nm, "-D", "--defined-only", str(by_soname[name])).splitlines(): + parts = line.split() + if len(parts) >= 3: + syms.add(parts[2].split("@")[0]) + exports[name] = syms + undefined = sorted({line.split()[-1].split("@")[0] + for line in run(nm, "-D", "--undefined-only", elf).splitlines() if line.split()}) + bound = collections.defaultdict(list) + missing = [] + for sym in undefined: + module = next((m for m in needed if sym in exports[m]), None) + if module is None: + missing.append(sym) + else: + bound[module].append(sym) + print(f"imports: {len(undefined)} symbols from {len(needed)} modules") + for module in needed: + print(f"imports: {module}: {len(bound[module])}") + bad = False + for module in needed: + if module in RISKY and bound[module]: + bad = True + print(f"imports: UNRESOLVABLE IN A TITLE ({module}): {' '.join(bound[module])}") + cxx = [s for s in undefined if CXX_RUNTIME.match(s) and s not in CXX_ALLOWED] + if cxx: + bad = True + print(f"imports: C++ RUNTIME IMPORTED (link libc++abi/libc++/libunwind first): {' '.join(cxx)}") + for sym in missing: + bad = True + print(f"imports: no stub module exports {sym}") + sys.exit(1 if bad else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/ps5/deps.sh b/scripts/ps5/deps.sh new file mode 100644 index 000000000..f6b237aa1 --- /dev/null +++ b/scripts/ps5/deps.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# PS5 title build: pinned inputs and their fetchers. Sourced by +# scripts/ps5/build_runtime.sh and scripts/build_ps5.sh; never run on its own. +# +# Everything lands in $G2R_PS5_WORK (default ~/.cache/gen2recomp-ps5), outside +# the repository. Archives are checked against their sha256 before they are +# unpacked, git sources are checked out at an exact commit. A directory that +# is already there with its ".pinned" stamp matching the pin is reused as is. + +G2R_PS5_WORK=${G2R_PS5_WORK:-$HOME/.cache/gen2recomp-ps5} + +# ps5-native-app-boilerplate: the title CRT, the clean-room sce_module/libc.prx, +# the ELF -> FSELF converter, and (through its own pinned setup script) the +# public PS5 payload SDK v0.42. Same commit EglantinePS5 pins. +BOILERPLATE_URL=https://github.com/drakmor/ps5-native-app-boilerplate.git +BOILERPLATE_REV=dd44bbdc75437332ed22e3ba95126733419ef25a + +# ps5-opengl SDK 1.0.2: Mesa 26.2 + the PS5 AGC winsys as static EGL/GL 4.6, +# the GL stack EglantinePS5 ships. The tarball also carries the sources of +# ps5-opengl itself, including integration/SDL2 (the SDL2 video driver and +# the two SDL patches this build applies). +GLSDK_URL=https://github.com/darkxex/ps5-opengl/releases/download/v1.0.2/ps5-opengl-sdk-1.0.2.tar.gz +GLSDK_SHA256=30d8e51dd062739b629346102be5932728ba714bc50b1bdcdf08306c8843d9a4 + +# PacBrew v0.40.2 prebuilt ports (built with the payload SDK): LuaJIT (FFI on), +# FreeType, OpenAL Soft 1.24.3, libogg/vorbis, libtheora, libmodplug, mpg123, +# zlib, libpng, bzip2. Only target/user/homebrew is unpacked. +PACBREW_URL=https://github.com/ps5-payload-dev/pacbrew-repo/releases/download/v0.40.2/ps5-payload-dev.tar.gz +PACBREW_SHA256=a85f65de418a8e6a898c6c3e3c870d50fff7618a200e4dd59ea9692af6ecec4d + +# ps5-payload-dev's SDL fork at the commit ps5-opengl's SDL2 bridge was +# qualified against (SDL 2.30.12): PS5 joystick (ScePad) and audio +# (SceAudioOut) backends; its OSMesa video driver is replaced by the bridge. +SDL_URL=https://github.com/ps5-payload-dev/SDL.git +SDL_REV=8c56053f13ca13a0c050de613706ff69eb615836 + +# LuaJIT 2.1 at PacBrew's commit, rebuilt here (not PacBrew's archive) with +# LUAJIT_USE_SYSMALLOC: LuaJIT's own allocator maps flexible memory, of which a +# title has only a few hundred MiB; through malloc it lands in the title's +# direct-memory heap (ports/ps5/launcher/heap.c). +LUAJIT_URL=https://github.com/LuaJIT/LuaJIT.git +LUAJIT_REV=f9140a622a0c44a99efb391cc1c2358bc8098ab7 + +# LOVE 11.5, the release the game targets (same tarball and hash as PacBrew's +# love recipe). +LOVE_URL=https://github.com/love2d/love/releases/download/11.5/love-11.5-linux-src.tar.gz +LOVE_SHA256=066e0843f71aa9fd28b8eaf27d41abb74bfaef7556153ac2e3cf08eafc874c39 + +ps5_say() { printf 'ps5: %s\n' "$*"; } +ps5_die() { printf 'ps5: error: %s\n' "$*" >&2; exit 1; } + +# ps5_fetch_archive +ps5_fetch_archive() { + local url=$1 sha=$2 file=$3 + mkdir -p "$(dirname "$file")" + if [ -f "$file" ] && printf '%s %s\n' "$sha" "$file" | sha256sum --check --strict --status; then + return 0 + fi + ps5_say "downloading $url" + rm -f "$file.part" + curl -fsSL --retry 3 -o "$file.part" "$url" || ps5_die "download failed: $url" + printf '%s %s\n' "$sha" "$file.part" | sha256sum --check --strict --status || + ps5_die "sha256 mismatch for $url (expected $sha)" + mv "$file.part" "$file" +} + +# ps5_fetch_git +ps5_fetch_git() { + local url=$1 rev=$2 dir=$3 + if [ -d "$dir/.git" ] && [ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$rev" ]; then + return 0 + fi + if [ ! -d "$dir/.git" ]; then + ps5_say "cloning $url" + rm -rf "$dir" + git clone -q "$url" "$dir" || ps5_die "clone failed: $url" + fi + # By sha: a pinned commit need not be on the default branch (the + # boilerplate's is not). + git -C "$dir" cat-file -e "$rev^{commit}" 2>/dev/null || git -C "$dir" fetch -q origin "$rev" || + ps5_die "fetch failed: $url $rev" + git -C "$dir" -c advice.detachedHead=false checkout -q --force "$rev" || + ps5_die "$url has no commit $rev" +} + +# ps5_stamp_ok : is already unpacked from ? +ps5_stamp_ok() { [ -f "$1/.pinned" ] && [ "$(cat "$1/.pinned")" = "$2" ]; } + +ps5_fetch_all() { + local dl=$G2R_PS5_WORK/dl src=$G2R_PS5_WORK/src + mkdir -p "$dl" "$src" + + # ---- boilerplate + payload SDK + libc.prx + ps5_fetch_git "$BOILERPLATE_URL" "$BOILERPLATE_REV" "$src/boilerplate" + if [ ! -x "$src/boilerplate/.deps/native/ps5-payload-sdk/bin/prospero-lld" ]; then + ps5_say "setting up the payload SDK (boilerplate's pinned v0.42)" + bash "$src/boilerplate/tools/setup-native-dependencies.sh" >/dev/null + fi + # sce_module/libc.prx is generated from the boilerplate's own source and + # must reproduce the checksum the boilerplate commits. + if [ ! -f "$src/boilerplate/runtime/libc.prx" ]; then + ps5_say "generating the boilerplate's clean-room libc.prx" + bash "$src/boilerplate/tools/rebuild-libc.sh" >/dev/null + fi + (cd "$src/boilerplate/runtime" && sha256sum --check --strict --status libc.prx.sha256) || + ps5_die "boilerplate runtime/libc.prx does not match its checksum" + + # ---- ps5-opengl SDK + ps5_fetch_archive "$GLSDK_URL" "$GLSDK_SHA256" "$dl/ps5-opengl-sdk-1.0.2.tar.gz" + if ! ps5_stamp_ok "$src/ps5-opengl-sdk-1.0.2" "$GLSDK_SHA256"; then + rm -rf "$src/ps5-opengl-sdk-1.0.2" + tar -xzf "$dl/ps5-opengl-sdk-1.0.2.tar.gz" -C "$src" + [ -f "$src/ps5-opengl-sdk-1.0.2/sdk/lib/libPS5OpenGL.a" ] || ps5_die "ps5-opengl SDK layout changed" + mkdir -p "$src/ps5-opengl-sdk-1.0.2/src" + tar -xf "$src/ps5-opengl-sdk-1.0.2/sources/ps5-opengl.tar" -C "$src/ps5-opengl-sdk-1.0.2/src" \ + ps5-opengl/integration/SDL2 ps5-opengl/native-app + echo "$GLSDK_SHA256" > "$src/ps5-opengl-sdk-1.0.2/.pinned" + fi + + # ---- PacBrew ports + ps5_fetch_archive "$PACBREW_URL" "$PACBREW_SHA256" "$dl/pacbrew-0.40.2.tar.gz" + if ! ps5_stamp_ok "$src/pacbrew-0.40.2" "$PACBREW_SHA256"; then + rm -rf "$src/pacbrew-0.40.2" + mkdir -p "$src/pacbrew-0.40.2" + tar -xzf "$dl/pacbrew-0.40.2.tar.gz" -C "$src/pacbrew-0.40.2" \ + --wildcards '*target/user/homebrew/*' + local hb + hb=$(find "$src/pacbrew-0.40.2" -type d -path '*target/user/homebrew' | head -1) + [ -f "$hb/lib/libluajit-5.1.a" ] || ps5_die "PacBrew archive layout changed" + ln -sfn "$hb" "$src/pacbrew-0.40.2/homebrew" + echo "$PACBREW_SHA256" > "$src/pacbrew-0.40.2/.pinned" + fi + + # ---- SDL fork + ps5_fetch_git "$SDL_URL" "$SDL_REV" "$src/SDL" + ps5_fetch_git "$LUAJIT_URL" "$LUAJIT_REV" "$src/LuaJIT" + + # ---- LOVE + ps5_fetch_archive "$LOVE_URL" "$LOVE_SHA256" "$dl/love-11.5-linux-src.tar.gz" +} diff --git a/src/cloud/GoogleAuth.lua b/src/cloud/GoogleAuth.lua index 9cc33e806..8e13e9f5a 100644 --- a/src/cloud/GoogleAuth.lua +++ b/src/cloud/GoogleAuth.lua @@ -36,7 +36,9 @@ local function isAndroid() return love and love.system and love.system.getOS() = function G.client() -- iOS too: Safari suspends the app, so the loopback listener the desktop -- flow waits on never hears Google either; the code popup is the way in - if isAndroid() or (love and love.system and love.system.getOS() == "iOS") then + -- ...and the PS5, which has no browser and cannot run a loopback listener + if isAndroid() or (love and love.system and love.system.getOS() == "iOS") + or (love and love.system and love.system.getOS() == "PS5") then local c = Clients.googleDevice() if c then return c, "device" end local why = Clients.why and Clients.why() diff --git a/src/core/FaithfulRes.lua b/src/core/FaithfulRes.lua index c7fc49462..419347e4e 100644 --- a/src/core/FaithfulRes.lua +++ b/src/core/FaithfulRes.lua @@ -88,6 +88,8 @@ end -- locked afterwards. function FaithfulRes.apply(v) if FaithfulRes.isMobile() then return false end + -- a console's window is the fixed TV output; resizing it would only break it + if require("src.core.Platform").isPS5() then return false end if not love or not love.window or not love.window.setMode or not love.window.getMode then return false diff --git a/src/core/GamepadMap.lua b/src/core/GamepadMap.lua index ff3026bcd..3027a24cf 100644 --- a/src/core/GamepadMap.lua +++ b/src/core/GamepadMap.lua @@ -79,6 +79,12 @@ local function nxActive() return false end +local function ps5Active() + if love and love._os == "PS5" then return true end + if love and love.system and love.system.getOS and love.system.getOS() == "PS5" then return true end + return false +end + function GamepadMap.gamepadBindings() if nxActive() then return GamepadMap.NX_GAMEPAD_BINDINGS end return GamepadMap.DEFAULT_GAMEPAD_BINDINGS @@ -97,6 +103,12 @@ function GamepadMap.buttonLabel(button) local labels = {a='B',b='A',x='Y',y='X'} if labels[button] then return labels[button] end end + -- PS5: a DualSense reports SDL's Xbox-position names with Cross as "a" + -- (confirm, same as the default table), so only the labels differ. + if ps5Active() then + local labels = {a='CROSS',b='CIRCLE',x='SQUARE',y='TRIANGLE'} + if labels[button] then return labels[button] end + end return button:upper() end diff --git a/src/core/Platform.lua b/src/core/Platform.lua index 3f012ac42..2568300a0 100644 --- a/src/core/Platform.lua +++ b/src/core/Platform.lua @@ -17,6 +17,11 @@ local function compute() or "Unknown" local nx = osName == "NX" local uwp = osName == "UWP" + -- PlayStation 5 homebrew (jailbroken console, native LOVE runtime). A TV + -- console like the Switch and Xbox: DualSense only, no mouse/keyboard/touch, + -- no file picker, no subprocesses, read-only game.love with a writable save + -- directory the player reaches over FTP. + local ps5 = osName == "PS5" local mobile = osName == "Android" or osName == "iOS" local nativePicker = love and love.system and type(love.system.pickFile) == "function" @@ -26,18 +31,19 @@ local function compute() os = osName, nx = nx, uwp = uwp, + ps5 = ps5, mobile = mobile, - console = nx or uwp, + console = nx or uwp or ps5, hasNativePicker = nativePicker, -- ONE definition, in src/core/HostShell.lua, beside the io.popen it -- describes. The literal three-OS test used to live here as well, so the -- shell that raises on the Switch was guarded by one copy and asked about -- through another. canSpawnProcess = HostShell.canSpawnProcess(), - romImportMode = nx and "save-directory" + romImportMode = (nx or ps5) and "save-directory" or (nativePicker and "native-picker") or "desktop", - networkValidated = not nx and not uwp, + networkValidated = not nx and not uwp and not ps5, -- networkValidated is the self-updater's gate and stays a per-platform -- policy call: a console package cannot replace itself on disk, so that -- answer never depends on whether a transport exists. Fetching a mod @@ -47,7 +53,7 @@ local function compute() -- console ports answer it with the native love.system.httpDownload bridge -- (#597). The UWP LOVE backend exports that bridge as well, but it only -- works from a love.thread there (see HostShell.haveUwpBridge). - canFetchRemote = (not nx and not uwp) or nativeHttp, + canFetchRemote = (not nx and not uwp and not ps5) or nativeHttp, } end @@ -75,6 +81,12 @@ function Platform.isUWP() return Platform.detect().uwp end +-- PlayStation 5 homebrew: love.system.getOS() answers "PS5". Prefer this, or +-- Platform.detect().console, over comparing the string at a call site. +function Platform.isPS5() + return Platform.detect().ps5 +end + function Platform.romImportMode() return Platform.detect().romImportMode end @@ -234,7 +246,7 @@ local function computeOnline() os = osName, hasEnet = hasEnet, hasSocket = hasSocket, available = false, relayOnly = false, canHost = false, canPortMap = false, canBroadcast = false, - validated = not d.nx and not d.uwp, + validated = not d.nx and not d.uwp and not d.ps5, consent = d.nx and "homebrew" or nil, } if osName == "Web" then @@ -267,6 +279,8 @@ local function computeOnline() r.reason = "Online play on the Switch is untested. Going online with homebrew can get a console banned by Nintendo." elseif d.uwp then r.reason = "Online play on Xbox is untested on hardware. Two apps on one console cannot link to each other." + elseif d.ps5 then + r.reason = "Online play on the PS5 is untested." end return r end diff --git a/src/core/VideoMode.lua b/src/core/VideoMode.lua index 39dc715ae..0a0d995f9 100644 --- a/src/core/VideoMode.lua +++ b/src/core/VideoMode.lua @@ -37,6 +37,8 @@ end -- Push the mode into the live window. Safe when love.window is missing. function VideoMode.apply(mode) if VideoMode.isMobile() then return end + -- the PS5 window is always the full-screen TV output + if require("src.core.Platform").isPS5() then return end if not love or not love.window or not love.window.setFullscreen then return end mode = VideoMode.normalize(mode) if mode == "borderless" then diff --git a/src/import/CacheFs.lua b/src/import/CacheFs.lua index 490bb1b6a..58fb1916b 100644 --- a/src/import/CacheFs.lua +++ b/src/import/CacheFs.lua @@ -481,6 +481,9 @@ function CacheFs.forgetRoot() customResolved = false customRoot = nil customWhy = nil + -- Packs live under the root, so a new root means new packs (or none): + -- their mounts came off with the rest above, and their indexes go too. + if CacheFs.forgetPacks then CacheFs.forgetPacks() end end -- Whether a folder change left a previous home on the read path, so the @@ -642,10 +645,286 @@ function CacheFs.bulkWriteSave(rel, data) return CacheFs.bulkWrites and bulkWrite(rel, data) or false end +-- --------------------------------------------------------------------------- +-- THE PACKED CACHE: one archive per version instead of ~18,000 loose files. +-- +-- A PS5 pays ~11 ms per file creation and then stalls 0.5-0.9 s in the +-- kernel about once a second; a Platinum import there took 44 minutes, and +-- the same bytes appended to one file took 8 s. So on the PS5 (or anywhere, +-- with POKEPORT_PACKED_CACHE=1) the import worker writes every +-- data/generated/... and assets/generated/... file of a version into +-- `rom-cache.zip` (src/import/CachePack) and the game reads it back +-- through PhysFS. Everywhere else nothing below runs and the loose tree is +-- exactly what it was. +-- +-- WHERE IT LIVES: next to the marker, `/rom-cache.zip`, where +-- is the live root (portable / chosen folder) or the save directory. +-- Red (prefix "") is never packed: it owns the cache root, it is a few hundred +-- files, and an archive mounted at "" for it could not be ordered against the +-- root's own loose files the way the version overlays below can. +-- +-- HOW IT IS READ. Entries are named version-relative ("data/generated/x.lua") +-- and the archive is mounted TWICE, which PhysFS allows only because the two +-- mounts spell the path differently -- it treats an identical string as +-- "already mounted" and silently adds nothing (measured; see PACK_ALT below): +-- +-- 1. at "" ("platinum"), APPENDED, as soon as anyone asks about the +-- version (mountPack). Everything that reads `platinum/data/...` by name +-- -- CacheFs.read with no root, the launcher, the Pokéwalker, the +-- cross-version lookups -- finds the pack's files there, and any loose +-- file at the same name (save directory or live root, both searched +-- first) wins. +-- 2. at "", PREPENDED, by mountVersion when the version boots -- BEFORE its +-- loose overlays, which are prepended after it and so sit in front. +-- require / newImage / newFile of `data/generated/...` then resolve +-- natively, in every Lua state, with no wrapper. The un-prefixed mount +-- also has to beat Red's cache at the root and a developer's source +-- tree, which is why it cannot be appended. +-- +-- PRECEDENCE, therefore: loose files of the version (a mod, Data.lua's +-- runtime copies, a hand-placed fix) > the pack > anything at the root. +-- +-- And CacheFs.read / CacheFs.exists fall back to the pack through this +-- module's own reader when the loose lookup misses, so they work with no +-- mount at all: inside the import worker (from the pack still being written) +-- and in the launcher before anything is mounted. +CacheFs.PACK_NAME = "rom-cache.zip" +local PACK_PART = ".part" + +local CachePack_ = nil +local function CachePack() + CachePack_ = CachePack_ or require("src.import.CachePack") + return CachePack_ +end + +-- POKEPORT_PACKED_CACHE=1 forces packing on any platform (desktop testing), +-- =0 forces it off (including on the PS5); otherwise only the PS5 packs. +function CacheFs.packedMode() + local env = os.getenv("POKEPORT_PACKED_CACHE") + if env == "1" then return true end + if env == "0" then return false end + local ok, on = pcall(function() return require("src.core.Platform").isPS5() end) + return ok and on == true +end + +-- Only the two generated trees go into a pack. The marker, the write probe +-- and anything else under the prefix stay loose: they are a handful of files +-- and the marker in particular must be written and read independently of the +-- archive it vouches for. +local function packable(name) + return name:sub(1, 15) == "data/generated/" + or name:sub(1, 17) == "assets/generated/" +end + +-- The real folder a pack of `prefix` lives in: the live root, else the save +-- directory. Returns home, isRoot. +local function packHome() + local root = CacheFs.root() + if root then return root, true end + local ok, dir = pcall(function() return love.filesystem.getSaveDirectory() end) + if ok and type(dir) == "string" and dir ~= "" then return dir, false end + return nil +end + +-- Real path of `rom-cache.zip`, and its save-dir-relative name. +-- `alt` returns the SECOND SPELLING of the same file: the separator before the +-- name doubled. Every OS collapses it; PhysFS compares mount sources with +-- strcmp, so it is a distinct search-path entry -- which is the only way to +-- have one archive at two mount points. +function CacheFs.packPath(prefix, alt) + prefix = prefix or CacheFs.prefix + local rel = prefix .. CacheFs.PACK_NAME + local home = packHome() + if not home then return nil, rel end + local dir = prefix:gsub("/+$", "") + local sep = alt and (SEP .. SEP) or SEP + local real = home .. SEP .. dir:gsub("/", SEP) .. sep .. CacheFs.PACK_NAME + return real, rel +end + +local function fileThere(real) + local f = real and io.open(real, "rb") + if not f then return false end + f:close() + return true +end + +-- Does `prefix` have a finished pack? io.* first (it is the path the writer +-- used); love.filesystem as well for the save directory, whose name io.* +-- cannot always open (a non-ASCII profile path on Windows). +local function packExists(prefix) + if prefix == "" then return false end + local real, rel = CacheFs.packPath(prefix) + if fileThere(real) then return true end + if not CacheFs.root() and love and love.filesystem and love.filesystem.getInfo then + -- getInfo through PhysFS would also find the archive's own mount, which + -- does not contain itself, so this only ever answers for the real file + return love.filesystem.getInfo(rel, "file") ~= nil + end + return false +end +CacheFs.packExists = packExists + +-- READERS, one per prefix, index only (CachePack readers re-open the file per +-- read, so none of this holds the archive open). `false` caches "no pack" +-- so the launcher's readiness pass, which asks dozens of exists() questions +-- per version, does not re-probe the disk for each. Invalidated by every +-- event in this Lua state that can change a pack, and by forgetPacks for the +-- one event that happens in another state (the import worker). +local packReaders = {} + +function CacheFs.forgetPacks() + packReaders = {} +end + +local function loveSource(rel) + return function() + local lfs = love and love.filesystem + if not (lfs and lfs.newFile) then return nil, "no love.filesystem" end + local f = lfs.newFile(rel) + if not (f and f:open("r")) then return nil, "cannot open " .. rel end + return { + read = function(_, n) return (f:read(n)) end, + seek = function(_, pos) return f:seek(pos) end, + size = function() return f:getSize() end, + close = function() f:close() end, + } + end +end + +local function packReader(prefix) + local cached = packReaders[prefix] + if cached ~= nil then return cached or nil end + packReaders[prefix] = false + if prefix == "" then return nil end + local real, rel = CacheFs.packPath(prefix) + local reader = nil + if fileThere(real) then + reader = CachePack().open(real) + elseif not CacheFs.root() and love and love.filesystem + and love.filesystem.getInfo and love.filesystem.getInfo(rel, "file") then + reader = CachePack().open(loveSource(rel)) + end + if reader then packReaders[prefix] = reader end + return reader +end + +-- The pack being written right now, in THIS Lua state (the import worker): +-- { prefix, writer, final, part, overflow }. +local activePack = nil + +-- Which pack answers for prefixed path `full`, and the entry name in it. +local function packFor(full) + if activePack then + local p = activePack.prefix + if full:sub(1, #p) == p then + local name = full:sub(#p + 1) + if packable(name) then return activePack.writer, name end + end + end + local prefix = CacheFs.prefix + if prefix == nil or prefix == "" or full:sub(1, #prefix) ~= prefix then return nil end + local name = full:sub(#prefix + 1) + if not packable(name) then return nil end + local reader = packReader(prefix) + if not reader then return nil end + return reader, name +end + +-- Start packing CacheFs.prefix's generated trees. Returns true, or false and +-- why -- in which case the caller carries on with loose files, which is +-- slower and otherwise identical. +function CacheFs.beginPack() + if activePack then return true end + local prefix = CacheFs.prefix or "" + if prefix == "" then return false, "Red's cache is never packed" end + local final, rel = CacheFs.packPath(prefix) + if not final then return false, "no folder to write the pack to" end + local part = final .. PACK_PART + local root = CacheFs.root() + if root then + ensureParents(root, rel) + elseif love and love.filesystem and love.filesystem.createDirectory then + love.filesystem.createDirectory((prefix:gsub("/+$", ""))) + end + os.remove(part) + local writer, err = CachePack().create(part) + if not writer then return false, "could not create " .. part .. ": " .. tostring(err) end + -- NOT TAKEN ON TRUST, for the same reason bulkWrite is not: the pack is + -- written with io.* to a path built from LOVE's name for the save directory + -- and read back through PhysFS. If the two do not meet, say so now. + if not root and love and love.filesystem and love.filesystem.getInfo then + writer.f:flush() + if not love.filesystem.getInfo(rel .. PACK_PART, "file") then + writer:abort() + return false, "io.* does not reach the save directory" + end + end + activePack = { prefix = prefix, writer = writer, final = final, part = part, + overflow = 0 } + return true +end + +function CacheFs.packActive() + return activePack ~= nil +end + +-- Finish the pack: central directory, close, then -- and only then -- rename +-- `.part` to the final name. Returns true, stats or nil, err. A pack that +-- cannot replace its predecessor is an error, not a warning: the old archive +-- would go on answering for the new import's files. +function CacheFs.finishPack() + local p = activePack + if not p then return true, nil end + activePack = nil + local w = p.writer + local stats = { entries = w.entries, bytes = w.bytes, + duplicates = w.duplicates, overflow = p.overflow, + path = p.final } + local ok, err = w:finish() + if not ok then + os.remove(p.part) + return nil, "could not finish " .. p.part .. ": " .. tostring(err) + end + stats.size = w.offset + if fileThere(p.final) then os.remove(p.final) end + if fileThere(p.final) then + os.remove(p.part) + return nil, "could not replace " .. p.final .. " (is it still mounted or open?)" + end + local renamed, renameErr = os.rename(p.part, p.final) + if not renamed then + os.remove(p.part) + return nil, "could not rename " .. p.part .. ": " .. tostring(renameErr) + end + packReaders[p.prefix] = nil + return true, stats +end + +-- Abandon the pack being written (the import failed): close and delete the +-- `.part`. Never leaves anything a later launch could mistake for a cache. +function CacheFs.abortPack() + local p = activePack + activePack = nil + if p then p.writer:abort() end +end + -- write cache-relative `rel` (forward-slash path) with the given bytes; -- returns ok, err like love.filesystem.write function CacheFs.write(rel, data) rel = withPrefix(rel) + if activePack then + local writer, name = packFor(rel) + if writer == activePack.writer then + local ok, why = writer:add(name, data) + if ok then return true end + if why ~= "full" then return false, why end + -- Past what a plain ZIP can index: this one goes loose. Loose files + -- are read ahead of the pack, so the cache stays whole -- just slower + -- to write on the console, which is why it is counted and reported. + activePack.overflow = activePack.overflow + 1 + end + end local root = CacheFs.root() if root then local f, err = openUnder(root, rel) @@ -667,9 +946,9 @@ function CacheFs.write(rel, data) return love.filesystem.write(rel, data) end --- read cache-relative `rel`; returns the bytes or nil -function CacheFs.read(rel) - rel = withPrefix(rel) +-- read cache-relative `rel`; returns the bytes or nil. A loose file wins; +-- the version's pack (or the one being written) answers when there is none. +local function looseRead(rel) local root = CacheFs.root() if root then local f = io.open(realPath(root, rel), "rb") @@ -681,9 +960,24 @@ function CacheFs.read(rel) return love.filesystem.read(rel) end --- does cache-relative `rel` exist as a file? -function CacheFs.exists(rel) +function CacheFs.read(rel) rel = withPrefix(rel) + local data = looseRead(rel) + if data ~= nil then return data end + local pack, name = packFor(rel) + if not pack then return nil end + local bytes, why = pack:read(name) + if bytes == nil and why == "stale" then + -- replaced under the cached index (the worker re-imported): re-index once + packReaders[CacheFs.prefix] = nil + pack, name = packFor(rel) + if pack then bytes = pack:read(name) end + end + return bytes +end + +-- does cache-relative `rel` exist as a file? +local function looseExists(rel) local root = CacheFs.root() if root then local f = io.open(realPath(root, rel), "rb") @@ -694,9 +988,23 @@ function CacheFs.exists(rel) return love.filesystem.getInfo(rel, "file") ~= nil end --- remove a single cache-relative file +function CacheFs.exists(rel) + rel = withPrefix(rel) + if looseExists(rel) then return true end + local pack, name = packFor(rel) + return pack ~= nil and pack:has(name) +end + +-- remove a single cache-relative file. Inside an import that is packing, the +-- entry is dropped from the archive too; a FINISHED pack cannot lose a file +-- (rewriting the archive for one deletion is not worth it), so a removal +-- there only reaches the loose copy. function CacheFs.remove(rel) rel = withPrefix(rel) + if activePack then + local writer, name = packFor(rel) + if writer == activePack.writer then writer:remove(name) end + end local root = CacheFs.root() if root then os.remove(realPath(root, rel)) @@ -1075,11 +1383,152 @@ local function mountGeneratedTrees(prefix) return mounted end +-- Put a pack on the read path at `point`. FFI first (absolute path, any +-- root); love.filesystem.mount as the fallback, which can only reach the save +-- directory. `alt` picks the second spelling (see CacheFs.packPath). +-- Recorded in ourMounts with `pack` (and `persistent` for the prefixed view) +-- so unmountPack / unmountVersion can find exactly these again. +local function mountPackAt(prefix, point, append, alt) + local real, rel = CacheFs.packPath(prefix, alt) + local persistent = point ~= "" + local fn = resolveMount() + if real and fn and fn(real, point, append) then + ourMounts[#ourMounts + 1] = { dir = real, point = point, append = append, + how = "ffi", pack = prefix, persistent = persistent } + return true + end + if not CacheFs.root() and love and love.filesystem and love.filesystem.mount then + local name = alt and (prefix:gsub("/+$", "") .. "//" .. CacheFs.PACK_NAME) or rel + local ok, mounted = pcall(love.filesystem.mount, name, point, append) + if ok and mounted then + ourMounts[#ourMounts + 1] = { dir = name, point = point, append = append, + how = "love", pack = prefix, persistent = persistent } + return true + end + end + return false +end + +local function packMounted(prefix, overlay) + for _, m in ipairs(ourMounts) do + if m.pack == prefix and (m.point == "") == (overlay == true) then return true end + end + return false +end + +-- The prefixed view of `prefix`'s pack (mount 1 above). Idempotent; returns +-- true when the pack is (now) visible at `/...`, false when there is no +-- pack or it could not be mounted. Cheap enough to call on every readiness +-- check: one io.open when there is no pack. +local packHeld = {} + +function CacheFs.mountPack(prefix) + prefix = prefix or CacheFs.prefix + if not prefix or prefix == "" then return false end + if packHeld[prefix] then return false end + if packMounted(prefix, false) then return true end + if not packExists(prefix) then return false end + return mountPackAt(prefix, (prefix:gsub("/+$", "")), true, false) +end + +-- Every version's pack, prefixed. mountVersion calls it so a running game's +-- cross-version reads (Pal Park, the Pokéwalker, adopted tilesets) see other +-- versions' packs under their own names, exactly as they see loose trees. +function CacheFs.mountPacks() + local ok, GV = pcall(require, "src.core.GameVersion") + if not (ok and GV and GV.ORDER) then return end + for _, id in ipairs(GV.ORDER) do + local okP, prefix = pcall(GV.cachePrefix, id) + if okP and type(prefix) == "string" and prefix ~= "" then + CacheFs.mountPack(prefix) + end + end +end + +-- Take `prefix`'s pack fully off the read path (both views) and drop its +-- index. Required before anything deletes or replaces the file: Windows +-- refuses to delete an open file and PhysFS holds a mounted archive open. A +-- mount PhysFS refuses (a file inside still open -- a streamed Source the +-- collector has not closed) gets one collection and a retry. Returns true +-- when nothing of it is mounted any more. +-- +-- `hold` keeps it off until releasePack: an import unmounts before its worker +-- deletes the file, and a readiness check in between (another launcher +-- column, a redraw) must not put it straight back. +function CacheFs.unmountPack(prefix, hold) + prefix = prefix or CacheFs.prefix + packReaders[prefix] = nil + if not prefix or prefix == "" then return true end + if hold then packHeld[prefix] = true end + local unmount = resolveUnmount() + local function takeOff(m) + if m.how == "ffi" and unmount then return unmount(m.dir) end + if m.how == "love" and love and love.filesystem and love.filesystem.unmount then + local ok, res = pcall(love.filesystem.unmount, m.dir) + return ok and res and true or false + end + return false + end + local keep, all = {}, true + for _, m in ipairs(ourMounts) do + if m.pack == prefix then + local off = takeOff(m) + if not off then + collectgarbage("collect"); collectgarbage("collect") + off = takeOff(m) + end + if not off then + keep[#keep + 1] = m + all = false + end + else + keep[#keep + 1] = m + end + end + ourMounts = keep + return all +end + +function CacheFs.releasePack(prefix) + packHeld[prefix or CacheFs.prefix] = nil +end + +-- Delete `prefix`'s pack -- the finished archive and any `.part` a killed +-- import left -- from the live root AND the save directory, the same two +-- homes a re-import clears loose trees from. Unmounts first. Returns +-- true, or false and why when a finished pack is still there afterwards (on +-- Windows: still open somewhere), which a re-import must treat as fatal. +function CacheFs.clearPack(prefix) + prefix = prefix or CacheFs.prefix + if not prefix or prefix == "" then return true end + if activePack and activePack.prefix == prefix then CacheFs.abortPack() end + CacheFs.unmountPack(prefix) + local real, rel = CacheFs.packPath(prefix) + if real then + os.remove(real .. PACK_PART) + os.remove(real) + end + if love and love.filesystem and love.filesystem.remove then + pcall(love.filesystem.remove, rel .. PACK_PART) + pcall(love.filesystem.remove, rel) + end + packReaders[prefix] = nil + if real and fileThere(real) then + return false, "could not delete " .. real .. " (is it still open?)" + end + return true +end + function CacheFs.mountVersion(version) local prefix = require("src.core.GameVersion").cachePrefix(version) if prefix == "" then return true end -- Red: already at the root local sub = prefix:gsub("/+$", "") -- "blue/" / "yellow/" -> bare dir + -- Packs first: the prefixed views make a packed version's files visible + -- under `/` (so the check below sees them), and other versions' packs + -- visible to the cross-version readers. + CacheFs.mountPacks() + -- Nothing to overlay: this version was never imported. Say so plainly -- rather than reporting a mount failure for a cache that does not exist. if not love.filesystem.getInfo(sub .. "/" .. PROBE, "file") then @@ -1108,6 +1557,15 @@ function CacheFs.mountVersion(version) return false, "could not overlay " .. sub .. "/ onto the read path" end + -- 0. A packed version's archive, PREPENDED at "" -- and FIRST, so every + -- loose overlay below is prepended after it and lands in front: a loose + -- file of this version beats the pack, and the pack beats Red's root + -- cache and the source. Second spelling, because the prefixed view + -- already holds the first (see CacheFs.packPath). + if packMounted(prefix, false) and not packMounted(prefix, true) then + mountPackAt(prefix, "", false, true) + end + -- 1. Whole version folder at "", by save-dir-relative name. No FFI, and -- it is enough wherever PhysFS has no colliding data/ to shadow it. -- @@ -1333,7 +1791,12 @@ function CacheFs._unmountRecorded(base, sub) local keep, any = {}, false for _, m in ipairs(ourMounts) do local d = normMountPath(m.dir) - if inside(d, home) or inside(d, rel) then + if m.persistent then + -- a pack's prefixed view (mountPack): it shows the version under its + -- own name, like the loose tree always is, so it outlives the overlay; + -- only unmountPack takes it down + keep[#keep + 1] = m + elseif inside(d, home) or inside(d, rel) then local off = takeOff(m) if not off then collectgarbage("collect"); collectgarbage("collect") diff --git a/src/import/CachePack.lua b/src/import/CachePack.lua new file mode 100644 index 000000000..6af43c51b --- /dev/null +++ b/src/import/CachePack.lua @@ -0,0 +1,439 @@ +-- A ROM cache in ONE FILE: a store-only ZIP archive the import writes instead +-- of 18,000 loose files, and that the game reads back through PhysFS. +-- +-- WHY A CONSOLE NEEDS THIS. On a PS5 (native title, /data) every file +-- creation costs ~11 ms for the first few thousand and then stalls 0.5-0.9 s +-- about once a second in the kernel's buffer cache (`getblk`). A Platinum +-- import is 18,433 files: 44 minutes on the console against 27 s on a PC. +-- The same 18,433 payloads appended to ONE file took 8.3 s with no stalls -- +-- so the cost is the file count, not the bytes, and the cure is one file. +-- +-- WHY ZIP, AND WHY STORE-ONLY. PhysFS reads ZIP natively, so a mounted pack +-- answers require / love.graphics.newImage / love.filesystem.newFile exactly +-- as the loose tree did, with no wrapper in the way. Nothing is compressed: +-- the bulk is PNGs, which already are, and a stored entry can be read back by +-- a seek and a read -- which is also how this module's own reader serves +-- CacheFs.read before anything is mounted. +-- +-- WHAT IT DOES NOT DO: zip64. A plain ZIP holds at most 65,535 entries and +-- cannot point past 4 GiB. The largest measured cache is ~18,400 files and +-- ~170 MB, so the writer REFUSES an entry that would cross either limit +-- (add returns false, "full") and CacheFs writes that file loose instead -- +-- loose files are read ahead of the pack anyway, so an over-full import is +-- slower, never wrong. +-- +-- Pure Lua (LuaJIT `bit`, and `ffi` when present for the CRC loop), no LÖVE: +-- it runs in the import worker's Lua state and under a bare luajit test. + +local bit = require("bit") +local band, bor, bxor, bnot = bit.band, bit.bor, bit.bxor, bit.bnot +local rshift, lshift = bit.rshift, bit.lshift +local byte, char, sub = string.byte, string.char, string.sub + +local CachePack = {} + +-- One short of 0xFFFF / 0xFFFFFFFF on purpose: readers treat those exact values +-- in the end-of-central-directory record as "look for a zip64 record", and a +-- pack with no zip64 record that happens to land on one is a pack some reader +-- will call corrupt. +CachePack.MAX_ENTRIES = 0xFFFE +CachePack.MAX_BYTES = 0xFFFFFFFE + +-- --------------------------------------------------------------------------- +-- CRC-32 (IEEE 802.3, the one ZIP uses). PhysFS verifies it when an entry is +-- read to the end, so it has to be right, not merely present. +-- +-- No zlib: the PS5 build only exports a fixed list of libc symbols to the FFI +-- (ports/ps5/launcher/ffi_exports.txt) and zlib's crc32 is not among them. +-- A table-driven loop over an FFI byte pointer is what LuaJIT compiles well; +-- the string.byte fallback is for a Lua without FFI. +local okFfi, ffi = pcall(require, "ffi") +if not okFfi then ffi = nil end + +local TABLE = {} +for i = 0, 255 do + local c = i + for _ = 1, 8 do + if band(c, 1) ~= 0 then c = bxor(rshift(c, 1), 0xEDB88320) + else c = rshift(c, 1) end + end + TABLE[i] = c +end +local CTABLE = nil +if ffi then + CTABLE = ffi.new("int32_t[256]") + for i = 0, 255 do CTABLE[i] = TABLE[i] end +end + +local function unsigned(v) + if v < 0 then return v + 4294967296 end + return v +end + +-- crc32(data [, previous [, plainLua]]) -> unsigned 32-bit value. +-- `previous` continues a running CRC; `plainLua` forces the string.byte loop +-- so a test can hold the two implementations against each other. +function CachePack.crc32(data, previous, plainLua) + local crc = bnot(bit.tobit(previous or 0)) + local n = #data + if CTABLE and n > 0 and not plainLua then + local p = ffi.cast("const uint8_t *", data) + local T = CTABLE + for i = 0, n - 1 do + crc = bxor(rshift(crc, 8), T[band(bxor(crc, p[i]), 255)]) + end + else + local T = TABLE + for i = 1, n do + crc = bxor(rshift(crc, 8), T[band(bxor(crc, byte(data, i)), 255)]) + end + end + return unsigned(bnot(crc)) +end + +-- --------------------------------------------------------------------------- +-- little-endian fields +local function u16(v) + return char(v % 256, math.floor(v / 256) % 256) +end +local function u32(v) + return char(v % 256, math.floor(v / 256) % 256, + math.floor(v / 65536) % 256, math.floor(v / 16777216) % 256) +end +local function r16(s, i) + local a, b = byte(s, i, i + 1) + return a + b * 256 +end +local function r32(s, i) + local a, b, c, d = byte(s, i, i + 3) + return a + b * 256 + c * 65536 + d * 16777216 +end + +-- MS-DOS date/time of `t` (os.time), the only timestamp a plain ZIP carries. +local function dosStamp(t) + local d = os.date("*t", t or os.time()) + local year = math.max(1980, math.min(2107, d.year)) + local date = (year - 1980) * 512 + d.month * 32 + d.day + local time = d.hour * 2048 + d.min * 32 + math.floor(d.sec / 2) + return time, date +end + +local SIG_LOCAL = 0x04034b50 +local SIG_CENTRAL = 0x02014b50 +local SIG_END = 0x06054b50 +local LOCAL_SIZE = 30 +local CENTRAL_SIZE = 46 +local END_SIZE = 22 + +-- --------------------------------------------------------------------------- +-- WRITER +-- +-- Entries are appended as they arrive (local header + bytes); the central +-- directory is written once, by finish. Until then the file is not a valid +-- archive, which is the point: the caller writes to a temporary name and +-- renames only after finish, so an import killed half way leaves a `.part` +-- nothing will ever mount -- never a pack that looks whole. +-- +-- The SAME NAME WRITTEN TWICE: the second copy is appended and the index is +-- re-pointed at it, so the central directory names each file once and the last +-- write wins -- exactly what overwriting a loose file did. The first copy's +-- bytes stay in the file as dead weight; `duplicates` counts them. +local Writer = {} +Writer.__index = Writer + +-- CachePack.create(path) -> writer | nil, err. Truncates `path`. +function CachePack.create(path) + local f, err = io.open(path, "w+b") + if not f then return nil, err end + local time, date = dosStamp() + return setmetatable({ + path = path, f = f, offset = 0, + order = {}, -- names in first-write order (central-directory order) + index = {}, -- name -> { offset, size, crc, dataAt } + entries = 0, duplicates = 0, bytes = 0, + time = time, date = date, + }, Writer) +end + +function Writer:count() return self.entries end + +-- add(name, data) -> true | false, why. why == "full" means the archive +-- cannot take this entry without zip64 (see MAX_ENTRIES / MAX_BYTES); the +-- caller is expected to store the file some other way. Any other `why` is an +-- I/O failure and the pack should be abandoned. +function Writer:add(name, data) + if not self.f then return false, "pack is closed" end + local existing = self.index[name] + if not existing and self.entries >= CachePack.MAX_ENTRIES then + return false, "full" + end + local size = #data + local headerAt = self.offset + local nameLen = #name + -- Everything this entry adds, plus the central-directory record it will + -- need, must still leave the directory's own offset below 4 GiB. + if headerAt + LOCAL_SIZE + nameLen + size + + (self.cdBytes or 0) + CENTRAL_SIZE + nameLen + END_SIZE + > CachePack.MAX_BYTES then + return false, "full" + end + local crc = CachePack.crc32(data) + local header = u32(SIG_LOCAL) .. u16(10) .. u16(0) .. u16(0) + .. u16(self.time) .. u16(self.date) .. u32(crc) .. u32(size) .. u32(size) + .. u16(nameLen) .. u16(0) .. name + local ok, err = self.f:write(header, data) + if not ok then return false, tostring(err or "write failed") end + local entry = { offset = headerAt, size = size, crc = crc, + dataAt = headerAt + LOCAL_SIZE + nameLen } + self.offset = headerAt + LOCAL_SIZE + nameLen + size + if existing then + self.duplicates = self.duplicates + 1 + self.bytes = self.bytes - existing.size + else + self.entries = self.entries + 1 + self.order[#self.order + 1] = name + self.cdBytes = (self.cdBytes or 0) + CENTRAL_SIZE + nameLen + end + self.bytes = self.bytes + size + self.index[name] = entry + return true +end + +function Writer:has(name) + return self.index[name] ~= nil +end + +-- Read an entry back while the pack is still being written: an import that +-- consults its own earlier output must see it, and it is not on disk anywhere +-- else. The seek back to the end is what keeps the next add appending (C +-- stdio also needs a positioning call between a read and a write). +function Writer:read(name) + local e = self.index[name] + if not (e and self.f) then return nil end + if e.size == 0 then return "" end + self.f:seek("set", e.dataAt) + local data = self.f:read(e.size) + self.f:seek("end") + return data +end + +-- Forget an entry (CacheFs.remove during an import). Its bytes stay; it just +-- is not in the central directory. +function Writer:remove(name) + local e = self.index[name] + if not e then return end + self.index[name] = nil + self.entries = self.entries - 1 + self.bytes = self.bytes - e.size + self.cdBytes = self.cdBytes - CENTRAL_SIZE - #name + for i, n in ipairs(self.order) do + if n == name then table.remove(self.order, i) break end + end +end + +-- Write the central directory and the end record, then close. Returns +-- true or nil, err. The file is a valid archive only after this returns true. +function Writer:finish() + local f = self.f + if not f then return nil, "pack is closed" end + local cdStart = self.offset + local parts, count = {}, 0 + for _, name in ipairs(self.order) do + local e = self.index[name] + if e then + count = count + 1 + parts[#parts + 1] = u32(SIG_CENTRAL) .. u16(20) .. u16(10) .. u16(0) .. u16(0) + .. u16(self.time) .. u16(self.date) .. u32(e.crc) .. u32(e.size) .. u32(e.size) + .. u16(#name) .. u16(0) .. u16(0) .. u16(0) .. u16(0) .. u32(0) + .. u32(e.offset) .. name + -- flushed in slices so 18,000 records are not one 1.5 MB concat + if #parts >= 512 then + local ok, err = f:write(table.concat(parts)) + if not ok then return nil, tostring(err) end + parts = {} + end + end + end + if #parts > 0 then + local ok, err = f:write(table.concat(parts)) + if not ok then return nil, tostring(err) end + end + local cdSize = f:seek("end") - cdStart + local tail = u32(SIG_END) .. u16(0) .. u16(0) .. u16(count) .. u16(count) + .. u32(cdSize) .. u32(cdStart) .. u16(0) + local ok, err = f:write(tail) + if not ok then return nil, tostring(err) end + local closed, closeErr = f:close() + self.f = nil + if not closed then return nil, tostring(closeErr) end + return true +end + +-- Close without a central directory and delete the file. +function Writer:abort() + if self.f then pcall(self.f.close, self.f) end + self.f = nil + os.remove(self.path) +end + +-- --------------------------------------------------------------------------- +-- READER +-- +-- CachePack.open(source) -> reader | nil, err. `source` is one of: +-- +-- * a path -- opened with io.open; +-- * a function returning a handle with read(n) / seek(absolutePos) / +-- size() / close() -- CacheFs wraps a love.filesystem File that way when +-- io.* cannot reach the save directory; +-- * such a handle itself. +-- +-- A path or a function is TRANSIENT: the file is closed once the central +-- directory is read and reopened for each entry. That costs an open per +-- read, and it is what lets the launcher keep a pack's index around without +-- holding the file open -- Windows refuses to delete or replace a file +-- somebody has open, and the next import has to do exactly that. +local Reader = {} +Reader.__index = Reader + +local function ioSource(path) + local f, err = io.open(path, "rb") + if not f then return nil, err end + return { + read = function(_, n) return f:read(n) end, + seek = function(_, pos) return f:seek("set", pos) end, + size = function() return f:seek("end") end, + close = function() f:close() end, + } +end + +function CachePack.open(source) + local opener = nil + if type(source) == "string" then + opener = function() return ioSource(source) end + elseif type(source) == "function" then + opener = source + end + local src, err = source, nil + if opener then + src, err = opener() + if not src then return nil, err end + end + local size = src:size() + if not size or size < END_SIZE then + src:close() + return nil, "too small to be a pack" + end + -- The end record is the last 22 bytes: this writer never adds a comment, + -- and a pack with one was not written here. + src:seek(size - END_SIZE) + local tail = src:read(END_SIZE) + if not tail or #tail ~= END_SIZE or r32(tail, 1) ~= SIG_END then + src:close() + return nil, "no end-of-central-directory record" + end + local count = r16(tail, 11) + local cdSize, cdStart = r32(tail, 13), r32(tail, 17) + src:seek(cdStart) + local cd = cdSize > 0 and src:read(cdSize) or "" + if not cd or #cd ~= cdSize then + src:close() + return nil, "truncated central directory" + end + local index, names = {}, {} + local at = 1 + for _ = 1, count do + if r32(cd, at) ~= SIG_CENTRAL then + src:close() + return nil, "bad central-directory record" + end + local method = r16(cd, at + 10) + local crc = r32(cd, at + 16) + local csize, usize = r32(cd, at + 20), r32(cd, at + 24) + local nlen, xlen, clen = r16(cd, at + 28), r16(cd, at + 30), r16(cd, at + 32) + local offset = r32(cd, at + 42) + local name = sub(cd, at + 46, at + 45 + nlen) + index[name] = { offset = offset, size = usize, csize = csize, + method = method, crc = crc } + names[#names + 1] = name + at = at + 46 + nlen + xlen + clen + end + if opener then + src:close() + src = nil + end + return setmetatable({ src = src, opener = opener, index = index, + names = names, fileSize = size }, Reader) +end + +function Reader:count() return #self.names end +function Reader:list() return self.names end +function Reader:has(name) return self.index[name] ~= nil end +function Reader:sizeOf(name) + local e = self.index[name] + return e and e.size or nil +end + +-- The bytes of `name`, or nil. Stored entries only (this writer never makes +-- anything else); the local header is re-read because its extra field may +-- differ from the central one in an archive written by another tool. +local function readEntry(src, e) + if not e.dataAt then + src:seek(e.offset) + local h = src:read(LOCAL_SIZE) + if not h or #h ~= LOCAL_SIZE or r32(h, 1) ~= SIG_LOCAL then + return nil, "bad local header" + end + e.dataAt = e.offset + LOCAL_SIZE + r16(h, 27) + r16(h, 29) + end + if e.size == 0 then return "" end + src:seek(e.dataAt) + return src:read(e.size) +end + +function Reader:read(name) + local e = self.index[name] + if not e then return nil end + if e.method ~= 0 then return nil, "compressed entry" end + if self.src then return readEntry(self.src, e) end + if not self.opener then return nil, "reader is closed" end + local src, err = self.opener() + if not src then return nil, err end + -- A different length is a different file: the pack was replaced under this + -- index (another Lua state re-imported). Refuse rather than read stale + -- offsets out of somebody else's archive. + if src:size() ~= self.fileSize then + src:close() + return nil, "stale" + end + local data, why = readEntry(src, e) + src:close() + return data, why +end + +-- Re-hash every entry against its recorded CRC. For tests and diagnostics; +-- returns true or false, firstBadName. +function Reader:verify() + local own = false + if not self.src and self.opener then + self.src = self.opener() + own = true + end + local bad = nil + for _, name in ipairs(self.names) do + local data = self:read(name) + if not data or CachePack.crc32(data) ~= self.index[name].crc then + bad = name + break + end + end + if own then self:close() end + if bad then return false, bad end + return true +end + +function Reader:close() + if self.src then self.src:close() end + self.src = nil +end + +return CachePack diff --git a/src/import/RomImporter.lua b/src/import/RomImporter.lua index 330695f1a..3cd854413 100644 --- a/src/import/RomImporter.lua +++ b/src/import/RomImporter.lua @@ -2112,6 +2112,12 @@ local function purgeSaveDirCache() removeTree(prefix .. "data/generated") removeTree(prefix .. "assets/generated") love.filesystem.remove(prefix .. MARKER_PATH) + -- ...and a packed copy of it (CacheFs "THE PACKED CACHE"), which the + -- portable root never mounts, so nothing holds it open + if prefix ~= "" then + love.filesystem.remove(prefix .. "rom-cache.zip") + love.filesystem.remove(prefix .. "rom-cache.zip.part") + end end end end @@ -2147,6 +2153,10 @@ function RomImporter.readyReport(version) CacheFs.prefix = GameVersion.cachePrefix(version) local raw = CacheFs.read(MARKER_PATH) CacheFs.prefix = saved + -- A packed version shows its files under its own name from here on, for + -- the launcher's readers that ask for `/data/...` by path. One + -- failed io.open for a version that is not packed. + if raw ~= nil then CacheFs.mountPack(GameVersion.cachePrefix(version)) end local head, absent = parseMarker(raw) if head ~= markerFor(version) then @@ -2408,6 +2418,15 @@ end -- love.filesystem paths on the Switch read "sdmc:/switch/...", but MTP tools -- (DBI, OpenMTP) show the card root as "1: SD Card", so the sdmc: prefix is -- noise there. Strip it ONLY when it is actually present. +-- How the player gets a file into the save directory: the Switch uses DBI over +-- MTP, the PS5 an FTP client. `sub` is the folder under imports/ ("" for ROMs). +function RomImporter.transferHint(saveDir, rel, sub) + if Platform.isPS5() then + return Strings("FTP -> %s/imports/%s", saveDir, sub) + end + return Strings("DBI MTP -> 1: SD Card/%simports/%s", rel, sub) +end + function RomImporter.mtpHintPath(saveDir) if type(saveDir) ~= "string" then return "" end if saveDir:sub(1, 6) == "sdmc:/" then return saveDir:sub(7) end @@ -3633,17 +3652,40 @@ function RomImporter:startData(data, displayName, sourcePath, verified) local CacheFs=require('src.import.CacheFs') local modules={[1]='src.import.RomExtractor',[2]='src.import.RomExtractorGen2', [3]='src.import.RomExtractorGen3',[4]='src.import.RomExtractorGen4',[5]='src.import.RomExtractorGen5'} + -- The old pack comes off the read path HERE, in the state that mounted it: + -- PhysFS mounts are process-wide but the record of them is this Lua + -- state's, and the worker is about to delete the file (which Windows + -- refuses while it is mounted). + if not CacheFs.unmountPack(info.cachePrefix, true) then + -- Something still has a file open inside it (a streamed sound, a + -- preview). Going on would delete the marker and then fail to replace + -- the archive -- or, where the OS lets an open file be unlinked, keep + -- serving the old one under the new marker. + CacheFs.releasePack(info.cachePrefix) + error("the previous " .. tostring(info.displayName or version) + .. " data is still in use; restart the game and import again") + end + -- ONE ARCHIVE PER VERSION where creating files is what costs (the PS5). + -- Red's cache is never packed (see CacheFs "THE PACKED CACHE"). + local packed=CacheFs.packedMode() and info.cachePrefix~='' local task=require('src.import.RomImportTask').new({action='extract',version=version, generation=info.generation,module=zeldaImport and 'src.zelda.ZeldaImport' or modules[info.generation],manifest=manifest, - prefix=info.cachePrefix,root=CacheFs.root(),path=self.romPath,bytes=info.generation<4 and self.romData or nil,clear=true,recovery=self.importRecovery}) + prefix=info.cachePrefix,root=CacheFs.root(),path=self.romPath,bytes=info.generation<4 and self.romData or nil,clear=true,recovery=self.importRecovery, + packed=packed}) if task then self.romData=nil local complete=false while not complete do local message=task:poll() while message do - if message.kind=='error' then error(message.error) end + if message.kind=='error' then CacheFs.releasePack(info.cachePrefix);error(message.error) end if message.kind=='complete' then complete=true end + if message.kind=='log' then + local Logger=require('src.core.Logger') + if message.level=='warn' then Logger.warn('%s',tostring(message.message)) + else Logger.info('%s',tostring(message.message)) end + BootTrace.mark(tostring(message.message)) + end if message.kind=='progress' then self.status=message.stage;self.progress=message.progress self.stageCurrent=message.current;self.stageTotal=message.total @@ -3653,6 +3695,12 @@ function RomImporter:startData(data, displayName, sourcePath, verified) if not complete then coroutine.yield("background") end end CacheFs.prefix=info.cachePrefix + -- The worker replaced (or removed) this version's pack in another Lua + -- state: drop what this one indexed, and show the new one under its + -- name before the gap check below reads through it. + CacheFs.forgetPacks() + CacheFs.releasePack(info.cachePrefix) + CacheFs.mountPack(info.cachePrefix) else -- Redirect every cache write to this version's subtree, then clear only -- that version's previous cache from both homes (save directory and, for @@ -3660,13 +3708,22 @@ function RomImporter:startData(data, displayName, sourcePath, verified) CacheFs = require("src.import.CacheFs") local prefix = info.cachePrefix CacheFs.prefix = prefix + love.filesystem.remove(prefix .. MARKER_PATH) + CacheFs.remove(MARKER_PATH) + -- the old pack too, as the worker does (see rom_import_worker) + local cleared, clearError = CacheFs.clearPack(prefix) + -- no other Lua state is involved on this path, so the hold taken above + -- (for the worker's sake) can go straight away + CacheFs.releasePack(prefix) + if not cleared then + error("could not remove the previous game data: " .. tostring(clearError)) + end removeTree(prefix .. "data/generated") removeTree(prefix .. "assets/generated") - love.filesystem.remove(prefix .. MARKER_PATH) CacheFs.removeTree("data/generated") CacheFs.removeTree("assets/generated") - CacheFs.remove(MARKER_PATH) BootTrace.mark("import " .. tostring(version) .. ": old cache cleared") + local packing = CacheFs.packedMode() and prefix ~= "" and CacheFs.beginPack() -- WHICH EXTRACTOR, and spelled out rather than inferred. -- @@ -3753,8 +3810,24 @@ function RomImporter:startData(data, displayName, sourcePath, verified) or RomExtractor.new(self.romData, manifest, onProgress) end extractor.checkpoint=function(operation) require('src.import.ImportRecovery').stage(markedStage,operation) end - extractor:run() + if packing then + -- a half-written pack is deleted, never left to look like a cache + local runOk, runError = pcall(extractor.run, extractor) + if not runOk then + CacheFs.abortPack() + error(runError, 0) + end + else + extractor:run() + end if extractor.close then extractor:close() end + if packing then + local finished, stats = CacheFs.finishPack() + if not finished then error("could not finish the packed game data: " .. tostring(stats)) end + require("src.core.Logger").info("import %s: packed %d files (%.1f MB), %d left loose", + tostring(version), stats.entries, stats.bytes / 1048576, stats.overflow) + CacheFs.mountPack(prefix) + end self.romData = nil collectgarbage("collect") BootTrace.mark(("import %s: extracted (%.0f MB lua)") @@ -4044,7 +4117,15 @@ function RomImporter:_advanceRomQueue() q.index = (q.index or 0) + 1 if require('src.import.RomImportTask').available() then q.active=nextUp - self:startPath(nextUp.path) + -- A save-directory entry is named relative to the save directory, and the + -- worker opens it with io.open, which resolves against the process's + -- working directory -- not the save directory (a PS5 title rejects the + -- relative name outright with EINVAL). Hand it the file's real path. + local path = nextUp.path + if nextUp.saveDir then + path = realPathFor(path) or (love.filesystem.getSaveDirectory() .. "/" .. path) + end + self:startPath(path) if self.workState~='working' then self._romQueueResume=true end return end @@ -4602,7 +4683,8 @@ function RomImporter:chooseMod() local rel = RomImporter.mtpHintPath(saveDir) if rel ~= "" and rel:sub(-1) ~= "/" then rel = rel .. "/" end self.modNotice = { ok = true, text = - Strings("Copy your mod .zip into:\n%s/imports/mods/\nDBI MTP -> 1: SD Card/%simports/mods/", saveDir, rel) } + Strings("Copy your mod .zip into:\n%s/imports/mods/\n%s", saveDir, + RomImporter.transferHint(saveDir, rel, "mods/")) } return end self.modNotice = { ok = false, @@ -4730,8 +4812,9 @@ function RomImporter:chooseModImport(modId) if rel ~= "" and rel:sub(-1) ~= "/" then rel = rel .. "/" end local want = table.concat(exts, "/") self.modNotice = { ok = true, text = - Strings("Copy your %s (.%s) into:\n%s/imports/\nDBI MTP -> 1: SD Card/%simports/", - tostring(entry.name), want, saveDir, rel) } + Strings("Copy your %s (.%s) into:\n%s/imports/\n%s", + tostring(entry.name), want, saveDir, + RomImporter.transferHint(saveDir, rel, "")) } end -- Which game a dropped .sav imports into: a .sav has no version signature of @@ -4831,7 +4914,8 @@ function RomImporter:chooseSaveImport(version) local rel = RomImporter.mtpHintPath(saveDir) if rel ~= "" and rel:sub(-1) ~= "/" then rel = rel .. "/" end self.saveNotice[version] = { ok = true, text = - Strings("Copy your .sav into:\n%s/imports/saves/\nDBI MTP -> 1: SD Card/%simports/saves/", saveDir, rel) } + Strings("Copy your .sav into:\n%s/imports/saves/\n%s", saveDir, + RomImporter.transferHint(saveDir, rel, "saves/")) } return end self.saveNotice[version] = { ok = false, @@ -5011,7 +5095,8 @@ function RomImporter:choose(version) self.notice = { version = self.chooseVersion, status = "Copy your .gb/.gbc into:", - detail = Strings("%s/imports/\nDBI MTP -> 1: SD Card/%simports/", saveDir, rel), + detail = Strings("%s/imports/\n%s", saveDir, + RomImporter.transferHint(saveDir, rel, "")), } return end @@ -6346,6 +6431,8 @@ function RomImporter:draw() label = Strings("Update from the OTA launcher (gen2recomp.nro)") elseif upAdvice == "package" then label = Strings("Install the newer package to update") + elseif upAdvice == "homebrew" then + label = Strings("Copy the new release to your PS5 to update") elseif upCannotCheck then label = Strings("Cannot check for updates on this build") action = "openurl" diff --git a/src/import/rom_import_worker.lua b/src/import/rom_import_worker.lua index b55b79831..9dc495a73 100644 --- a/src/import/rom_import_worker.lua +++ b/src/import/rom_import_worker.lua @@ -78,24 +78,48 @@ local ok,err=xpcall(function() assert(written,'Data folder is not writable: '..tostring(writeError)) assert(CacheFs.read(probe)=='import write check','Could not read back from the selected data folder.') CacheFs.remove(probe) + local removed=0 local function removeSaveTree(path) local info=love.filesystem.getInfo(path) if not info then return end if info.type=='directory' then for _,name in ipairs(love.filesystem.getDirectoryItems(path)) do removeSaveTree(path..'/'..name) end end - love.filesystem.remove(path) + if love.filesystem.remove(path) then removed=removed+1 end end if job.clear then + local clearStarted=love.timer.getTime() + -- The marker goes FIRST: from here until the launcher writes a new one, a + -- killed import must read as "not imported", whatever is left on disk. + love.filesystem.remove(CacheFs.prefix..'rom-cache.complete') + CacheFs.remove('rom-cache.complete') + -- The old pack as well as the loose trees, whichever kind this import + -- writes: a pack left behind a loose import would go on answering for every + -- file the new one does not produce. The launcher unmounted it before + -- starting this worker; a pack that still cannot be deleted is still open + -- somewhere, and replacing the cache under it is not possible. + local cleared,clearError=CacheFs.clearPack() + assert(cleared,'Could not remove the previous game data: '..tostring(clearError)) for _,rel in ipairs({'data/generated','assets/generated'}) do removeSaveTree(CacheFs.prefix..rel);CacheFs.removeTree(rel) end - love.filesystem.remove(CacheFs.prefix..'rom-cache.complete') - CacheFs.remove('rom-cache.complete') + -- Measured because the first packed import on a console that still holds a + -- loose cache deletes ~18,000 files one slow syscall at a time. + channel:push({kind='log',message=('import %s: old cache cleared in %.1f s (%d save-directory entries removed)') + :format(tostring(job.version),love.timer.getTime()-clearStarted,removed)}) end -- This state only writes the cache; the launcher writes the marker afterwards, -- through the ordinary synced path (see CacheFs.bulkWrites). CacheFs.bulkWrites=true + -- ONE ARCHIVE INSTEAD OF ~18,000 FILES where file creation is the cost (the + -- PS5; see CacheFs "THE PACKED CACHE"). Begun after the probe and the clear, + -- so both of those stay ordinary loose operations. + if job.packed then + local packed,packWhy=CacheFs.beginPack() + if not packed then + channel:push({kind='log',level='warn',message='import '..tostring(job.version)..': packed cache unavailable, writing loose files: '..tostring(packWhy)}) + end + end local lastStage,lastAt=nil,0 local function progress(value,total,stage,current,stageTotal) local now=love.timer.getTime() @@ -114,9 +138,25 @@ local ok,err=xpcall(function() extractor.checkpoint=function(operation) Recovery.stage(lastStage,operation) end extractor:run() if extractor.close then extractor:close();extractor=nil end + -- The pack becomes a pack here -- central directory written, `.part` + -- renamed -- and only then is the launcher told the import is complete, so + -- the marker it writes next can never vouch for an archive that is not whole. + if CacheFs.packActive and CacheFs.packActive() then + Recovery.stage(lastStage,'Finishing packed game data') + local packStarted=love.timer.getTime() + local finished,stats=CacheFs.finishPack() + assert(finished,'Could not finish the packed game data: '..tostring(stats)) + channel:push({kind='log',message=('import %s: packed %d files (%.1f MB) into %s in %.1f s; %d rewritten, %d left loose (archive full)') + :format(tostring(job.version),stats.entries,stats.bytes/1048576,tostring(stats.path), + love.timer.getTime()-packStarted,stats.duplicates,stats.overflow), + level=stats.overflow>0 and 'warn' or nil}) + end channel:push({kind='complete'}) end,debug.traceback) if not ok then + -- a half-written pack is deleted, never left to look like a cache + local CacheFs=package.loaded['src.import.CacheFs'] + if CacheFs and CacheFs.abortPack then pcall(CacheFs.abortPack) end if extractor and extractor.close then pcall(extractor.close,extractor) end channel:push({kind='error',error=tostring(err)}) end diff --git a/src/translation/Service.lua b/src/translation/Service.lua index 205c92e4a..76dbad987 100644 --- a/src/translation/Service.lua +++ b/src/translation/Service.lua @@ -138,10 +138,19 @@ function Service.update() local info=require("src.core.GameVersion").info(job.version) or {} local root=require("src.import.CacheFs").root() or love.filesystem.getSaveDirectory() local directory=root.."/"..(info.cachePrefix or "").."data/generated/" + local packedDirectory=nil if job.version~="launcher"then local textFile=io.open(directory.."text.lua","rb") - if not textFile then return end - textFile:close() + if textFile then textFile:close() + else + -- a packed cache (the PS5's rom-cache.zip) has no loose text.lua on disk; + -- its prefixed mount serves the same files through love.filesystem, which + -- the worker thread shares, so the worker reads them from there instead + local prefix=info.cachePrefix or "" + if prefix=="" or not require("src.import.CacheFs").mountPack(prefix) + or not love.filesystem.getInfo(prefix.."data/generated/text.lua","file") then return end + packedDirectory=prefix.."data/generated/" + end end love.filesystem.createDirectory("translations/fonts") local font,url=Languages.font(target) @@ -155,7 +164,7 @@ function Service.update() -- app has them is asked here, on the main thread local android=love.system.getOS()=="Android" Service.thread=love.thread.newThread("src/translation/worker.lua") - Service.thread:start(Json.encode({source=source,target=target,dataDirectory=directory,launcher=job.version=="launcher", + Service.thread:start(Json.encode({source=source,target=target,dataDirectory=directory,packedDirectory=packedDirectory,launcher=job.version=="launcher", hardware=job.options.translationHardware or "auto", android=android,bridge=android and type(love.system.translateOffline)=="function", cachePath=path(job.version,source,target),font=font,fontUrl=url, diff --git a/src/translation/worker.lua b/src/translation/worker.lua index 6afe1745a..ddd4ebd98 100644 --- a/src/translation/worker.lua +++ b/src/translation/worker.lua @@ -39,8 +39,14 @@ local ok,err=xpcall(function() else for _,name in ipairs(require("src.translation.Sources").MODULES)do local file=io.open(job.dataDirectory..name..".lua","rb") - if file then - local chunk=loadstring(file:read("*a"),"translation source "..name);file:close() + local bytes=nil + if file then bytes=file:read("*a");file:close() + elseif job.packedDirectory then + -- packed cache: no loose file, so read it through the archive's mount + bytes=love.filesystem.read(job.packedDirectory..name..".lua") + end + if bytes then + local chunk=loadstring(bytes,"translation source "..name) if chunk then setfenv(chunk,{});local valid,table_=pcall(chunk);if valid and type(table_)=="table"then data[name]=table_ end end end end diff --git a/src/ui/OptionsMenu.lua b/src/ui/OptionsMenu.lua index d20c7b834..67c3bfdf8 100644 --- a/src/ui/OptionsMenu.lua +++ b/src/ui/OptionsMenu.lua @@ -577,6 +577,17 @@ local function buildRows(game) rows = filtered end end + -- VIDEO MODE and FAITHFUL RES resize the window; the PS5's is the fixed + -- full-screen TV output, so neither row means anything there. + if require("src.core.Platform").isPS5() then + local filtered = {} + for _, row in ipairs(rows) do + if row.id ~= "videoMode" and row.id ~= "faithfulRes" then + filtered[#filtered + 1] = row + end + end + rows = filtered + end -- PARTY ICONS is a Gen2 switch: Gen1's party rows draw menu icon classes -- and have no dex-pic mode to toggle to. if not GameVersion.isGen2() then diff --git a/src/update/Check.lua b/src/update/Check.lua index 901392a89..c1caa6362 100644 --- a/src/update/Check.lua +++ b/src/update/Check.lua @@ -182,6 +182,11 @@ function Check.capability() or os.getenv("POKEPORT_NOTIFY_ONLY_UPDATES") == "1" if suppressed then cap.mode, cap.reason = "suppressed", "POKEPORT_NOTIFY_ONLY_UPDATES is set" + elseif host == "PS5" then + -- A homebrew title is installed by the player (ShadowMountPlus mounts the + -- game.love they copied over FTP), so it never replaces itself: the banner + -- only says a release exists. + cap.mode, cap.reason = "notify-only", "PS5 titles are updated by hand" elseif not canHost then cap.mode, cap.reason = "unavailable", "the save directory did not take a test write, so no payload could be mounted" @@ -204,11 +209,14 @@ function Check.capability() -- "download" self-update works here -- "ota" NX -- the native OTA launcher NRO is the update path -- "package" a packaged container (UWP/MSIX) -- install a newer package + -- "homebrew" PS5 homebrew -- copy the new release over FTP -- "releases" no transport, but a desktop that can open a page if cap.mode == "self-update" then cap.advice = "download" elseif host == "NX" then cap.advice = "ota" + elseif host == "PS5" then + cap.advice = "homebrew" elseif Platform.isPackagedContainer() then cap.advice = "package" else @@ -229,7 +237,7 @@ end -- The launcher turns this into a sentence and decides whether to draw a button; -- tools/auto_update_check.lua asserts the launcher handles every value this can -- return, so a fifth one cannot be added here and ignored there. -Check.ADVICE = { download = true, ota = true, package = true, releases = true } +Check.ADVICE = { download = true, ota = true, package = true, homebrew = true, releases = true } function Check.advice() return Check.capability().advice or "releases" diff --git a/tools/cache_pack_check.lua b/tools/cache_pack_check.lua new file mode 100644 index 000000000..f139229f9 --- /dev/null +++ b/tools/cache_pack_check.lua @@ -0,0 +1,267 @@ +-- THE PACKED CACHE: the store-only ZIP the import writes on the PS5 instead of +-- ~18,000 loose files (src/import/CachePack, CacheFs "THE PACKED CACHE"). +-- +-- What has to hold, and what each block below proves: +-- +-- * the CRC is the real CRC-32 -- PhysFS verifies it when an entry is read +-- to the end, so a wrong one is a cache that loads half a sprite and then +-- fails -- and the FFI loop agrees with the plain one; +-- * a pack round-trips: every name, every byte, empty files and all 256 byte +-- values included; a name written twice keeps the LAST bytes, like an +-- overwritten loose file; +-- * an import can read back what it already wrote, before the archive has a +-- central directory; +-- * the zip64 limits are refused rather than silently corrupting the +-- archive, and CacheFs then writes the file loose so the cache is still +-- whole; +-- * CacheFs.read / exists answer from the pack once the import is over, with +-- a loose file of the same name winning. +-- +-- No PhysFS here (mount behaviour is checked by the headless import/boot runs +-- in the acceptance notes); this is the bytes. +-- python tools/run_lua_check.py tools/cache_pack_check.lua + +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.harness") +local check, eq = T.check, T.eq +love = love or require("tests.love_stub") + +local CachePack = require("src.import.CachePack") +local CacheFs = require("src.import.CacheFs") + +local SEP = package.config:sub(1, 1) +local TMP = (os.getenv("TEMP") or os.getenv("TMPDIR") or "/tmp") + .. SEP .. ("g2r_cache_pack_%d_%d"):format(os.time(), math.random(1, 1e6)) +CacheFs.mkdirReal(TMP) + +local made = {} +local function path(name) + local p = TMP .. SEP .. name + made[#made + 1] = p + return p +end + +local function readAll(p) + local f = io.open(p, "rb") + if not f then return nil end + local s = f:read("*a") + f:close() + return s +end + +-- ---------------------------------------------------------------- CRC-32 +eq(CachePack.crc32("123456789"), 0xCBF43926, "CRC-32 check value") +eq(CachePack.crc32(""), 0, "CRC-32 of nothing") +eq(CachePack.crc32("The quick brown fox jumps over the lazy dog"), 0x414FA339, + "CRC-32 of the pangram") +do + local parts = {} + for i = 0, 255 do parts[#parts + 1] = string.char(i) end + local all = table.concat(parts):rep(37) + eq(CachePack.crc32(all), CachePack.crc32(all, nil, true), + "the FFI loop and the plain loop agree") + eq(CachePack.crc32(all:sub(1001), CachePack.crc32(all:sub(1, 1000))), + CachePack.crc32(all), "a running CRC continues") +end + +-- ---------------------------------------------------------------- round trip +local bytes256 +do + local parts = {} + for i = 0, 255 do parts[#parts + 1] = string.char(i) end + bytes256 = table.concat(parts) +end + +local files = { + ["data/generated/constants.lua"] = "return { answer = 42 }\n", + ["data/generated/empty.lua"] = "", + ["assets/generated/gen4/sprites/a.png"] = bytes256:rep(300), + ["assets/generated/gen4/deep/er/still/b.bin"] = ("\0\255"):rep(5000), +} + +local packPath = path("round.zip") +local w = assert(CachePack.create(packPath)) +for name, data in pairs(files) do + check(w:add(name, data), "add " .. name) +end +-- written twice: last write wins, counted as a duplicate +check(w:add("data/generated/constants.lua", "return 'second'"), "re-add") +files["data/generated/constants.lua"] = "return 'second'" +-- added and then removed (CacheFs.remove mid-import) +check(w:add("data/generated/gone.lua", "x"), "add a file to remove") +w:remove("data/generated/gone.lua") + +-- read back BEFORE finish: an import that consults its own earlier output +eq(w:read("assets/generated/gen4/sprites/a.png"), files["assets/generated/gen4/sprites/a.png"], + "an entry reads back while the pack is still being written") +eq(w:read("data/generated/constants.lua"), "return 'second'", + "the in-progress read sees the last write") +check(w:add("data/generated/after_read.lua", "appended after a read"), + "an add after a read still appends") +files["data/generated/after_read.lua"] = "appended after a read" +check(not w:has("data/generated/gone.lua"), "a removed entry is gone") +eq(w.duplicates, 1, "one rewritten entry") +check(w:finish(), "finish") + +local r = assert(CachePack.open(packPath)) +local count = 0 +for _ in pairs(files) do count = count + 1 end +eq(r:count(), count, "every name once in the central directory") +for name, data in pairs(files) do + eq(r:read(name), data, "round trip " .. name) +end +check(not r:has("data/generated/gone.lua"), "the removed entry is not listed") +check(r:verify(), "every stored CRC matches its bytes") + +-- the end record says what the central directory says +do + local raw = readAll(packPath) + local tail = raw:sub(-22) + local function r16(s, i) local a, b = s:byte(i, i + 1) return a + b * 256 end + check(tail:sub(1, 4) == "PK\5\6", "end-of-central-directory signature") + eq(r16(tail, 9), count, "entries on this disk") + eq(r16(tail, 11), count, "entries in total") + check(raw:sub(1, 4) == "PK\3\4", "starts with a local header") +end + +-- a corrupt tail is refused, not misread +do + local bad = path("bad.zip") + local f = io.open(bad, "wb"); f:write("not a zip at all, honestly"); f:close() + check(CachePack.open(bad) == nil, "a file with no end record is not a pack") +end + +-- ---------------------------------------------------------------- limits +do + local full = path("full.zip") + local fw = assert(CachePack.create(full)) + local okAll = true + for i = 1, CachePack.MAX_ENTRIES do + if not fw:add(("e/%05d"):format(i), "x") then okAll = false break end + end + check(okAll, "a pack takes " .. CachePack.MAX_ENTRIES .. " entries") + local ok, why = fw:add("e/one-too-many", "x") + check(not ok and why == "full", "entry " .. (CachePack.MAX_ENTRIES + 1) .. " is refused as full") + check(fw:add("e/00001", "y"), "rewriting an existing name is still allowed when full") + check(fw:finish(), "a full pack still finishes") + local fr = assert(CachePack.open(full)) + eq(fr:count(), CachePack.MAX_ENTRIES, "and reads back with every entry") + eq(fr:read("e/00001"), "y", "including the rewritten one") + eq(fr:read(("e/%05d"):format(CachePack.MAX_ENTRIES)), "x", "and the last one") + check(CachePack.MAX_ENTRIES < 0xFFFF, "the entry cap stays below the zip64 marker") +end +do + local saved = CachePack.MAX_BYTES + CachePack.MAX_BYTES = 4096 + local big = path("big.zip") + local bw = assert(CachePack.create(big)) + check(bw:add("a", ("a"):rep(1000)), "under the byte cap") + local ok, why = bw:add("b", ("b"):rep(4000)) + check(not ok and why == "full", "an entry that would cross the byte cap is refused") + bw:abort() + check(readAll(big) == nil, "abort deletes the partial file") + CachePack.MAX_BYTES = saved +end + +-- ---------------------------------------------------------------- CacheFs +-- The import's view: CacheFs.write into the pack, read-back during the +-- import, overflow to loose, then reads through the finished pack. +do + local root = TMP .. SEP .. "root" + CacheFs.mkdirReal(root) + local realRoot = CacheFs.root + CacheFs.root = function() return root end + CacheFs.prefix = "platinum/" + + check(CacheFs.beginPack(), "beginPack under a root") + check(CacheFs.packActive(), "a pack is being written") + local savedMax = CachePack.MAX_ENTRIES + CachePack.MAX_ENTRIES = 2 + check(CacheFs.write("data/generated/a.lua", "A"), "write a -> pack") + check(CacheFs.write("assets/generated/x/b.png", "B"), "write b -> pack") + check(CacheFs.write("data/generated/c.lua", "C"), "write c -> loose (pack full)") + check(CacheFs.write("notes.txt", "N"), "a file outside the generated trees") + CachePack.MAX_ENTRIES = savedMax + + eq(CacheFs.read("data/generated/a.lua"), "A", "read back from the pack mid-import") + check(CacheFs.exists("assets/generated/x/b.png"), "exists from the pack mid-import") + local function real(rel) return root .. SEP .. ("platinum/" .. rel):gsub("/", SEP) end + check(readAll(real("data/generated/a.lua")) == nil, "a packed file is not on disk loose") + eq(readAll(real("data/generated/c.lua")), "C", "the overflow file is on disk loose") + eq(readAll(real("notes.txt")), "N", "non-generated files stay loose") + check(readAll(real("rom-cache.zip")) == nil, "no final pack before finish") + check(readAll(real("rom-cache.zip.part")) ~= nil, "a .part while writing") + + local ok, stats = CacheFs.finishPack() + check(ok, "finishPack: " .. tostring(stats)) + eq(stats and stats.entries, 2, "two entries packed") + eq(stats and stats.overflow, 1, "one left loose") + check(not CacheFs.packActive(), "no pack being written after finish") + check(readAll(real("rom-cache.zip.part")) == nil, "the .part was renamed away") + check(CacheFs.packExists("platinum/"), "the pack exists under its final name") + + CacheFs.forgetPacks() + eq(CacheFs.read("data/generated/a.lua"), "A", "read through the finished pack") + eq(CacheFs.read("assets/generated/x/b.png"), "B", "and an asset") + eq(CacheFs.read("data/generated/c.lua"), "C", "and the overflow, loose") + check(not CacheFs.exists("data/generated/nope.lua"), "a missing file is missing") + + -- a loose file of the same name wins over the pack (mods, Data.lua's copies) + local f = io.open(real("data/generated/a.lua"), "wb"); f:write("LOOSE"); f:close() + eq(CacheFs.read("data/generated/a.lua"), "LOOSE", "loose beats the pack") + os.remove(real("data/generated/a.lua")) + + -- the marker is never packed + check(CacheFs.write("rom-cache.complete", "M"), "marker write") + eq(readAll(real("rom-cache.complete")), "M", "the marker is a loose file") + + -- an aborted import leaves nothing that looks like a cache + check(CacheFs.beginPack(), "a second import begins") + check(CacheFs.write("data/generated/a.lua", "A2"), "writes into the new .part") + CacheFs.abortPack() + check(readAll(real("rom-cache.zip.part")) == nil, "abort removed the .part") + check(CacheFs.packExists("platinum/"), "and left the finished pack alone") + + -- clearing removes the pack and any .part + local part = io.open(real("rom-cache.zip.part"), "wb"); part:write("junk"); part:close() + check(CacheFs.clearPack("platinum/"), "clearPack") + check(readAll(real("rom-cache.zip")) == nil, "the pack is gone") + check(readAll(real("rom-cache.zip.part")) == nil, "and the stale .part") + CacheFs.forgetPacks() + check(not CacheFs.exists("data/generated/a.lua"), "nothing answers from a cleared pack") + + -- Red is never packed + CacheFs.prefix = "" + local redOk = CacheFs.beginPack() + check(not redOk, "Red's cache is never packed") + + for _, rel in ipairs({ "data/generated/c.lua", "notes.txt", "rom-cache.complete" }) do + os.remove(real(rel)) + end + for _, d in ipairs({ "platinum/data/generated", "platinum/assets/generated/x", + "platinum/assets/generated", "platinum/data", + "platinum/assets", "platinum" }) do + CacheFs.rmdirReal(root .. SEP .. d:gsub("/", SEP)) + end + CacheFs.rmdirReal(root) + CacheFs.root = realRoot + CacheFs.prefix = "" +end + +-- ---------------------------------------------------------------- mode +do + -- POKEPORT_PACKED_CACHE is read from the environment; without it, only the + -- PS5 packs. This run is not a PS5. + local env = os.getenv("POKEPORT_PACKED_CACHE") + if env == nil then + check(not CacheFs.packedMode(), "loose by default off the PS5") + elseif env == "1" then + check(CacheFs.packedMode(), "POKEPORT_PACKED_CACHE=1 forces packing") + end +end + +for _, p in ipairs(made) do os.remove(p) end +CacheFs.rmdirReal(TMP) + +T.finish("cache pack")