diff --git a/.github/scripts/ft-wasm/build-wasm.sh b/.github/scripts/ft-wasm/build-wasm.sh new file mode 100755 index 00000000..32886130 --- /dev/null +++ b/.github/scripts/ft-wasm/build-wasm.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# Build free-threaded CPython (--disable-gil) with pthreads for wasm32-emscripten, +# with CPython's own driver (Platforms/emscripten). Step one of #175: a build +# experiment. Usage: build-wasm.sh [VERSION] +# +# Environment: +# FT_WORK scratch directory (default .tmp/ft-wasm) +# FT_OUT receives the archive the workflow uploads (default .tmp/ft-wasm-out) +# FT_WASM_EMSDK_VERSION Emscripten to install; empty = the version CPython itself pins +# (Platforms/emscripten/config.toml, 4.0.19 in 3.15.0) +# FT_WASM_TRAMPOLINE_PATCH apply (default) | skip; see patch_trampoline below +# +# Needs: a C toolchain for the native build Python, git, curl, node on PATH +# (CPython 3.15.0 pins node 24 in config.toml), python3 >= 3.11 to run the driver. +# +# This is a separate script, not a mode of python-multiplatform/scripts/wasm/build-cpython.sh: +# that script is pinned to CPython 3.14.2 / Emscripten 5.0.3 / pyemscripten_2026_0, clones a +# pinned git commit, writes into .caches/ and stages the runtime the wasmJs suite runs against. +# This job builds a different CPython (3.15.0 tarball) with a different Emscripten and must not +# touch that runtime. The flags below are the #175 investigation's proposal, not a decision. +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" + +export FT_WORK="${FT_WORK:-$PWD/.tmp/ft-wasm}" +export FT_OUT="${FT_OUT:-$PWD/.tmp/ft-wasm-out}" +# fetch_source, require_gil_disabled, record_sha256, py_minor +# shellcheck source-path=SCRIPTDIR source=../ft-mobile/common.sh +source "$here/../ft-mobile/common.sh" +log() { echo "[ft-wasm] $*" >&2; } +die() { echo "[ft-wasm] ERROR: $*" >&2; exit 1; } + +PY_VERSION="${1:-3.15.0}" +TRAMPOLINE_PATCH="${FT_WASM_TRAMPOLINE_PATCH:-apply}" +minor="$(py_minor "$PY_VERSION")" +HOST=wasm32-emscripten + +# Compile and link flags (agent proposal from the #175 investigation, not confirmed). +# The driver hardcodes CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 and appends user args after +# it; autoconf keeps the last assignment, so the full string is repeated here. +# -fwasm-exceptions -sSUPPORT_LONGJMP=wasm same unwinding ABI as build-cpython.sh +# -pthread atomics + shared memory (configure also adds it +# to CFLAGS_NODIST for --enable-wasm-pthreads) +# -sPTHREAD_POOL_SIZE=8 pre-created Workers: Thread.start() blocks until +# the thread runs, and a Worker cannot start while +# the JS thread that asked for it is blocked +# -sMAXIMUM_MEMORY=2GB shared memory needs a declared maximum +ABI_FLAGS="-fwasm-exceptions -sSUPPORT_LONGJMP=wasm" +HOST_CFLAGS="-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 $ABI_FLAGS -pthread" +HOST_LDFLAGS="$ABI_FLAGS -pthread -sPTHREAD_POOL_SIZE=8 -sMAXIMUM_MEMORY=2GB" + +src="$(fetch_source "$PY_VERSION")" +cd "$src" +[ -d Platforms/emscripten ] || die "no Platforms/emscripten in CPython $PY_VERSION (3.15+ expected)" + +# ---- Emscripten ----------------------------------------------------------- +# One emsdk under FT_WORK. The driver is then run without --emsdk-cache, so it uses the +# activated environment ("Build will use EMSDK from current environment"). +pinned="$(sed -n 's/^emscripten-version = "\(.*\)"$/\1/p' Platforms/emscripten/config.toml)" +[ -n "$pinned" ] || die "no emscripten-version in Platforms/emscripten/config.toml" +EMSDK_VERSION="${FT_WASM_EMSDK_VERSION:-$pinned}" +log "Emscripten pinned by CPython $PY_VERSION: $pinned; using: $EMSDK_VERSION" +emsdk="$FT_WORK/emsdk" +if [ ! -x "$emsdk/emsdk" ]; then + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$emsdk" +fi +"$emsdk/emsdk" install "$EMSDK_VERSION" +"$emsdk/emsdk" activate "$EMSDK_VERSION" +# shellcheck disable=SC1091 +EMSDK_QUIET=1 source "$emsdk/emsdk_env.sh" +emcc --version > "$FT_WORK/emcc-version.txt" +emcc_line="$(head -n 1 "$FT_WORK/emcc-version.txt")" +log "$emcc_line" +case "$emcc_line" in *" $EMSDK_VERSION "*) ;; *) die "emcc is not $EMSDK_VERSION" ;; esac +node --version + +# ---- patch: call trampoline under shared memory --------------------------- +# Python/emscripten_trampoline.c instantiates a wasm-gc trampoline module in pre-run. That +# module is compiled with -Wl,--import-memory and no -pthread (Makefile.pre.in), so it imports +# a NON-shared env.memory; giving it the shared wasmMemory of a pthread build is a link error, +# and the `new WebAssembly.Instance` call is outside the try block, so startup throws. +# Pyodide #237 also reports that the addFunction'ed pointer exists only in the main thread's +# table ("table index is out of bounds" on the first call from a spawned thread). +# Minimal fix: when memory is shared, return 0, which selects the existing JS trampoline +# (_PyEM_TrampolineCall_JS, wasmTable.get(func) on the calling thread). The edit is asserted, +# so an upstream change fails the build instead of silently skipping the patch. +# FT_WASM_TRAMPOLINE_PATCH=skip leaves the source alone, to observe the unpatched failure. +patch_trampoline() { + python3 - Python/emscripten_trampoline.c <<'PATCH' +import sys +from pathlib import Path +p = Path(sys.argv[1]); t = p.read_text() +marker = "pmp-175: shared memory" +if marker not in t: + old = "function getPyEMTrampolinePtr() {\n" + new = (old + + " // " + marker + ": the wasm-gc trampoline imports non-shared memory and its\n" + " // addFunction pointer is valid only on this thread; use the JS trampoline.\n" + " if (typeof SharedArrayBuffer !== \"undefined\" && wasmMemory.buffer instanceof SharedArrayBuffer) {\n" + " return 0;\n" + " }\n") + assert t.count(old) == 1, "getPyEMTrampolinePtr changed shape; re-check the patch" + p.write_text(t.replace(old, new)) + print("patched Python/emscripten_trampoline.c (shared memory -> JS trampoline)") +PATCH +} +case "$TRAMPOLINE_PATCH" in + apply) patch_trampoline ;; + skip) log "trampoline patch skipped (FT_WASM_TRAMPOLINE_PATCH=skip)" ;; + *) die "FT_WASM_TRAMPOLINE_PATCH must be apply or skip, got '$TRAMPOLINE_PATCH'" ;; +esac + +# ---- native build Python (free-threaded too, as in ft-mobile) ------------- +python3 --version +python3 Platforms/emscripten configure-build-python -- --disable-gil +python3 Platforms/emscripten make-build-python +python3 Platforms/emscripten pythoninfo-build +cross-build/build/python -c 'import sys, sysconfig; assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1; print("build python free-threaded:", sys.version)' + +# ---- libffi and mpdec with -pthread --------------------------------------- +# The driver builds them with CFLAGS="-O2 -fPIC -DWASM_BIGINT" (make_libffi.sh) and +# CFLAGS=-fPIC (mpdec). emcc appends EMCC_CFLAGS to every invocation, so -pthread reaches +# both without editing the driver. Checked below by the objects' target_features section. +EMCC_CFLAGS="-pthread" python3 Platforms/emscripten make-libffi +EMCC_CFLAGS="-pthread" python3 Platforms/emscripten make-mpdec + +prefix="cross-build/$HOST/prefix" +for lib in "$prefix/lib/libffi.a" "$prefix/lib/libmpdec.a"; do + [ -f "$lib" ] || die "no $lib" + # wasm target_features entries are : "+", 7, "atomics". + python3 - "$lib" <<'CHECK' || die "$lib was not compiled with atomics (-pthread)" +import sys +data = open(sys.argv[1], "rb").read() +sys.exit(0 if b"+\x07atomics" in data else 1) +CHECK + log "ok: $lib has +atomics" +done + +# ---- host configure ------------------------------------------------------- +python3 Platforms/emscripten configure-host --clean --host-runner node -- \ + "CFLAGS=$HOST_CFLAGS" \ + "LDFLAGS=$HOST_LDFLAGS" \ + --disable-gil --enable-wasm-pthreads + +hostdir="cross-build/$HOST/build/python" +require_gil_disabled "$hostdir/pyconfig.h" +# configure adds -sUSE_PTHREADS to LDFLAGS_NODIST (CONFIGURE_LDFLAGS_NODIST in the +# Makefile) only when --enable-wasm-pthreads took effect. +grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \ + || die "configure did not enable wasm pthreads (no -sUSE_PTHREADS in LDFLAGS_NODIST)" +grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t" +log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t" + +# ---- host build ----------------------------------------------------------- +# make-host also runs `python.sh --version`, so a startup failure (for example the trampoline +# instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step. +python3 Platforms/emscripten make-host + +# ---- verification --------------------------------------------------------- +# python.wasm must use shared memory with a declared maximum. +node - "$hostdir/python.wasm" <<'NODE' || die "python.wasm has no shared memory" +const b = require("fs").readFileSync(process.argv[2]); +const leb = (i) => { let r = 0, s = 0; for (;;) { const x = b[i++]; r += (x & 0x7f) * 2 ** s; s += 7; if (!(x & 0x80)) return [r, i]; } }; +const str = (i) => { let n; [n, i] = leb(i); return [b.slice(i, i + n).toString(), i + n]; }; +const limits = (i) => { const flags = b[i++]; let min, max = null; [min, i] = leb(i); if (flags & 1) [max, i] = leb(i); return [{ flags, min, max }, i]; }; +let i = 8, mem = null; +while (i < b.length && !mem) { + const id = b[i++]; let size; [size, i] = leb(i); const end = i + size; + if (id === 2) { let n, j; [n, j] = leb(i); + for (let k = 0; k < n && !mem; k++) { let m, f; [m, j] = str(j); [f, j] = str(j); const kind = b[j++]; + if (kind === 0) { [, j] = leb(j); } + else if (kind === 1) { j++; [, j] = limits(j); } + else if (kind === 2) { let l; [l, j] = limits(j); mem = { where: `import ${m}.${f}`, ...l }; } + else if (kind === 3) { j += 2; } + else if (kind === 4) { j++; [, j] = leb(j); } + else throw new Error(`unknown import kind ${kind}`); } } + if (id === 5) { let j; [, j] = leb(i); const [l] = limits(j); mem = { where: "defined", ...l }; } + i = end; +} +if (!mem) { console.error("no memory found"); process.exit(1); } +const shared = (mem.flags & 2) !== 0; +console.log(`python.wasm memory: ${mem.where} shared=${shared} min=${mem.min} max=${mem.max} pages`); +process.exit(shared && mem.max !== null ? 0 : 1); +NODE + +# The interpreter itself: GIL off at run time, four real threads. +run_smoke() { + local label="$1"; shift + local out="$FT_WORK/smoke-$label.log" rc + set +e + timeout 600 "$@" "$here/smoke.py" > "$out" 2>&1 + rc=$? + set -e + cat "$out" + [ "$rc" -eq 0 ] || die "smoke test ($label) exited $rc (124 = timed out, a thread start may have deadlocked)" + grep -q '^FT-WASM-SMOKE-OK ' "$out" || die "smoke test ($label) did not print FT-WASM-SMOKE-OK" + log "ok: smoke test ($label)" +} +run_smoke build-tree "$PWD/$hostdir/python.sh" + +# ---- archive -------------------------------------------------------------- +# The driver has no install or package step for Emscripten, so the archive is assembled here. +# The stdlib directory is named by the built interpreter's own sysconfig, so a non-FT build +# (python3.15) fails the name check instead of being renamed. +stdlib_name="$(timeout 300 "$PWD/$hostdir/python.sh" -c 'import os, sysconfig; print(os.path.basename(sysconfig.get_path("stdlib")))' | tail -n 1)" +[ "$stdlib_name" = "python${minor}t" ] || die "interpreter's stdlib dir is '$stdlib_name', expected python${minor}t" +log "ok: interpreter reports lib/$stdlib_name" + +name="python-${PY_VERSION}t-$HOST" +stage="$FT_WORK/stage/$name" +rm -rf "$FT_WORK/stage"; mkdir -p "$stage/lib/$stdlib_name/lib-dynload" "$stage/include/$stdlib_name" +cp "$hostdir/python.mjs" "$hostdir/python.wasm" "$stage/" +cp "$hostdir/node_entry.mjs" "$hostdir/streams.mjs" "$stage/" +# Everything but the test suite; the zip-and-omit selection is left to whoever stages it. +tar -C Lib --exclude='./test' --exclude='__pycache__' -cf - . | tar -C "$stage/lib/$stdlib_name" -xf - +cp "$hostdir/$(cat "$hostdir/pybuilddir.txt")"/_sysconfigdata_*.py "$stage/lib/$stdlib_name/" +touch "$stage/lib/$stdlib_name/lib-dynload/.empty" +cp -R Include/. "$stage/include/$stdlib_name/" +cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/" +{ + echo "CPython $PY_VERSION, --disable-gil --enable-wasm-pthreads, $HOST" + echo "emcc: $emcc_line" + echo "CFLAGS=$HOST_CFLAGS" + echo "LDFLAGS=$HOST_LDFLAGS" + echo "trampoline patch: $TRAMPOLINE_PATCH" + echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)" +} > "$stage/BUILD-INFO.txt" + +archive="$FT_OUT/$name.tar.gz" +mkdir -p "$FT_OUT" +tar -C "$FT_WORK/stage" -czf "$archive" "$name" + +# Re-check the archive itself, unpacked somewhere else, and run it with PYTHONHOME pointing +# at it (node_entry.mjs copies process.env into the Emscripten ENV, and NODEFS mounts /). +check="$FT_WORK/wasm-check" +rm -rf "$check"; mkdir -p "$check" +tar -xzf "$archive" -C "$check" +root="$check/$name" +require_gil_disabled "$root/include/python${minor}t/pyconfig.h" +[ -d "$root/lib/python${minor}t" ] || die "no lib/python${minor}t in the archive" +[ -f "$root/lib/python${minor}t/os.py" ] || die "no lib/python${minor}t/os.py in the archive" +ls "$root/lib/python${minor}t"/_sysconfigdata_*.py > /dev/null || die "no _sysconfigdata in the archive" +nodeflags="" +[ "$(node -p 'process.versions.node.split(".")[0]')" = "24" ] && nodeflags="--experimental-wasm-jspi" +# shellcheck disable=SC2086 +run_smoke archive env PYTHONHOME="$root" node $nodeflags "$root/node_entry.mjs" \ + --this-program="$root/bin/python${minor}t" + +echo "wasm32-emscripten archive (Emscripten $EMSDK_VERSION, trampoline patch: $TRAMPOLINE_PATCH):" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +record_sha256 "$archive" diff --git a/.github/scripts/ft-wasm/smoke.py b/.github/scripts/ft-wasm/smoke.py new file mode 100644 index 00000000..78c3758c --- /dev/null +++ b/.github/scripts/ft-wasm/smoke.py @@ -0,0 +1,42 @@ +# Smoke test for the free-threaded wasm32-emscripten build (#175), run by build-wasm.sh +# under node. Prints one line starting with FT-WASM-SMOKE-OK, which the script greps for; +# any failed assertion exits non-zero instead. +import sys +import sysconfig +import threading + +assert sys.platform == "emscripten", sys.platform +assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1, "not a free-threaded build" +assert sys._is_gil_enabled() is False, "the GIL is enabled at run time" + +N_THREADS = 4 +N_ITER = 200_000 +results = [None] * N_THREADS + + +def work(i): + total = 0 + for k in range(N_ITER): + total += k * (i + 1) + results[i] = (threading.get_ident(), total) + + +threads = [threading.Thread(target=work, args=(i,)) for i in range(N_THREADS)] +for t in threads: + t.start() +for t in threads: + t.join(timeout=120) +alive = [t.name for t in threads if t.is_alive()] +assert not alive, f"threads still running after join: {alive}" + +expected = [(i + 1) * N_ITER * (N_ITER - 1) // 2 for i in range(N_THREADS)] +got = [r[1] if r else None for r in results] +assert got == expected, f"results {got} != {expected}" +idents = {r[0] for r in results} +assert len(idents) == N_THREADS, f"expected {N_THREADS} distinct thread idents, got {idents}" +assert threading.get_ident() not in idents, "a worker ran on the main thread" +# Importing threading must not have re-enabled the GIL. +assert sys._is_gil_enabled() is False, "the GIL was re-enabled during the run" + +print(f"FT-WASM-SMOKE-OK gil={sys._is_gil_enabled()} threads={N_THREADS} " + f"abiflags={sys.abiflags} version={sys.version.split()[0]}") diff --git a/.github/workflows/build-ft-wasm.yml b/.github/workflows/build-ft-wasm.yml new file mode 100644 index 00000000..b0242b7e --- /dev/null +++ b/.github/workflows/build-ft-wasm.yml @@ -0,0 +1,85 @@ +name: Build free-threaded CPython for wasm + +# Builds CPython with --disable-gil and pthreads ("3.15t") for wasm32-emscripten with +# CPython's own driver (Platforms/emscripten), then runs it under node with four threads. +# Step one of #175, a build experiment; job approved by the owner on 2026-11-13 ("#175 may +# add the new wasm 3.15t CI build job"). It does not touch wasm.yml or the runtime the +# wasmJs suite uses. See docs/platforms/free-threaded-wasm.md. +# +# Experiment choices (agent proposal, not decisions): Emscripten defaults to the version +# CPython itself pins (Platforms/emscripten/config.toml, 4.0.19 for 3.15.0), not the 5.0.3 +# of python-multiplatform/scripts/wasm/build-cpython.sh; the compile/link flags are the +# #175 investigation's; the call trampoline is patched to fall back to its JS path under +# shared memory (trampoline_patch=skip runs without it). + +on: + # Runs on develop when the build itself changes (workflow_dispatch needs the file on the + # default branch, main, which develop reaches only through a release). + push: + branches: [develop] + paths: + - ".github/workflows/build-ft-wasm.yml" + - ".github/scripts/ft-wasm/**" + - ".github/scripts/ft-mobile/common.sh" + # And on a pull request that changes it, so a change is built before it merges (a new workflow + # cannot be dispatched until it reaches main). + pull_request: + paths: + - ".github/workflows/build-ft-wasm.yml" + - ".github/scripts/ft-wasm/**" + - ".github/scripts/ft-mobile/common.sh" + workflow_dispatch: + inputs: + python_version: + description: CPython version to build (a python.org source release, 3.15 or later) + default: "3.15.0" + required: true + type: string + emscripten_version: + description: Emscripten to use (empty = the version CPython pins in Platforms/emscripten/config.toml) + default: "" + required: false + type: string + trampoline_patch: + description: Call-trampoline shared-memory patch (skip = observe the unpatched build) + default: apply + required: true + type: choice + options: [apply, skip] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + wasm: + name: wasm32-emscripten pthreads (free-threaded) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + FT_WORK: ${{ github.workspace }}/.tmp/ft-wasm + FT_OUT: ${{ github.workspace }}/.tmp/ft-wasm-out + FT_WASM_EMSDK_VERSION: ${{ inputs.emscripten_version || '' }} + FT_WASM_TRAMPOLINE_PATCH: ${{ inputs.trampoline_patch || 'apply' }} + steps: + - uses: actions/checkout@v4 + + # CPython 3.15.0 pins node 24 (Platforms/emscripten/config.toml, node-version). + - uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install build Python prerequisites + run: > + sudo apt-get update && sudo apt-get install -y + build-essential pkg-config libssl-dev zlib1g-dev libffi-dev + libbz2-dev liblzma-dev libsqlite3-dev libzstd-dev + + - name: Build and verify + run: .github/scripts/ft-wasm/build-wasm.sh "${{ inputs.python_version || '3.15.0' }}" + + - uses: actions/upload-artifact@v4 + with: + name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten + path: ${{ env.FT_OUT }}/*.tar.gz + if-no-files-found: error diff --git a/docs/platforms/free-threaded-wasm.md b/docs/platforms/free-threaded-wasm.md new file mode 100644 index 00000000..aab98375 --- /dev/null +++ b/docs/platforms/free-threaded-wasm.md @@ -0,0 +1,153 @@ +# Free-threaded CPython for wasm32-emscripten + +Status: build recipe written from the CPython 3.15.0 source; **not yet run in CI.** Nothing +here has been compiled, linked or run. Tracking issue: #175 (this is its first step, the +build experiment). The owner approved the CI job on 2026-11-13 ("#175 may add the new wasm +3.15t CI build job"). + +python-multiplatform gets its parallelism from free-threaded CPython (decision 8 in +`AGENTS.md`). The wasmJs target today runs a GIL, no-threads CPython 3.14.2 +([`wasm-design.md`](wasm-design.md)). No prebuilt free-threaded CPython with pthreads exists +for Emscripten, so the workflow `.github/workflows/build-ft-wasm.yml` builds one with +CPython's own driver and checks that it runs threads without the GIL under node. It does not +change `wasm.yml`, `python-multiplatform/scripts/wasm/build-cpython.sh` or the runtime the +wasmJs suite uses. + +## What is built + +| Host | Tooling | Output | +|---|---|---| +| `wasm32-emscripten`, pthreads, `--disable-gil` | `Platforms/emscripten` (CPython 3.15) | `python-3.15.0t-wasm32-emscripten.tar.gz` | + +The archive holds `python.mjs`, `python.wasm`, CPython's `node_entry.mjs` and `streams.mjs`, +`lib/python3.15t/` (the standard library without `test/`, plus `_sysconfigdata_*.py` and an +empty `lib-dynload/`), `include/python3.15t/` (with the host `pyconfig.h`) and +`BUILD-INFO.txt` (Emscripten version, flags, whether the trampoline patch was applied). The +driver has no install or package step for Emscripten, so the script assembles it. + +## Choices made for this experiment + +These are agent proposals from the #175 investigation, not decisions: + +- **Emscripten**: the version CPython itself pins (`emscripten-version` in + `Platforms/emscripten/config.toml`, 4.0.19 for 3.15.0), not the 5.0.3 that + `build-cpython.sh` pins for `pyemscripten_2026_0`. Which version the library should use is + open. The dispatch input `emscripten_version` overrides it, so 5.0.3 can be tried without + editing the script. +- **Node**: 24, the version `config.toml` pins, passed as `--host-runner node`. +- **Flags**: + + ```sh + python3 Platforms/emscripten configure-build-python -- --disable-gil + python3 Platforms/emscripten make-build-python + EMCC_CFLAGS=-pthread python3 Platforms/emscripten make-libffi + EMCC_CFLAGS=-pthread python3 Platforms/emscripten make-mpdec + python3 Platforms/emscripten configure-host --clean --host-runner node -- \ + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -pthread" \ + "LDFLAGS=-fwasm-exceptions -sSUPPORT_LONGJMP=wasm -pthread -sPTHREAD_POOL_SIZE=8 -sMAXIMUM_MEMORY=2GB" \ + --disable-gil --enable-wasm-pthreads + python3 Platforms/emscripten make-host + ``` + + The driver puts `CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2` before user arguments and + autoconf keeps the last assignment, so the full string is repeated. `--enable-wasm-pthreads` + makes configure add `-pthread`, `-sUSE_PTHREADS` and `-sPROXY_TO_PTHREAD` itself. + `-sPTHREAD_POOL_SIZE=8` pre-creates Workers, because `Thread.start()` waits for the new + thread and a Worker cannot start while the JS thread that requested it is blocked. + `-sMAXIMUM_MEMORY=2GB` gives the shared memory a maximum, which shared memory requires. +- **libffi and mpdec** are rebuilt with `-pthread` through `EMCC_CFLAGS` (emcc appends it to + every invocation), because the driver builds them without it and wasm-ld is expected to + refuse non-atomic objects in a shared-memory link. +- The build Python is free-threaded too, as in [`free-threaded-mobile.md`](free-threaded-mobile.md). + +## The one source patch + +`Python/emscripten_trampoline.c` instantiates a wasm-gc call trampoline in pre-run. That +module is compiled with `-Wl,--import-memory` and no `-pthread` (`Makefile.pre.in`), so it +imports a non-shared memory, and instantiating it against the shared memory of a pthread +build is expected to throw outside the `try` block, at startup. Pyodide #237 reports this on +3.14, plus a second problem: the trampoline's `addFunction` pointer exists only in the main +thread's table. `build-wasm.sh` adds an early `return 0` to `getPyEMTrampolinePtr()` when +`wasmMemory.buffer` is a `SharedArrayBuffer`, which selects CPython's existing JS trampoline +(`wasmTable.get(func)` on the calling thread). The edit is asserted, so an upstream change +fails the build instead of skipping the patch. The dispatch input `trampoline_patch: skip` +builds without it, to observe the unpatched failure. + +No patch is applied for `--disable-gil` itself; the investigation found none needed. + +## Scripts + +`.github/scripts/ft-wasm/`: + +- `build-wasm.sh [VERSION]`: unpacks `Python-.tgz` (through + `.github/scripts/ft-mobile/common.sh`), installs Emscripten into `$FT_WORK/emsdk`, applies + the patch, runs the steps above, verifies, assembles and re-verifies the archive, and + writes its sha256 to the job summary. Honours `FT_WORK`, `FT_OUT` (default under `.tmp/`), + `FT_WASM_EMSDK_VERSION` and `FT_WASM_TRAMPOLINE_PATCH`. +- `smoke.py`: run under node by the script, see below. + +The build is meant for CI (`AGENTS.md` rule 10). The workflow runs on `workflow_dispatch` +and on pushes to `develop` that change its own files. + +## Verification built into the workflow + +The script stops at the first failed check, in this order: + +1. `emcc --version` reports the requested Emscripten version. +2. The build Python reports `Py_GIL_DISABLED == 1`. +3. `libffi.a` and `libmpdec.a` carry `+atomics` in their wasm `target_features`. +4. The host `pyconfig.h` has `#define Py_GIL_DISABLED 1`, the host Makefile has + `-sUSE_PTHREADS` in `CONFIGURE_LDFLAGS_NODIST` and `ABIFLAGS` is `t`. +5. `make-host` links `python.wasm` and runs `python.sh --version` (a startup failure, for + example the trampoline, fails here). +6. `python.wasm` has a shared memory with a declared maximum (read from its import or memory + section; the page counts are logged). +7. `smoke.py` under `python.sh` in the build tree: `sys.platform == "emscripten"`, + `Py_GIL_DISABLED == 1`, `sys._is_gil_enabled() is False`, then four `threading.Thread`s + each compute a sum; all must join within 120 s with the right results and four distinct + thread idents, and the GIL must still be off. It prints `FT-WASM-SMOKE-OK ...`, which the + script greps for. The run has a 600 s timeout (exit 124 means a thread start may have + deadlocked). +8. The interpreter's own `sysconfig.get_path("stdlib")` ends in `python3.15t`; the archive's + stdlib directory is named from it. +9. The unpacked archive has `include/python3.15t/pyconfig.h` with `Py_GIL_DISABLED 1`, + `lib/python3.15t/os.py` and `_sysconfigdata_*.py`, and `smoke.py` passes again with + `PYTHONHOME` pointing at it. + +The archive is uploaded as the workflow artifact `python-t-wasm32-emscripten`. No +release is published or changed. + +## Not verified + +- The workflow has not run yet. Whether 3.15.0 configures, compiles and links with + `--disable-gil --enable-wasm-pthreads` on Emscripten 4.0.19 is unknown. The only real + evidence is upstream's free-threaded build **without** pthreads (CPython gh-158997, on + `main` with Emscripten 6.0.10) and Pyodide's **GIL** build with pthreads (3.14, Emscripten + 5.0.3). No project is known to have combined the two. +- Whether `EMCC_CFLAGS=-pthread` is enough for libffi and mpdec, and whether the + `-sUSE_BZIP2` port links into a shared-memory build, is a prediction. +- Whether the JS trampoline fallback is enough, and what it costs per call compared with the + wasm-gc trampoline (`WasmCallTrampolineTest` covers only the current GIL build). +- Only node is exercised; no browser (which also needs cross-origin isolation for + `SharedArrayBuffer`). +- The smoke test checks that four threads run and join with the GIL off. It is not a + thread-safety test, does not run CPython's test suite, and does not measure parallel + speed-up. +- mimalloc's WASI path on Emscripten has no thread-exit hook; the investigation predicts a + slow per-thread leak. Not measured. + +## Open: the Kotlin side (later steps of #175, not implemented) + +- Kotlin/Wasm imports `intrinsics.memory` as non-shared (`min=0, max=none`); a shared + memory cannot satisfy that import, so the library cannot instantiate this `python.wasm` + until the import is marked shared with a maximum. That would revive the binary patch + [`wasm-design.md`](wasm-design.md) calls obsolete. +- Kotlin calls CPython on the JS thread that loaded the module (`noInitialRun`, no `main`), + so `-sPROXY_TO_PTHREAD` does not apply there; a first `Thread.start()` from Kotlin relies on + the pre-created worker pool. +- Upcalls from a non-main Python thread: the Kotlin trampolines are placed with `table.set` + only in the main thread's table; each Worker has its own table, so every Worker needs the + same entries (SPEC U-5, the pythonx-concurrent wasm dispatchers). +- Staging this archive as a wasmJs runtime (it lacks `wasmExports,wasmMemory` in + `-sEXPORTED_RUNTIME_METHODS` and the `pyemscripten` platform tag that `build-cpython.sh` + adds). diff --git a/docs/platforms/wasm-design.md b/docs/platforms/wasm-design.md index d0d38560..cd1eb372 100644 --- a/docs/platforms/wasm-design.md +++ b/docs/platforms/wasm-design.md @@ -125,6 +125,10 @@ Details and the postcondition that guards the entry-module ordering are in the w - Whether compiled wheels other than pydantic-core load. - Compose (`@Composable` from Python) on wasm is `planned` (SPEC B-6, N-2); `suspend` upcalls are `planned` on wasm because there are no threads (SPEC U-5). +- Threads (#175): a free-threaded CPython 3.15t build with pthreads is a CI experiment + (`.github/workflows/build-ft-wasm.yml`), separate from the build above and not used by the library. + What it checks, and the Kotlin-side work it does not do (shared memory import, worker pool, + per-Worker function tables), is in [`free-threaded-wasm.md`](free-threaded-wasm.md). - Absolute wasmJs nanoseconds depend on suite scope and host lifetime (a filtered run is 55–74% slower in pure-Python rows); quote ratios, or full-suite figures labelled as such ([`downcall-design.md`](../design/downcall-design.md), "Warmup").