From 2091e31de998648aa897188666be5085cd930f1a Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:12:32 +0100 Subject: [PATCH 01/10] Add lazy-loading nanobind binding over the JPDFium safe API --- .gitattributes | 3 + .github/workflows/ci.yml | 99 +++++++++++ .gitignore | 9 + CMakeLists.txt | 54 ++++++ README.md | 53 ++++++ docs/ARCHITECTURE.md | 49 ++++++ docs/COMPATIBILITY.md | 17 ++ pyproject.toml | 40 +++++ src/native/pyjpdfium_native.cpp | 272 +++++++++++++++++++++++++++++++ src/pyjpdfium/__init__.py | 66 ++++++++ src/pyjpdfium/_loader.py | 111 +++++++++++++ src/pyjpdfium/errors.py | 124 ++++++++++++++ src/pyjpdfium/models.py | 82 ++++++++++ src/pyjpdfium/pdf.py | 229 ++++++++++++++++++++++++++ src/pyjpdfium/runtime.py | 42 +++++ tests/conftest.py | 50 ++++++ tests/native/test_concurrency.py | 86 ++++++++++ tests/native/test_documents.py | 101 ++++++++++++ tests/native/test_output.py | 54 ++++++ tests/native/test_render.py | 83 ++++++++++ tests/native/test_shutdown.py | 51 ++++++ tests/unit/test_errors.py | 45 +++++ tests/unit/test_lazy_import.py | 44 +++++ tests/unit/test_loader.py | 61 +++++++ 24 files changed, 1825 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/COMPATIBILITY.md create mode 100644 pyproject.toml create mode 100644 src/native/pyjpdfium_native.cpp create mode 100644 src/pyjpdfium/__init__.py create mode 100644 src/pyjpdfium/_loader.py create mode 100644 src/pyjpdfium/errors.py create mode 100644 src/pyjpdfium/models.py create mode 100644 src/pyjpdfium/pdf.py create mode 100644 src/pyjpdfium/runtime.py create mode 100644 tests/conftest.py create mode 100644 tests/native/test_concurrency.py create mode 100644 tests/native/test_documents.py create mode 100644 tests/native/test_output.py create mode 100644 tests/native/test_render.py create mode 100644 tests/native/test_shutdown.py create mode 100644 tests/unit/test_errors.py create mode 100644 tests/unit/test_lazy_import.py create mode 100644 tests/unit/test_loader.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4a39c88 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.pdf binary +*.png binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..98607c3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,99 @@ +name: CI + +on: + push: + branches: [main] + pull_request: {} + workflow_dispatch: + inputs: + jpdfium-ref: + description: JPDFium ref providing the safe jpdf_* core + default: feat/native-safe-domain + type: string + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Until the safe API lands on JPDFium main, build the core from its feature branch. + JPDFIUM_REF: ${{ github.event.inputs.jpdfium-ref || 'feat/native-safe-domain' }} + +jobs: + unit: + name: Unit (no natives) / ${{ matrix.os }} / py${{ matrix.python }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python: ['3.10', '3.13'] + steps: + - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python }} + - run: python -m pip install "pytest>=8" + - name: Unit tests + env: + PYTHONPATH: src + run: python -m pytest tests/unit -q + + native-linux: + name: Real native core / linux-x64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1 + - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1 + with: + repository: Stirling-Tools/JPDFium + ref: ${{ env.JPDFIUM_REF }} + path: jpdfium + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install native build deps + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build pkg-config \ + libpcre2-dev libfreetype6-dev libharfbuzz-dev libicu-dev \ + libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf + + - name: Fetch pinned PDFium and verify its published digest + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG=$(grep -v '^#' jpdfium/native/pdfium.version | grep -v '^$' | head -n1) + ASSET=pdfium-linux-x64.tar.gz + gh release download "$TAG" -R Stirling-Tools/JPDFium -p "$ASSET" -D "$RUNNER_TEMP" + DIGEST=$(gh api "repos/Stirling-Tools/JPDFium/releases/tags/$TAG" \ + --jq ".assets[] | select(.name == \"$ASSET\") | .digest") + echo "${DIGEST#sha256:} $RUNNER_TEMP/$ASSET" | sha256sum -c - + mkdir -p jpdfium/native/pdfium + tar -xzf "$RUNNER_TEMP/$ASSET" -C jpdfium/native/pdfium + + - name: Build the JPDFium core + run: bash jpdfium/native/build-real.sh + + - name: Native core tests + run: ctest --test-dir jpdfium/native/build-real --output-on-failure + + - name: Build pyjpdfium + env: + JPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + JPDFIUM_INCLUDE_DIR: ${{ github.workspace }}/jpdfium/native/bridge/include + run: | + python -m pip install "scikit-build-core>=0.11" "nanobind>=2.9" "pytest>=8" + python -m pip install --no-build-isolation . + + - name: Python tests (unit + native) + env: + PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + run: | + # Native tests skip when the extension is missing; fail loudly here instead. + python -c "import pyjpdfium; print(pyjpdfium.runtime_info())" + python -m pytest -q -rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc1d64b --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +build/ +dist/ +*.egg-info/ +__pycache__/ +.pytest_cache/ +*.so +*.pyd +*.dylib +src/pyjpdfium/_libs/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..1090a26 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,54 @@ +cmake_minimum_required(VERSION 3.20) +project(pyjpdfium LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# The extension links the one shared JPDFium core; it never compiles its own copy of the domain. +set(JPDFIUM_NATIVE_DIR "" CACHE PATH "Staged JPDFium natives (libjpdfium plus its runtime closure)") +set(JPDFIUM_INCLUDE_DIR "" CACHE PATH "Directory containing jpdfium_api.h") +option(PYJPDFIUM_BUNDLE_NATIVES "Copy the staged natives into the wheel under pyjpdfium/_libs" OFF) + +if(NOT JPDFIUM_NATIVE_DIR) + message(FATAL_ERROR "Set JPDFIUM_NATIVE_DIR to a staged JPDFium native directory") +endif() +if(NOT JPDFIUM_INCLUDE_DIR) + set(JPDFIUM_INCLUDE_DIR "${JPDFIUM_NATIVE_DIR}/include") +endif() +if(NOT EXISTS "${JPDFIUM_INCLUDE_DIR}/jpdfium_api.h") + message(FATAL_ERROR "jpdfium_api.h not found in ${JPDFIUM_INCLUDE_DIR}; set JPDFIUM_INCLUDE_DIR") +endif() + +find_library(JPDFIUM_LIBRARY NAMES jpdfium PATHS "${JPDFIUM_NATIVE_DIR}" NO_DEFAULT_PATH REQUIRED) +message(STATUS "JPDFium core: ${JPDFIUM_LIBRARY}") + +find_package(Python 3.10 COMPONENTS Interpreter Development.Module REQUIRED) +execute_process( + COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir + OUTPUT_STRIP_TRAILING_WHITESPACE + OUTPUT_VARIABLE nanobind_ROOT) +find_package(nanobind CONFIG REQUIRED) + +nanobind_add_module(_native NB_STATIC src/native/pyjpdfium_native.cpp) +target_include_directories(_native PRIVATE "${JPDFIUM_INCLUDE_DIR}") +target_link_libraries(_native PRIVATE "${JPDFIUM_LIBRARY}") +if(MSVC) + target_compile_options(_native PRIVATE /W4 /WX) +else() + target_compile_options(_native PRIVATE -Wall -Wextra -Werror) +endif() + +# Bundled wheels find the core next to the package; dev builds preload it from JPDFIUM_NATIVE_DIR. +if(APPLE) + set_target_properties(_native PROPERTIES INSTALL_RPATH "@loader_path/_libs") +elseif(UNIX) + set_target_properties(_native PROPERTIES INSTALL_RPATH "$ORIGIN/_libs") +endif() + +install(TARGETS _native LIBRARY DESTINATION pyjpdfium) + +if(PYJPDFIUM_BUNDLE_NATIVES) + install(DIRECTORY "${JPDFIUM_NATIVE_DIR}/" DESTINATION pyjpdfium/_libs + FILES_MATCHING PATTERN "*.so*" PATTERN "*.dylib" PATTERN "*.dll" + PATTERN "native-libs.*") +endif() diff --git a/README.md b/README.md new file mode 100644 index 0000000..8e290f1 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# PyJPDFium + +Python bindings to [JPDFium](https://github.com/Stirling-Tools/JPDFium)'s native PDF core +(PDFium). No JVM required. + +PyJPDFium is a thin [nanobind](https://github.com/wjakob/nanobind) binding over JPDFium's safe +`jpdf_*` C API. The native core, not Python or the GIL, owns PDFium serialization, object +lifetimes and handle validation, so the same rules hold for the Java and Python bindings. + +Status: early development. This first pass covers documents, pages, rendering and transactional +PNG output. + +```python +import pyjpdfium + +with pyjpdfium.open_document("input.pdf") as document: + with document.page(0) as page: + page.render("page.png", dpi=(300, 300)) # atomic publish; failure leaves page.png as-is + with page.render(dpi=150, pixel_format="gray") as bitmap: + pixels = bitmap.view() # zero-copy, read-only, shape (h, w) +``` + +## Behaviour + +- `import pyjpdfium` loads no native library. The core loads and starts on first use. +- Objects are safe to share across threads. All PDFium work is serialized natively, and the GIL + is released during native calls, so non-PDFium Python work keeps running. +- A closed object's handle can never be reused: use after close raises `StaleHandleError`. +- `close()` is idempotent. Closing a document with open pages raises `ResourceBusyError` and + leaves the document usable. +- Bitmaps are independent of their page and document, and a `view()` keeps its pixels alive even + after the bitmap is closed. +- One process uses one native core. Mixing it with JPDFium's legacy Java bridge in the same + process is refused. + +## Building + +PyJPDFium links a staged JPDFium native build; it never compiles its own copy of the core. + +```bash +export JPDFIUM_NATIVE_DIR=/path/to/JPDFium/native/dist/linux-x64 +export JPDFIUM_INCLUDE_DIR=/path/to/JPDFium/native/bridge/include +pip install --no-build-isolation . +PYJPDFIUM_NATIVE_DIR=$JPDFIUM_NATIVE_DIR pytest +``` + +`PYJPDFIUM_NATIVE_DIR` points the runtime at the staged natives. Wheels built with +`PYJPDFIUM_BUNDLE_NATIVES=ON` carry them in `pyjpdfium/_libs` instead. +`PYJPDFIUM_RENDERER` selects `auto` (default), `agg` or `skia`. + +Native tests (`-m native`) need the real core and are skipped when the extension is not built. + +See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..ba2f05f --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,49 @@ +# Architecture + +PyJPDFium is a peer of JPDFium's Java binding. Both bind one safe native C API, +`jpdfium_api.h`, implemented once in JPDFium's `libjpdfium` core. The core, not Python, owns: + +- PDFium serialization: one native domain per process; the GIL is never the lock. +- Object lifetime: generation-checked handles, parent retention, `BUSY` instead of partial close. +- Output safety: staged, bounded, atomically published files. + +The full contracts live in JPDFium's `docs/design/` (`ARCHITECTURE.md`, `NATIVE_ABI.md`, +`LIFETIME.md`, `CALLBACKS.md`, `CAPABILITIES.md`). + +## Layers + +```text +pyjpdfium pure Python API: Document, Page, Bitmap, runtime_* +pyjpdfium._loader lazy load, core pinning, identity checks, runtime init +pyjpdfium._native nanobind module: one function per jpdf_* call, GIL released around it +libjpdfium the shared native core (never compiled into the extension) +``` + +`import pyjpdfium` and its pure modules load nothing native. The first call that needs the core +runs `_loader.native()`, which: + +1. Loads the core from `PYJPDFIUM_NATIVE_DIR` or the bundled `pyjpdfium/_libs`, by exact path, so + the extension binds to that copy. +2. Checks the core's API version and struct layouts against the compiled header, and refuses to + continue if more than one `libjpdfium` is mapped into the process. +3. Calls `jpdf_runtime_init` (explicit lazy initialization). + +## Errors + +Native status codes map one-to-one onto the classes in `pyjpdfium.errors`. Unknown codes become +`InternalError`, never success. Errors carry `status` and `status_name`, and their message is the +native detail captured on the failing thread before the GIL is reacquired. + +## Lifetime in Python + +`close()` is idempotent and thread-safe. Context managers close on exit. Objects that are never +closed are reclaimed by `weakref.finalize`; because a page holds its document, a page is always +finalized before its document. Native code never cascades a close, and Python never forces one. + +`Bitmap.view()` returns a read-only memoryview whose owner holds a native view. The view pins the +pixels, so it stays valid after the bitmap, page or document is closed. + +## Isolation + +The native core runs in-process. It removes the JVM, not the crash domain: a native fault ends the +interpreter. Untrusted or hard-budget jobs should run in a worker process. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md new file mode 100644 index 0000000..f547cdc --- /dev/null +++ b/docs/COMPATIBILITY.md @@ -0,0 +1,17 @@ +# Compatibility + +| Component | Requirement | +|---|---| +| Python | 3.10 or newer, CPython | +| JPDFium core | Safe API 1.x, minor at least the one PyJPDFium was built against | +| Platforms verified | linux-x64 (glibc) | +| Platforms pending | linux-arm64, macOS, Windows: the build supports them; no CI coverage yet | + +On Windows the build also needs `jpdfium.lib`, the import library produced next to `jpdfium.dll`. + +## Not in this release + +- OCR, text-layer writing and the OCRmyPDF plugin. +- PDF/A output: rejected by design. +- Opening from bytes or file-like objects, saving or editing documents. +- Free-threaded CPython builds are untested. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c6d2a3e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["scikit-build-core>=0.11", "nanobind>=2.9"] +build-backend = "scikit_build_core.build" + +[project] +name = "pyjpdfium" +version = "0.1.0.dev0" +description = "Python bindings to JPDFium's native PDF core. No JVM required." +readme = "README.md" +license = "MIT" +license-files = ["LICENSE"] +requires-python = ">=3.10" +authors = [{ name = "Stirling Tools" }] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: C++", + "Topic :: Multimedia :: Graphics :: Graphics Conversion", +] +dependencies = [] + +[project.optional-dependencies] +test = ["pytest>=8"] + +[project.urls] +Source = "https://github.com/Stirling-Tools/PyJPDFium" + +[tool.scikit-build] +minimum-version = "0.11" +build-dir = "build/{wheel_tag}" +wheel.packages = ["src/pyjpdfium"] +cmake.build-type = "Release" + +[tool.scikit-build.cmake.define] +JPDFIUM_NATIVE_DIR = { env = "JPDFIUM_NATIVE_DIR", default = "" } +JPDFIUM_INCLUDE_DIR = { env = "JPDFIUM_INCLUDE_DIR", default = "" } +PYJPDFIUM_BUNDLE_NATIVES = { env = "PYJPDFIUM_BUNDLE_NATIVES", default = "OFF" } + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = ["native: needs the real JPDFium native core"] diff --git a/src/native/pyjpdfium_native.cpp b/src/native/pyjpdfium_native.cpp new file mode 100644 index 0000000..9fd3bb4 --- /dev/null +++ b/src/native/pyjpdfium_native.cpp @@ -0,0 +1,272 @@ +// pyjpdfium_native.cpp - nanobind binding over JPDFium's safe jpdf_* C API. +// Every native call runs with the GIL released; errors are raised after it is reacquired. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "jpdfium_api.h" + +namespace nb = nanobind; + +namespace { + +// Must run on the failing thread before any other jpdf_* call; detail is per thread and call. +std::string last_error_detail() { + uint64_t needed = 0; + jpdf_last_error_message(nullptr, 0, &needed); + if (needed <= 1) return {}; + std::string detail(static_cast(needed), '\0'); + if (jpdf_last_error_message(detail.data(), needed, &needed) != JPDF_OK) return {}; + detail.pop_back(); + return detail; +} + +[[noreturn]] void raise_status(jpdf_status status, const std::string& detail) { + nb::object errors = nb::module_::import_("pyjpdfium.errors"); + nb::object exc = errors.attr("_from_status")(status, jpdf_status_name(status), detail); + PyErr_SetObject(reinterpret_cast(Py_TYPE(exc.ptr())), exc.ptr()); + throw nb::python_error(); +} + +template +void run(Call&& call) { + jpdf_status status; + std::string detail; + { + nb::gil_scoped_release release; + status = call(); + if (status != JPDF_OK) detail = last_error_detail(); + } + if (status != JPDF_OK) raise_status(status, detail); +} + +jpdf_render_options render_options(uint32_t flags, double dpi_x, double dpi_y, int32_t width, + int32_t height, int32_t pixel_format, uint64_t max_pixels) { + jpdf_render_options o{}; + o.struct_size = sizeof(o); + o.flags = flags; + o.dpi_x = dpi_x; + o.dpi_y = dpi_y; + o.width = width; + o.height = height; + o.pixel_format = pixel_format; + o.max_pixels = max_pixels; + return o; +} + +jpdf_write_options write_options(bool durable, int32_t png_compression, uint64_t max_bytes) { + jpdf_write_options w{}; + w.struct_size = sizeof(w); + w.flags = durable ? JPDF_WRITE_DURABLE : 0u; + w.image_format = JPDF_IMAGE_PNG; + w.png_compression = png_compression; + w.max_bytes = max_bytes; + return w; +} + +bool struct_sizes_match() { + const int64_t expected[][2] = { + {JPDF_STRUCT_RUNTIME_OPTIONS, sizeof(jpdf_runtime_options)}, + {JPDF_STRUCT_RUNTIME_INFO, sizeof(jpdf_runtime_info)}, + {JPDF_STRUCT_OPEN_OPTIONS, sizeof(jpdf_open_options)}, + {JPDF_STRUCT_PAGE_INFO, sizeof(jpdf_page_info)}, + {JPDF_STRUCT_RENDER_OPTIONS, sizeof(jpdf_render_options)}, + {JPDF_STRUCT_BITMAP_INFO, sizeof(jpdf_bitmap_info)}, + {JPDF_STRUCT_BITMAP_VIEW, sizeof(jpdf_bitmap_view)}, + {JPDF_STRUCT_WRITE_OPTIONS, sizeof(jpdf_write_options)}, + }; + for (const auto& pair : expected) { + if (jpdf_abi_struct_size(static_cast(pair[0])) != pair[1]) return false; + } + return true; +} + +nb::dict runtime_info() { + jpdf_runtime_info info{}; + info.struct_size = sizeof(info); + run([&] { return jpdf_runtime_get_info(&info); }); + nb::dict d; + d["state"] = info.state; + d["api_version"] = nb::make_tuple(info.api_major, info.api_minor); + d["renderer"] = info.renderer; + d["build_id"] = info.build_id ? std::string(info.build_id) : std::string(); + d["live_documents"] = info.live_documents; + d["live_pages"] = info.live_pages; + d["live_bitmaps"] = info.live_bitmaps; + d["live_views"] = info.live_views; + d["domain_acquisitions"] = info.domain_acquisitions; + d["domain_wait_ns"] = info.domain_wait_ns; + d["domain_hold_ns"] = info.domain_hold_ns; + return d; +} + +nb::bytes bitmap_copy(jpdf_handle bitmap) { + jpdf_bitmap_info info{}; + info.struct_size = sizeof(info); + run([&] { return jpdf_bitmap_get_info(bitmap, &info); }); + PyObject* raw = PyBytes_FromStringAndSize(nullptr, static_cast(info.byte_size)); + if (!raw) throw nb::python_error(); + nb::bytes out = nb::steal(raw); + char* dst = PyBytes_AS_STRING(raw); + uint64_t written = 0; + // The new bytes object is not shared yet, so filling it without the GIL is safe. + run([&] { + return jpdf_bitmap_copy_pixels(bitmap, reinterpret_cast(dst), info.byte_size, + &written); + }); + return out; +} + +using PixelArray = nb::ndarray; + +size_t channels_for(int32_t pixel_format) { + if (pixel_format == JPDF_PIXEL_RGBA8) return 4; + if (pixel_format == JPDF_PIXEL_RGB8) return 3; + return 1; +} + +// The capsule owns the native view, so the memoryview keeps the pixels pinned on its own. +PixelArray bitmap_view(jpdf_handle bitmap) { + jpdf_bitmap_view view{}; + view.struct_size = sizeof(view); + run([&] { return jpdf_bitmap_map(bitmap, &view); }); + auto* token = new jpdf_handle(view.view); + nb::capsule owner(token, [](void* p) noexcept { + auto* h = static_cast(p); + jpdf_view_release(*h); + delete h; + }); + const size_t channels = channels_for(view.pixel_format); + const size_t shape[3] = {static_cast(view.height), static_cast(view.width), + channels}; + const int64_t strides[3] = {view.stride, static_cast(channels), 1}; + return PixelArray(view.data, channels == 1 ? 2 : 3, shape, owner, strides); +} + +} // namespace + +NB_MODULE(_native, m) { + m.doc() = "Low-level binding over jpdfium_api.h. Use the pyjpdfium package instead."; + + m.attr("COMPILED_API_VERSION") = (JPDF_API_VERSION_MAJOR << 16) | JPDF_API_VERSION_MINOR; + nb::dict status; + for (jpdf_status s : {JPDF_OK, + JPDF_E_INVALID_ARGUMENT, + JPDF_E_INVALID_HANDLE, + JPDF_E_STALE_HANDLE, + JPDF_E_WRONG_HANDLE_TYPE, + JPDF_E_BUSY, + JPDF_E_UNSUPPORTED, + JPDF_E_IO, + JPDF_E_PASSWORD, + JPDF_E_FORMAT, + JPDF_E_NOT_FOUND, + JPDF_E_LIMIT, + JPDF_E_NO_MEMORY, + JPDF_E_BUFFER_TOO_SMALL, + JPDF_E_NOT_INITIALIZED, + JPDF_E_QUIESCING, + JPDF_E_STOPPED, + JPDF_E_FAILED, + JPDF_E_AUTHORITY, + JPDF_E_REENTRANT, + JPDF_E_CANCELLED, + JPDF_E_TIMEOUT, + JPDF_E_INTERNAL}) { + status[jpdf_status_name(s)] = s; + } + m.attr("STATUS") = status; + + m.def("api_version", [] { return jpdf_api_version(); }); + m.def("struct_sizes_match", &struct_sizes_match); + m.def("capability", [](const std::string& name) { + int32_t level = 0; + run([&] { return jpdf_capability(name.c_str(), &level); }); + return level; + }); + + m.def("runtime_init", [](int32_t renderer) { + jpdf_runtime_options o{}; + o.struct_size = sizeof(o); + o.renderer = renderer; + run([&] { return jpdf_runtime_init(&o); }); + }); + m.def("runtime_info", &runtime_info); + m.def("runtime_quiesce", [] { run([] { return jpdf_runtime_quiesce(); }); }); + m.def("runtime_shutdown", [] { run([] { return jpdf_runtime_shutdown(); }); }); + + m.def( + "doc_open", + [](const std::string& path, std::optional password) { + jpdf_open_options o{}; + o.struct_size = sizeof(o); + o.password = password ? password->c_str() : nullptr; + jpdf_handle doc = 0; + run([&] { return jpdf_doc_open_file(path.c_str(), &o, &doc); }); + return doc; + }, + nb::arg("path"), nb::arg("password").none()); + m.def("doc_close", [](jpdf_handle doc) { run([&] { return jpdf_doc_close(doc); }); }); + m.def("doc_page_count", [](jpdf_handle doc) { + int32_t count = 0; + run([&] { return jpdf_doc_page_count(doc, &count); }); + return count; + }); + + m.def("page_open", [](jpdf_handle doc, int32_t index) { + jpdf_handle page = 0; + run([&] { return jpdf_page_open(doc, index, &page); }); + return page; + }); + m.def("page_close", [](jpdf_handle page) { run([&] { return jpdf_page_close(page); }); }); + m.def("page_info", [](jpdf_handle page) { + jpdf_page_info info{}; + info.struct_size = sizeof(info); + run([&] { return jpdf_page_get_info(page, &info); }); + return std::make_tuple(info.index, info.width_pt, info.height_pt, info.rotation); + }); + + m.def("page_render", [](jpdf_handle page, uint32_t flags, double dpi_x, double dpi_y, + int32_t width, int32_t height, int32_t pixel_format, + uint64_t max_pixels) { + jpdf_render_options o = + render_options(flags, dpi_x, dpi_y, width, height, pixel_format, max_pixels); + jpdf_handle bitmap = 0; + run([&] { return jpdf_page_render(page, &o, &bitmap); }); + return bitmap; + }); + m.def("page_render_to_file", + [](jpdf_handle page, uint32_t flags, double dpi_x, double dpi_y, int32_t width, + int32_t height, int32_t pixel_format, uint64_t max_pixels, const std::string& path, + bool durable, int32_t png_compression, uint64_t max_bytes) { + jpdf_render_options o = + render_options(flags, dpi_x, dpi_y, width, height, pixel_format, max_pixels); + jpdf_write_options w = write_options(durable, png_compression, max_bytes); + run([&] { return jpdf_page_render_to_file(page, &o, path.c_str(), &w); }); + }); + + m.def("bitmap_info", [](jpdf_handle bitmap) { + jpdf_bitmap_info info{}; + info.struct_size = sizeof(info); + run([&] { return jpdf_bitmap_get_info(bitmap, &info); }); + return std::make_tuple(info.pixel_format, info.width, info.height, info.stride, + info.dpi_x, info.dpi_y, info.byte_size); + }); + m.def("bitmap_copy", &bitmap_copy); + m.def("bitmap_view", &bitmap_view); + m.def("bitmap_write_file", [](jpdf_handle bitmap, const std::string& path, bool durable, + int32_t png_compression, uint64_t max_bytes) { + jpdf_write_options w = write_options(durable, png_compression, max_bytes); + run([&] { return jpdf_bitmap_write_file(bitmap, path.c_str(), &w); }); + }); + m.def("bitmap_close", + [](jpdf_handle bitmap) { run([&] { return jpdf_bitmap_close(bitmap); }); }); +} diff --git a/src/pyjpdfium/__init__.py b/src/pyjpdfium/__init__.py new file mode 100644 index 0000000..becbf6a --- /dev/null +++ b/src/pyjpdfium/__init__.py @@ -0,0 +1,66 @@ +"""Python bindings to JPDFium's native PDF core. No JVM required. + +Importing this package loads no native library. The core loads on first use, for example +``open_document``. +""" + +from __future__ import annotations + +import importlib +from typing import Any + +__version__ = "0.1.0.dev0" + +_EXPORTS = { + "open_document": "pyjpdfium.pdf", + "Document": "pyjpdfium.pdf", + "Page": "pyjpdfium.pdf", + "Bitmap": "pyjpdfium.pdf", + "runtime_info": "pyjpdfium.runtime", + "capability": "pyjpdfium.runtime", + "quiesce": "pyjpdfium.runtime", + "shutdown": "pyjpdfium.runtime", + "PixelFormat": "pyjpdfium.models", + "RenderFlags": "pyjpdfium.models", + "Renderer": "pyjpdfium.models", + "RuntimeState": "pyjpdfium.models", + "Capability": "pyjpdfium.models", + "PageSize": "pyjpdfium.models", + "BitmapInfo": "pyjpdfium.models", + "RuntimeInfo": "pyjpdfium.models", + "PyJPDFiumError": "pyjpdfium.errors", + "NativeLoadError": "pyjpdfium.errors", + "InvalidArgumentError": "pyjpdfium.errors", + "HandleError": "pyjpdfium.errors", + "InvalidHandleError": "pyjpdfium.errors", + "StaleHandleError": "pyjpdfium.errors", + "WrongHandleTypeError": "pyjpdfium.errors", + "ResourceBusyError": "pyjpdfium.errors", + "UnsupportedError": "pyjpdfium.errors", + "PdfIOError": "pyjpdfium.errors", + "PasswordError": "pyjpdfium.errors", + "PdfFormatError": "pyjpdfium.errors", + "PageNotFoundError": "pyjpdfium.errors", + "LimitExceededError": "pyjpdfium.errors", + "NativeMemoryError": "pyjpdfium.errors", + "RuntimeStateError": "pyjpdfium.errors", + "ReentrancyError": "pyjpdfium.errors", + "OperationCancelledError": "pyjpdfium.errors", + "OperationTimeoutError": "pyjpdfium.errors", + "InternalError": "pyjpdfium.errors", +} + +__all__ = sorted(_EXPORTS) + + +def __getattr__(name: str) -> Any: + module = _EXPORTS.get(name) + if module is None: + raise AttributeError(f"module 'pyjpdfium' has no attribute {name!r}") + value = getattr(importlib.import_module(module), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(_EXPORTS)) diff --git a/src/pyjpdfium/_loader.py b/src/pyjpdfium/_loader.py new file mode 100644 index 0000000..2b0ea7b --- /dev/null +++ b/src/pyjpdfium/_loader.py @@ -0,0 +1,111 @@ +"""Explicit lazy loading of the native core. Nothing here runs at package import.""" + +from __future__ import annotations + +import ctypes +import importlib +import os +import sys +import threading +from pathlib import Path +from types import ModuleType + +from .errors import NativeLoadError +from .models import Renderer + +NATIVE_DIR_ENV = "PYJPDFIUM_NATIVE_DIR" +RENDERER_ENV = "PYJPDFIUM_RENDERER" + +_lock = threading.Lock() +_module: ModuleType | None = None +# Keeps Windows DLL directory registrations and preloaded cores alive for the process. +_pins: list[object] = [] + + +def native() -> ModuleType: + """Returns the initialized extension, loading and starting the core on first call.""" + module = _module + if module is not None: + return module + with _lock: + if _module is None: + _load() + assert _module is not None + return _module + + +def native_dir() -> Path | None: + configured = os.environ.get(NATIVE_DIR_ENV) + if configured: + return Path(configured) + bundled = Path(__file__).with_name("_libs") + return bundled if bundled.is_dir() else None + + +def _core_name() -> str: + if sys.platform == "win32": + return "jpdfium.dll" + if sys.platform == "darwin": + return "libjpdfium.dylib" + return "libjpdfium.so" + + +def _pin_core(directory: Path) -> None: + core = directory / _core_name() + if not core.is_file(): + raise NativeLoadError(f"{core} not found; set {NATIVE_DIR_ENV} to staged JPDFium natives") + if sys.platform == "win32": + _pins.append(os.add_dll_directory(str(directory))) + return + # Loading this exact file first makes the extension bind to it rather than another copy. + _pins.append(ctypes.CDLL(str(core), mode=ctypes.RTLD_GLOBAL)) + + +def _loaded_core_paths() -> set[str]: + maps = Path("/proc/self/maps") + if not maps.exists(): + return set() + paths = set() + for line in maps.read_text().splitlines(): + parts = line.split(maxsplit=5) + if len(parts) == 6 and Path(parts[5]).name.startswith("libjpdfium.so"): + paths.add(os.path.realpath(parts[5])) + return paths + + +def _check_identity(module: ModuleType) -> None: + library = module.api_version() + compiled = module.COMPILED_API_VERSION + if library >> 16 != compiled >> 16 or (library & 0xFFFF) < (compiled & 0xFFFF): + raise NativeLoadError( + f"JPDFium core API {library >> 16}.{library & 0xFFFF} is incompatible with " + f"this build of pyjpdfium ({compiled >> 16}.{compiled & 0xFFFF})" + ) + if not module.struct_sizes_match(): + raise NativeLoadError("JPDFium core struct layouts differ from the compiled header") + cores = _loaded_core_paths() + if len(cores) > 1: + # Two cores means two PDFium domains, each believing it is the only serializer. + raise NativeLoadError(f"multiple JPDFium cores loaded: {sorted(cores)}") + + +def _renderer() -> Renderer: + value = os.environ.get(RENDERER_ENV, "auto").strip().upper() + try: + return Renderer[value] + except KeyError: + raise NativeLoadError(f"{RENDERER_ENV} must be auto, agg or skia, not {value!r}") from None + + +def _load() -> None: + global _module + directory = native_dir() + if directory is not None: + _pin_core(directory) + try: + module = importlib.import_module("pyjpdfium._native") + except ImportError as exc: + raise NativeLoadError(f"could not load the JPDFium native core: {exc}") from exc + _check_identity(module) + module.runtime_init(int(_renderer())) + _module = module diff --git a/src/pyjpdfium/errors.py b/src/pyjpdfium/errors.py new file mode 100644 index 0000000..3c2048a --- /dev/null +++ b/src/pyjpdfium/errors.py @@ -0,0 +1,124 @@ +"""Exceptions mapped from JPDFium native status codes. Pure Python; loads nothing native.""" + +from __future__ import annotations + + +class PyJPDFiumError(Exception): + """Base class. ``status`` is the native code, or None for errors raised in Python.""" + + def __init__( + self, message: str, *, status: int | None = None, status_name: str | None = None + ) -> None: + super().__init__(message) + self.status = status + self.status_name = status_name + + +class NativeLoadError(PyJPDFiumError): + """The native core is missing, incompatible or loaded more than once.""" + + +class InvalidArgumentError(PyJPDFiumError, ValueError): + pass + + +class HandleError(PyJPDFiumError): + """A native handle was rejected before it was dereferenced.""" + + +class InvalidHandleError(HandleError): + pass + + +class StaleHandleError(HandleError): + """The object was closed; its handle can never become valid again.""" + + +class WrongHandleTypeError(HandleError, TypeError): + pass + + +class ResourceBusyError(PyJPDFiumError): + """Close or shutdown refused while dependents remain; nothing was changed.""" + + +class UnsupportedError(PyJPDFiumError): + pass + + +class PdfIOError(PyJPDFiumError, OSError): + pass + + +class PasswordError(PyJPDFiumError): + pass + + +class PdfFormatError(PyJPDFiumError): + pass + + +class PageNotFoundError(PyJPDFiumError, IndexError): + pass + + +class LimitExceededError(PyJPDFiumError): + pass + + +class NativeMemoryError(PyJPDFiumError, MemoryError): + pass + + +class RuntimeStateError(PyJPDFiumError): + """The runtime is not initialized, is quiescing, stopped, failed, or owned elsewhere.""" + + +class ReentrancyError(PyJPDFiumError): + pass + + +class OperationCancelledError(PyJPDFiumError): + pass + + +class OperationTimeoutError(PyJPDFiumError, TimeoutError): + pass + + +class InternalError(PyJPDFiumError): + pass + + +# Mirrors jpdfium_api.h; tests check it against the codes compiled into the extension. +STATUS_ERRORS: dict[int, type[PyJPDFiumError]] = { + -1: InvalidArgumentError, + -2: InvalidHandleError, + -3: StaleHandleError, + -4: WrongHandleTypeError, + -5: ResourceBusyError, + -6: UnsupportedError, + -7: PdfIOError, + -8: PasswordError, + -9: PdfFormatError, + -10: PageNotFoundError, + -11: LimitExceededError, + -12: NativeMemoryError, + -13: InternalError, + -14: RuntimeStateError, + -15: RuntimeStateError, + -16: RuntimeStateError, + -17: RuntimeStateError, + -18: RuntimeStateError, + -19: ReentrancyError, + -20: OperationCancelledError, + -21: OperationTimeoutError, + -99: InternalError, +} + + +def _from_status(status: int, name: str, detail: str) -> PyJPDFiumError: + # Unknown codes are never success: they surface as internal errors. + cls = STATUS_ERRORS.get(status, InternalError) + message = f"{detail} ({name})" if detail else name + return cls(message, status=status, status_name=name) diff --git a/src/pyjpdfium/models.py b/src/pyjpdfium/models.py new file mode 100644 index 0000000..41d6745 --- /dev/null +++ b/src/pyjpdfium/models.py @@ -0,0 +1,82 @@ +"""Value types and enums. Pure Python; loads nothing native.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, IntFlag + + +class Renderer(IntEnum): + AUTO = -1 + AGG = 0 + SKIA = 1 + + +class RuntimeState(IntEnum): + UNINITIALIZED = 0 + RUNNING = 1 + QUIESCING = 2 + DESTROYING = 3 + STOPPED = 4 + FAILED = 5 + + +class Capability(IntEnum): + ABSENT = 0 + IMPLEMENTED = 1 + QUALIFIED = 2 + + +class PixelFormat(IntEnum): + RGBA = 1 + RGB = 2 + GRAY = 3 + + @property + def channels(self) -> int: + return {PixelFormat.RGBA: 4, PixelFormat.RGB: 3, PixelFormat.GRAY: 1}[self] + + +class RenderFlags(IntFlag): + NONE = 0 + ANNOTATIONS = 0x0001 + LCD_TEXT = 0x0002 + GRAYSCALE = 0x0004 + PRINTING = 0x0008 + NO_SMOOTH_TEXT = 0x0010 + NO_SMOOTH_IMAGE = 0x0020 + NO_SMOOTH_PATH = 0x0040 + TRANSPARENT = 0x0100 + + +@dataclass(frozen=True) +class PageSize: + """Displayed page size in points, after /Rotate.""" + + width: float + height: float + + +@dataclass(frozen=True) +class BitmapInfo: + width: int + height: int + stride: int + pixel_format: PixelFormat + dpi: tuple[float, float] + byte_size: int + + +@dataclass(frozen=True) +class RuntimeInfo: + state: RuntimeState + api_version: tuple[int, int] + renderer: Renderer | None + build_id: str + live_documents: int + live_pages: int + live_bitmaps: int + live_views: int + domain_acquisitions: int + domain_wait_ns: int + domain_hold_ns: int diff --git a/src/pyjpdfium/pdf.py b/src/pyjpdfium/pdf.py new file mode 100644 index 0000000..3457d58 --- /dev/null +++ b/src/pyjpdfium/pdf.py @@ -0,0 +1,229 @@ +"""Documents, pages and rendered bitmaps over the native core's generation-checked handles.""" + +from __future__ import annotations + +import os +import threading +import weakref +from collections.abc import Callable +from typing import Union + +from ._loader import native +from .errors import InvalidArgumentError, PyJPDFiumError, StaleHandleError +from .models import BitmapInfo, PageSize, PixelFormat, RenderFlags + +StrPath = Union[str, "os.PathLike[str]"] +Dpi = Union[float, "tuple[float, float]"] + +# 0 asks the native core for its default pixel budget, and for no byte limit on output. +_LIBRARY_BUDGET = 0 +_NO_LIMIT = 0 + + +def _fspath(path: StrPath) -> str: + return os.fsdecode(os.fspath(path)) + + +def _close_quietly(close: Callable[[int], None], token: int) -> None: + # Finalizers only reclaim what was never closed; they must not raise. + try: + close(token) + except PyJPDFiumError: + pass + + +class _Handle: + """Idempotent close over one native token; finalization covers objects never closed.""" + + def __init__(self, token: int, close: Callable[[int], None]) -> None: + self._token = token + self._finalizer = weakref.finalize(self, _close_quietly, close, token) + self._close = close + self._close_lock = threading.Lock() + + @property + def closed(self) -> bool: + return not self._finalizer.alive + + def _require_open(self) -> int: + if self.closed: + raise StaleHandleError(f"{type(self).__name__} is closed") + return self._token + + def close(self) -> None: + with self._close_lock: + if self.closed: + return + # BUSY propagates and leaves this object open, matching the native contract. + self._close(self._token) + self._finalizer.detach() + + def __enter__(self): + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +def open_document(path: StrPath, *, password: str | None = None) -> Document: + lib = native() + return Document(lib.doc_open(_fspath(path), password), _fspath(path)) + + +class Document(_Handle): + def __init__(self, token: int, path: str) -> None: + super().__init__(token, native().doc_close) + self.path = path + + @property + def page_count(self) -> int: + return native().doc_page_count(self._require_open()) + + def page(self, index: int) -> Page: + token = native().page_open(self._require_open(), index) + return Page(token, self) + + def close(self) -> None: + """Raises ResourceBusyError while pages are open; the document stays usable.""" + super().close() + + def __repr__(self) -> str: + state = "closed" if self.closed else "open" + return f"" + + +def _dpi_pair(dpi: Dpi) -> tuple[float, float]: + if isinstance(dpi, (int, float)): + return float(dpi), float(dpi) + x, y = dpi + return float(x), float(y) + + +def _render_args( + dpi: Dpi, + size: tuple[int, int] | None, + pixel_format: PixelFormat | str, + annotations: bool, + transparent: bool, + flags: RenderFlags, + max_pixels: int | None, +) -> tuple[int, float, float, int, int, int, int]: + fmt = PixelFormat[pixel_format.upper()] if isinstance(pixel_format, str) else pixel_format + all_flags = RenderFlags(flags) + if annotations: + all_flags |= RenderFlags.ANNOTATIONS + if transparent: + all_flags |= RenderFlags.TRANSPARENT + dpi_x, dpi_y = _dpi_pair(dpi) + width, height = size if size is not None else (0, 0) + if max_pixels is not None and max_pixels <= 0: + raise InvalidArgumentError("max_pixels must be positive") + return ( + int(all_flags), + dpi_x, + dpi_y, + width, + height, + int(fmt), + max_pixels or _LIBRARY_BUDGET, + ) + + +class Page(_Handle): + def __init__(self, token: int, document: Document) -> None: + super().__init__(token, native().page_close) + # A page keeps its document alive, mirroring the native parent retention. + self.document = document + index, width, height, rotation = native().page_info(token) + self.index = index + self.size = PageSize(width, height) + self.rotation = rotation + + def render( + self, + path: StrPath | None = None, + *, + dpi: Dpi = 72.0, + size: tuple[int, int] | None = None, + pixel_format: PixelFormat | str = PixelFormat.RGB, + annotations: bool = True, + transparent: bool = False, + flags: RenderFlags = RenderFlags.NONE, + max_pixels: int | None = None, + durable: bool = False, + png_compression: int | None = None, + max_bytes: int | None = None, + ) -> Bitmap | None: + """Renders to a Bitmap, or publishes a PNG at ``path`` and returns None. + + File output is transactional: on any failure ``path`` is left untouched. + """ + args = _render_args(dpi, size, pixel_format, annotations, transparent, flags, max_pixels) + token = self._require_open() + if path is None: + return Bitmap(native().page_render(token, *args)) + native().page_render_to_file( + token, + *args, + _fspath(path), + durable, + -1 if png_compression is None else png_compression, + max_bytes or _NO_LIMIT, + ) + return None + + def __repr__(self) -> str: + state = "closed" if self.closed else "open" + return f"" + + +class Bitmap(_Handle): + """Independently owned pixels: closing the page or document does not affect them.""" + + def __init__(self, token: int) -> None: + super().__init__(token, native().bitmap_close) + fmt, width, height, stride, dpi_x, dpi_y, byte_size = native().bitmap_info(token) + self.info = BitmapInfo(width, height, stride, PixelFormat(fmt), (dpi_x, dpi_y), byte_size) + + @property + def width(self) -> int: + return self.info.width + + @property + def height(self) -> int: + return self.info.height + + @property + def pixel_format(self) -> PixelFormat: + return self.info.pixel_format + + def tobytes(self) -> bytes: + return native().bitmap_copy(self._require_open()) + + def view(self) -> memoryview: + """Zero-copy read-only view, shape (height, width[, channels]). + + The view pins the pixels itself, so it stays valid after this bitmap is closed. + """ + return memoryview(native().bitmap_view(self._require_open())) + + def save( + self, + path: StrPath, + *, + durable: bool = False, + png_compression: int | None = None, + max_bytes: int | None = None, + ) -> None: + """Publishes a PNG transactionally; on failure ``path`` is left untouched.""" + native().bitmap_write_file( + self._require_open(), + _fspath(path), + durable, + -1 if png_compression is None else png_compression, + max_bytes or _NO_LIMIT, + ) + + def __repr__(self) -> str: + state = "closed" if self.closed else "open" + return f"" diff --git a/src/pyjpdfium/runtime.py b/src/pyjpdfium/runtime.py new file mode 100644 index 0000000..74e32ec --- /dev/null +++ b/src/pyjpdfium/runtime.py @@ -0,0 +1,42 @@ +"""Runtime identity, capabilities and explicit lifecycle control.""" + +from __future__ import annotations + +from ._loader import native +from .models import Capability, Renderer, RuntimeInfo, RuntimeState + + +def runtime_info() -> RuntimeInfo: + raw = native().runtime_info() + renderer = raw["renderer"] + return RuntimeInfo( + state=RuntimeState(raw["state"]), + api_version=tuple(raw["api_version"]), + renderer=Renderer(renderer) if renderer >= 0 else None, + build_id=raw["build_id"], + live_documents=raw["live_documents"], + live_pages=raw["live_pages"], + live_bitmaps=raw["live_bitmaps"], + live_views=raw["live_views"], + domain_acquisitions=raw["domain_acquisitions"], + domain_wait_ns=raw["domain_wait_ns"], + domain_hold_ns=raw["domain_hold_ns"], + ) + + +def capability(name: str) -> Capability: + """Readiness of a named operation such as ``"pdf.render"``.""" + return Capability(native().capability(name)) + + +def quiesce() -> None: + """Stops admitting new work; open pages and documents can still be closed.""" + native().runtime_quiesce() + + +def shutdown() -> None: + """Destroys PDFium once nothing is open. Final for this process. + + Raises ResourceBusyError, leaving the runtime quiescing, while documents or pages are open. + """ + native().runtime_shutdown() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3620da8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +PAGES = 4 + + +def fixture_pdf(pages: int = PAGES) -> bytes: + """Page i is 612x792pt with a black 100pt square at x = 72 + 100 * i, y = 600.""" + objects = ["<< /Type /Catalog /Pages 2 0 R >>"] + kids = " ".join(f"{3 + i * 2} 0 R" for i in range(pages)) + objects.append(f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>") + for i in range(pages): + content = f"0 0 0 rg {72 + i * 100} 600 100 100 re f" + objects.append( + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {4 + i * 2} 0 R >>" + ) + objects.append(f"<< /Length {len(content)} >>\nstream\n{content}\nendstream") + out = bytearray(b"%PDF-1.7\n") + offsets = [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n{body}\nendobj\n".encode() + xref = len(out) + out += f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + for offset in offsets: + out += f"{offset:010d} 00000 n \n".encode() + out += ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n" + ).encode() + return bytes(out) + + +@pytest.fixture +def pdf_path(tmp_path: Path) -> Path: + path = tmp_path / "fixture.pdf" + path.write_bytes(fixture_pdf()) + return path + + +def pytest_collection_modifyitems(config, items): + if importlib.util.find_spec("pyjpdfium._native") is not None: + return + skip = pytest.mark.skip(reason="pyjpdfium._native is not built") + for item in items: + if "native" in item.keywords: + item.add_marker(skip) diff --git a/tests/native/test_concurrency.py b/tests/native/test_concurrency.py new file mode 100644 index 0000000..5b07e4f --- /dev/null +++ b/tests/native/test_concurrency.py @@ -0,0 +1,86 @@ +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +import pyjpdfium +from pyjpdfium import errors + +pytestmark = pytest.mark.native + +THREADS = 8 + + +def render_bytes(page, dpi=36): + with page.render(dpi=dpi, pixel_format="gray") as bitmap: + return bitmap.tobytes() + + +def test_independent_documents_render_identically(pdf_path): + expected = {} + with pyjpdfium.open_document(pdf_path) as doc: + for i in range(4): + with doc.page(i) as page: + expected[i] = render_bytes(page) + + def work(n): + index = n % 4 + with pyjpdfium.open_document(pdf_path) as doc, doc.page(index) as page: + return index, render_bytes(page) + + with ThreadPoolExecutor(THREADS) as pool: + results = list(pool.map(work, range(200))) + assert all(data == expected[index] for index, data in results) + + +def test_one_shared_document_across_threads(pdf_path): + # The native domain owns serialization, so one document can be shared across threads. + with pyjpdfium.open_document(pdf_path) as doc: + pages = [doc.page(i) for i in range(4)] + expected = [render_bytes(p, 150) for p in pages] + before = pyjpdfium.runtime_info() + with ThreadPoolExecutor(THREADS) as pool: + results = list(pool.map(lambda n: (n % 4, render_bytes(pages[n % 4], 150)), range(64))) + after = pyjpdfium.runtime_info() + for p in pages: + p.close() + assert expected[0] != expected[1] + assert all(data == expected[index] for index, data in results) + # Threads only queue on the domain when the GIL is released around the native call. + assert after.domain_wait_ns - before.domain_wait_ns > 1_000_000 + + +def test_close_race_only_succeeds_or_reports_stale(pdf_path): + stale = 0 + for _ in range(50): + doc = pyjpdfium.open_document(pdf_path) + page = doc.page(0) + start = threading.Barrier(5) + outcomes = [] + + def renderer(): + start.wait() + for _ in range(5): + try: + page.render(dpi=18, pixel_format="gray").close() + outcomes.append("ok") + except errors.StaleHandleError: + outcomes.append("stale") + + def closer(): + start.wait() + page.close() + doc.close() + + threads = [threading.Thread(target=renderer) for _ in range(4)] + threads.append(threading.Thread(target=closer)) + for t in threads: + t.start() + for t in threads: + t.join() + assert set(outcomes) <= {"ok", "stale"} + assert len(outcomes) == 20 + stale += outcomes.count("stale") + assert stale > 0 + info = pyjpdfium.runtime_info() + assert info.live_pages == 0 diff --git a/tests/native/test_documents.py b/tests/native/test_documents.py new file mode 100644 index 0000000..3875a50 --- /dev/null +++ b/tests/native/test_documents.py @@ -0,0 +1,101 @@ +import gc + +import pytest + +import pyjpdfium +from pyjpdfium import errors + +pytestmark = pytest.mark.native + + +def test_open_count_and_page_geometry(pdf_path): + with pyjpdfium.open_document(pdf_path) as doc: + assert doc.page_count == 4 + with doc.page(1) as page: + assert page.index == 1 + assert page.size == pyjpdfium.PageSize(612.0, 792.0) + assert page.rotation == 0 + + +def test_close_is_idempotent_and_closed_objects_are_rejected(pdf_path): + doc = pyjpdfium.open_document(pdf_path) + page = doc.page(0) + page.close() + page.close() + assert page.closed + with pytest.raises(errors.StaleHandleError): + page.render() + doc.close() + doc.close() + with pytest.raises(errors.StaleHandleError): + doc.page_count # noqa: B018 + + +def test_parent_close_is_busy_without_partial_close(pdf_path): + doc = pyjpdfium.open_document(pdf_path) + page = doc.page(2) + with pytest.raises(errors.ResourceBusyError): + doc.close() + assert not doc.closed + assert doc.page_count == 4 + page.close() + doc.close() + assert doc.closed + + +def test_native_rejects_stale_tokens_even_after_slot_reuse(pdf_path): + from pyjpdfium._loader import native + + lib = native() + first = lib.doc_open(str(pdf_path), None) + lib.doc_close(first) + second = lib.doc_open(str(pdf_path), None) + try: + assert first != second + with pytest.raises(errors.StaleHandleError): + lib.doc_page_count(first) + page = lib.page_open(second, 0) + with pytest.raises(errors.WrongHandleTypeError): + lib.doc_page_count(page) + with pytest.raises(errors.InvalidHandleError): + lib.doc_page_count(0x0100000100FFFFFF) + lib.page_close(page) + finally: + lib.doc_close(second) + + +def test_input_errors_map_to_exceptions(pdf_path, tmp_path): + with pytest.raises(errors.PdfIOError): + pyjpdfium.open_document(tmp_path / "missing.pdf") + junk = tmp_path / "junk.pdf" + junk.write_bytes(b"not a pdf") + with pytest.raises(errors.PdfFormatError) as info: + pyjpdfium.open_document(junk) + assert info.value.status_name == "JPDF_E_FORMAT" + with pyjpdfium.open_document(pdf_path) as doc: + with pytest.raises(errors.PageNotFoundError): + doc.page(4) + with pytest.raises(IndexError): + doc.page(-1) + + +def test_unicode_path(tmp_path): + from conftest import fixture_pdf + + path = tmp_path / "dokument-üß-文件.pdf" + path.write_bytes(fixture_pdf()) + with pyjpdfium.open_document(path) as doc: + assert doc.page_count == 4 + + +def test_unreferenced_objects_are_reclaimed(pdf_path): + before = pyjpdfium.runtime_info() + doc = pyjpdfium.open_document(pdf_path) + doc.page(0) + doc.page(1).render() + del doc + gc.collect() + after = pyjpdfium.runtime_info() + assert after.live_documents == before.live_documents + assert after.live_pages == before.live_pages + assert after.live_bitmaps == before.live_bitmaps diff --git a/tests/native/test_output.py b/tests/native/test_output.py new file mode 100644 index 0000000..21bf888 --- /dev/null +++ b/tests/native/test_output.py @@ -0,0 +1,54 @@ +import struct + +import pytest + +import pyjpdfium +from pyjpdfium import errors + +pytestmark = pytest.mark.native + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def png_chunks(data): + assert data[:8] == PNG_SIGNATURE + pos = 8 + while pos < len(data): + (length,) = struct.unpack(">I", data[pos : pos + 4]) + kind = data[pos + 4 : pos + 8] + yield kind, data[pos + 8 : pos + 8 + length] + pos += 12 + length + + +def staging_leftovers(directory): + return [p.name for p in directory.iterdir() if ".jpdf-" in p.name] + + +def test_render_to_file_writes_png_with_dpi(pdf_path, tmp_path): + out = tmp_path / "page.png" + with pyjpdfium.open_document(pdf_path) as doc, doc.page(0) as page: + assert page.render(out, dpi=(300, 150), pixel_format="gray") is None + chunks = dict(png_chunks(out.read_bytes())) + width, height, depth, color = struct.unpack(">IIBB", chunks[b"IHDR"][:10]) + assert (width, height, depth, color) == (2550, 1650, 8, 0) + ppm_x, ppm_y, unit = struct.unpack(">IIB", chunks[b"pHYs"]) + assert (round(ppm_x * 0.0254), round(ppm_y * 0.0254), unit) == (300, 150, 1) + assert staging_leftovers(tmp_path) == [] + + +def test_failed_write_leaves_destination_untouched(pdf_path, tmp_path): + out = tmp_path / "page.png" + out.write_bytes(b"previous content") + with pyjpdfium.open_document(pdf_path) as doc, doc.page(1) as page: + with pytest.raises(errors.LimitExceededError): + page.render(out, dpi=100, max_bytes=64) + bitmap = page.render(dpi=50) + with pytest.raises(errors.LimitExceededError): + bitmap.save(out, max_bytes=10) + assert out.read_bytes() == b"previous content" + assert staging_leftovers(tmp_path) == [] + with pytest.raises(errors.PdfIOError): + bitmap.save(tmp_path / "missing-dir" / "page.png") + bitmap.save(out, durable=True) + assert out.read_bytes()[:8] == PNG_SIGNATURE + bitmap.close() diff --git a/tests/native/test_render.py b/tests/native/test_render.py new file mode 100644 index 0000000..95fc2d4 --- /dev/null +++ b/tests/native/test_render.py @@ -0,0 +1,83 @@ +import pytest + +import pyjpdfium +from pyjpdfium import PixelFormat, errors + +pytestmark = pytest.mark.native + + +def pixel(bitmap, x, y): + view = bitmap.view() + if view.ndim == 2: + return view[y, x] + return tuple(view[y, x, c] for c in range(view.shape[2])) + + +def test_render_size_format_and_content(pdf_path): + with pyjpdfium.open_document(pdf_path) as doc, doc.page(0) as page: + with page.render(dpi=72) as bitmap: + assert (bitmap.width, bitmap.height) == (612, 792) + assert bitmap.pixel_format is PixelFormat.RGB + assert bitmap.info.stride == 612 * 3 + assert bitmap.info.dpi == (72.0, 72.0) + # Square spans x 72-172pt, y 600-700pt; image rows run top down. + assert pixel(bitmap, 122, 792 - 650) == (0, 0, 0) + assert pixel(bitmap, 400, 792 - 650) == (255, 255, 255) + with page.render(dpi=(36, 18), pixel_format="gray") as gray: + assert (gray.width, gray.height) == (306, 198) + assert gray.view().shape == (198, 306) + with page.render(size=(100, 50), pixel_format=PixelFormat.RGBA, transparent=True) as rgba: + assert (rgba.width, rgba.height) == (100, 50) + assert pixel(rgba, 99, 49)[3] == 0 + + +def test_bitmap_outlives_page_and_document(pdf_path): + doc = pyjpdfium.open_document(pdf_path) + page = doc.page(3) + bitmap = page.render(dpi=20, pixel_format="gray") + data = bitmap.tobytes() + page.close() + doc.close() + assert bitmap.tobytes() == data + bitmap.close() + + +def test_view_retains_pixels_after_bitmap_close(pdf_path): + with pyjpdfium.open_document(pdf_path) as doc, doc.page(0) as page: + bitmap = page.render(dpi=20) + expected = bitmap.tobytes() + view = bitmap.view() + bitmap.close() + assert view.tobytes() == expected + assert pyjpdfium.runtime_info().live_views >= 1 + with pytest.raises(errors.StaleHandleError): + bitmap.view() + with pytest.raises(TypeError): + view[0, 0, 0] = 1 + del view + assert pyjpdfium.runtime_info().live_views == 0 + + +def test_invalid_render_options(pdf_path): + with pyjpdfium.open_document(pdf_path) as doc, doc.page(0) as page: + with pytest.raises(errors.InvalidArgumentError): + page.render(dpi=0) + with pytest.raises(errors.InvalidArgumentError): + page.render(pixel_format="rgb", transparent=True) + with pytest.raises(errors.LimitExceededError): + page.render(dpi=72, max_pixels=100) + with pytest.raises(KeyError): + page.render(pixel_format="cmyk") + + +def test_capabilities(): + assert pyjpdfium.capability("pdf.render") is pyjpdfium.Capability.IMPLEMENTED + assert pyjpdfium.capability("ocr.recognize") is pyjpdfium.Capability.ABSENT + + +def test_runtime_identity(): + info = pyjpdfium.runtime_info() + assert info.state is pyjpdfium.RuntimeState.RUNNING + assert info.api_version[0] == 1 + assert info.renderer is not None + assert info.build_id.startswith("jpdfium ") diff --git a/tests/native/test_shutdown.py b/tests/native/test_shutdown.py new file mode 100644 index 0000000..a2eb038 --- /dev/null +++ b/tests/native/test_shutdown.py @@ -0,0 +1,51 @@ +import subprocess +import sys + +import pytest + +pytestmark = pytest.mark.native + +# Shutdown is final for a process, so it runs in a fresh interpreter. +PROBE = """ +import sys +import pyjpdfium +from pyjpdfium import errors, RuntimeState + +doc = pyjpdfium.open_document(sys.argv[1]) +page = doc.page(0) +bitmap = page.render(dpi=10, pixel_format="gray") +try: + pyjpdfium.shutdown() +except errors.ResourceBusyError as exc: + assert "1 documents and 1 pages" in str(exc), exc +else: + raise AssertionError("shutdown with live pages must be refused") +assert pyjpdfium.runtime_info().state is RuntimeState.QUIESCING +try: + pyjpdfium.open_document(sys.argv[1]) +except errors.RuntimeStateError as exc: + assert exc.status_name == "JPDF_E_QUIESCING" +else: + raise AssertionError("no new work while quiescing") +page.close() +doc.close() +pyjpdfium.shutdown() +assert pyjpdfium.runtime_info().state is RuntimeState.STOPPED +try: + pyjpdfium.open_document(sys.argv[1]) +except errors.RuntimeStateError as exc: + assert exc.status_name == "JPDF_E_STOPPED" +else: + raise AssertionError("stopped is final") +assert len(bitmap.tobytes()) == 85 * 110 +bitmap.close() +print("shutdown-ok") +""" + + +def test_shutdown_lifecycle(pdf_path): + result = subprocess.run( + [sys.executable, "-c", PROBE, str(pdf_path)], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + assert "shutdown-ok" in result.stdout diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py new file mode 100644 index 0000000..e5598a7 --- /dev/null +++ b/tests/unit/test_errors.py @@ -0,0 +1,45 @@ +import pytest + +from pyjpdfium import errors + + +def test_unknown_status_is_never_success(): + exc = errors._from_status(-1234, "JPDF_E_UNKNOWN", "") + assert isinstance(exc, errors.InternalError) + assert exc.status == -1234 + + +def test_detail_and_name_in_message(): + exc = errors._from_status(-3, "JPDF_E_STALE_HANDLE", "page handle rejected") + assert isinstance(exc, errors.StaleHandleError) + assert isinstance(exc, errors.HandleError) + assert str(exc) == "page handle rejected (JPDF_E_STALE_HANDLE)" + assert exc.status_name == "JPDF_E_STALE_HANDLE" + + +@pytest.mark.parametrize( + ("status", "builtin"), + [ + (-1, ValueError), + (-4, TypeError), + (-7, OSError), + (-10, IndexError), + (-12, MemoryError), + (-21, TimeoutError), + ], +) +def test_builtin_bases_construct_and_catch(status, builtin): + exc = errors._from_status(status, "X", "detail") + assert isinstance(exc, builtin) + with pytest.raises(builtin): + raise exc + + +@pytest.mark.native +def test_table_covers_every_compiled_status(): + from pyjpdfium._loader import native + + for name, code in native().STATUS.items(): + if code == 0: + continue + assert code in errors.STATUS_ERRORS, name diff --git a/tests/unit/test_lazy_import.py b/tests/unit/test_lazy_import.py new file mode 100644 index 0000000..13abfcc --- /dev/null +++ b/tests/unit/test_lazy_import.py @@ -0,0 +1,44 @@ +import subprocess +import sys + +PROBE = """ +import sys +import pyjpdfium +import pyjpdfium.errors, pyjpdfium.models, pyjpdfium.pdf, pyjpdfium.runtime +assert pyjpdfium.PixelFormat.RGB == 2 +loaded = [m for m in sys.modules if m.startswith("pyjpdfium._native")] +assert not loaded, loaded +try: + maps = open("/proc/self/maps").read() +except OSError: + maps = "" +assert "libjpdfium" not in maps and "libpdfium" not in maps +print("lazy-ok") +""" + + +def test_import_loads_no_native_code(): + # A fresh interpreter, so nothing another test loaded can hide an eager import. + result = subprocess.run( + [sys.executable, "-c", PROBE], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + assert "lazy-ok" in result.stdout + + +def test_unknown_attribute_raises(): + import pyjpdfium + + try: + pyjpdfium.does_not_exist # noqa: B018 + except AttributeError as exc: + assert "does_not_exist" in str(exc) + else: + raise AssertionError("expected AttributeError") + + +def test_every_export_resolves(): + import pyjpdfium + + for name in pyjpdfium.__all__: + assert getattr(pyjpdfium, name) is not None diff --git a/tests/unit/test_loader.py b/tests/unit/test_loader.py new file mode 100644 index 0000000..f578bf8 --- /dev/null +++ b/tests/unit/test_loader.py @@ -0,0 +1,61 @@ +from types import SimpleNamespace + +import pytest + +from pyjpdfium import _loader +from pyjpdfium.errors import NativeLoadError +from pyjpdfium.models import Renderer + + +def fake_module(library=0x10000, compiled=0x10000, layouts=True): + return SimpleNamespace( + api_version=lambda: library, + COMPILED_API_VERSION=compiled, + struct_sizes_match=lambda: layouts, + ) + + +@pytest.fixture(autouse=True) +def single_core(monkeypatch): + monkeypatch.setattr(_loader, "_loaded_core_paths", lambda: {"/lib/libjpdfium.so"}) + + +def test_compatible_core_is_accepted(): + _loader._check_identity(fake_module(library=0x10003, compiled=0x10001)) + + +@pytest.mark.parametrize( + ("library", "compiled"), + [(0x20000, 0x10000), (0x10000, 0x10002)], + ids=["major-mismatch", "older-minor"], +) +def test_incompatible_api_is_rejected(library, compiled): + with pytest.raises(NativeLoadError, match="incompatible"): + _loader._check_identity(fake_module(library=library, compiled=compiled)) + + +def test_layout_mismatch_is_rejected(): + with pytest.raises(NativeLoadError, match="layouts"): + _loader._check_identity(fake_module(layouts=False)) + + +def test_two_cores_are_rejected(monkeypatch): + monkeypatch.setattr( + _loader, "_loaded_core_paths", lambda: {"/a/libjpdfium.so", "/b/libjpdfium.so"} + ) + with pytest.raises(NativeLoadError, match="multiple JPDFium cores"): + _loader._check_identity(fake_module()) + + +def test_renderer_selection(monkeypatch): + monkeypatch.setenv(_loader.RENDERER_ENV, "agg") + assert _loader._renderer() is Renderer.AGG + monkeypatch.setenv(_loader.RENDERER_ENV, "vulkan") + with pytest.raises(NativeLoadError, match="PYJPDFIUM_RENDERER"): + _loader._renderer() + + +def test_missing_core_is_reported(tmp_path, monkeypatch): + monkeypatch.setenv(_loader.NATIVE_DIR_ENV, str(tmp_path)) + with pytest.raises(NativeLoadError, match="not found"): + _loader._pin_core(_loader.native_dir()) From 2d21a493d99bdbcc8d01d169f325880c388bc1f0 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:49:35 +0100 Subject: [PATCH 02/10] Document sharing the native domain with the Java binding --- README.md | 5 +++-- docs/ARCHITECTURE.md | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e290f1..8f5f556 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,9 @@ with pyjpdfium.open_document("input.pdf") as document: leaves the document usable. - Bitmaps are independent of their page and document, and a `view()` keeps its pixels alive even after the bitmap is closed. -- One process uses one native core. Mixing it with JPDFium's legacy Java bridge in the same - process is refused. +- One process uses one native core. JPDFium's Java binding is admitted by the same native + domain, so the two can share a process only when both load the same `libjpdfium` file; the + loader refuses a second copy. ## Building diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ba2f05f..8927446 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -7,6 +7,10 @@ PyJPDFium is a peer of JPDFium's Java binding. Both bind one safe native C API, - Object lifetime: generation-checked handles, parent retention, `BUSY` instead of partial close. - Output safety: staged, bounded, atomically published files. +JPDFium's Java binding reaches PDFium through generated `jpdf_guard_*` shims admitted by the same +domain, so both bindings can run in one process without a second serializer, provided they load +the same `libjpdfium` file. + The full contracts live in JPDFium's `docs/design/` (`ARCHITECTURE.md`, `NATIVE_ABI.md`, `LIFETIME.md`, `CALLBACKS.md`, `CAPABILITIES.md`). From c66a48251d7afe37f74a0bd743ec251639b5fd58 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:12:06 +0100 Subject: [PATCH 03/10] Add page rasterization and rectification with reference parity tests --- .github/workflows/ci.yml | 8 ++ README.md | 11 ++- docs/COMPATIBILITY.md | 7 ++ pyproject.toml | 1 + src/native/pyjpdfium_native.cpp | 55 ++++++++++++ src/pyjpdfium/__init__.py | 6 ++ src/pyjpdfium/models.py | 43 ++++++++++ src/pyjpdfium/pdf.py | 43 +++++++++- src/pyjpdfium/raster.py | 83 ++++++++++++++++++ tests/native/test_raster.py | 116 +++++++++++++++++++++++++ tests/native/test_rectify.py | 56 ++++++++++++ tests/parity/fetch_reference.py | 47 ++++++++++ tests/parity/reference_runner.py | 74 ++++++++++++++++ tests/parity/requirements.txt | 6 ++ tests/parity/test_raster_parity.py | 133 +++++++++++++++++++++++++++++ tests/pngread.py | 75 ++++++++++++++++ 16 files changed, 760 insertions(+), 4 deletions(-) create mode 100644 src/pyjpdfium/raster.py create mode 100644 tests/native/test_raster.py create mode 100644 tests/native/test_rectify.py create mode 100644 tests/parity/fetch_reference.py create mode 100644 tests/parity/reference_runner.py create mode 100644 tests/parity/requirements.txt create mode 100644 tests/parity/test_raster_parity.py create mode 100644 tests/pngread.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98607c3..1449223 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,3 +97,11 @@ jobs: # Native tests skip when the extension is missing; fail loudly here instead. python -c "import pyjpdfium; print(pyjpdfium.runtime_info())" python -m pytest -q -rs + + - name: Raster parity with OCRmyPDF's pypdfium rasterizer + env: + PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + run: | + python -m pip install --require-hashes --only-binary=:all: -r tests/parity/requirements.txt + export PYJPDFIUM_PARITY_SRC="$(python tests/parity/fetch_reference.py "$RUNNER_TEMP/parity")" + python -m pytest tests/parity -q -rs diff --git a/README.md b/README.md index 8f5f556..03e580c 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ PyJPDFium is a thin [nanobind](https://github.com/wjakob/nanobind) binding over `jpdf_*` C API. The native core, not Python or the GIL, owns PDFium serialization, object lifetimes and handle validation, so the same rules hold for the Java and Python bindings. -Status: early development. This first pass covers documents, pages, rendering and transactional -PNG output. +Status: early development. Covered so far: documents, pages, rendering, transactional PNG output, +OCRmyPDF-compatible page rasterization and perspective rectification. ```python import pyjpdfium @@ -18,6 +18,10 @@ with pyjpdfium.open_document("input.pdf") as document: page.render("page.png", dpi=(300, 300)) # atomic publish; failure leaves page.png as-is with page.render(dpi=150, pixel_format="gray") as bitmap: pixels = bitmap.view() # zero-copy, read-only, shape (h, w) + crop, geometry = bitmap.rectify([(40, 30), (300, 42), (296, 90), (36, 80)]) + +# OCRmyPDF's rasterize_pdf_page contract, natively (see JPDFium docs/design/RASTER.md). +pyjpdfium.rasterize_page("input.pdf", 0, "page.png", device="pnggray", raster_dpi=300) ``` ## Behaviour @@ -50,5 +54,8 @@ PYJPDFIUM_NATIVE_DIR=$JPDFIUM_NATIVE_DIR pytest `PYJPDFIUM_RENDERER` selects `auto` (default), `agg` or `skia`. Native tests (`-m native`) need the real core and are skipped when the extension is not built. +Raster parity with OCRmyPDF's own rasterizer is opt-in: install `tests/parity/requirements.txt` +(hash-pinned), set `PYJPDFIUM_PARITY_SRC` to the output of `tests/parity/fetch_reference.py`, and +run `pytest tests/parity`. See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index f547cdc..3687f90 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -9,6 +9,13 @@ On Windows the build also needs `jpdfium.lib`, the import library produced next to `jpdfium.dll`. +## Rasterization + +`rasterize_page` follows OCRmyPDF 17.13's pypdfium rasterizer, with Ghostscript's behaviour where +the two disagree. It produces PNG only: `jpeg`, `jpeggray` and `tiff` devices are refused, and +`png256` is written as RGB. JPDFium's `docs/design/RASTER.md` lists every deviation and the +measured pixel parity. + ## Not in this release - OCR, text-layer writing and the OCRmyPDF plugin. diff --git a/pyproject.toml b/pyproject.toml index c6d2a3e..b75753f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,3 +38,4 @@ PYJPDFIUM_BUNDLE_NATIVES = { env = "PYJPDFIUM_BUNDLE_NATIVES", default = "OFF" } [tool.pytest.ini_options] testpaths = ["tests"] markers = ["native: needs the real JPDFium native core"] +norecursedirs = ["parity"] diff --git a/src/native/pyjpdfium_native.cpp b/src/native/pyjpdfium_native.cpp index 9fd3bb4..e659e70 100644 --- a/src/native/pyjpdfium_native.cpp +++ b/src/native/pyjpdfium_native.cpp @@ -6,11 +6,13 @@ #include #include #include +#include #include #include #include #include +#include #include "jpdfium_api.h" @@ -82,6 +84,10 @@ bool struct_sizes_match() { {JPDF_STRUCT_BITMAP_INFO, sizeof(jpdf_bitmap_info)}, {JPDF_STRUCT_BITMAP_VIEW, sizeof(jpdf_bitmap_view)}, {JPDF_STRUCT_WRITE_OPTIONS, sizeof(jpdf_write_options)}, + {JPDF_STRUCT_RASTER_OPTIONS, sizeof(jpdf_raster_options)}, + {JPDF_STRUCT_RASTER_RESULT, sizeof(jpdf_raster_result)}, + {JPDF_STRUCT_RECTIFY_OPTIONS, sizeof(jpdf_rectify_options)}, + {JPDF_STRUCT_RECTIFY_RESULT, sizeof(jpdf_rectify_result)}, }; for (const auto& pair : expected) { if (jpdf_abi_struct_size(static_cast(pair[0])) != pair[1]) return false; @@ -269,4 +275,53 @@ NB_MODULE(_native, m) { }); m.def("bitmap_close", [](jpdf_handle bitmap) { run([&] { return jpdf_bitmap_close(bitmap); }); }); + m.def("bitmap_rectify", [](jpdf_handle bitmap, const std::vector& quad, int32_t width, + int32_t height, int32_t interpolation, int32_t border, + uint64_t max_pixels) { + if (quad.size() != 8) throw nb::value_error("quad needs 8 coordinates"); + jpdf_rectify_options o{}; + o.struct_size = sizeof(o); + o.width = width; + o.height = height; + o.interpolation = interpolation; + o.border = border; + o.max_pixels = max_pixels; + jpdf_rectify_result r{}; + r.struct_size = sizeof(r); + jpdf_handle out = 0; + run([&] { return jpdf_bitmap_rectify(bitmap, quad.data(), &o, &out, &r); }); + return std::make_tuple(out, std::vector(r.quad, r.quad + 8), + std::vector(r.to_source, r.to_source + 9)); + }); + + m.def("rasterize_page_file", + [](const std::string& path, int32_t page_index, int32_t device, double dpi_x, + double dpi_y, double out_dpi_x, double out_dpi_y, int32_t rotation_ccw, + uint32_t flags, uint64_t max_pixels, std::optional password, + const std::string& output, bool durable, int32_t png_compression, + uint64_t max_bytes) { + jpdf_raster_options o{}; + o.struct_size = sizeof(o); + o.flags = flags; + o.raster_dpi_x = dpi_x; + o.raster_dpi_y = dpi_y; + o.output_dpi_x = out_dpi_x; + o.output_dpi_y = out_dpi_y; + o.rotation_ccw = rotation_ccw; + o.device = device; + o.max_pixels = max_pixels; + o.password = password ? password->c_str() : nullptr; + jpdf_write_options w = write_options(durable, png_compression, max_bytes); + jpdf_raster_result r{}; + r.struct_size = sizeof(r); + run([&] { + return jpdf_rasterize_page_file(path.c_str(), page_index, &o, output.c_str(), + &w, &r); + }); + return std::make_tuple(r.width, r.height, r.bits_per_pixel, r.dpi_x, r.dpi_y); + }, + nb::arg("path"), nb::arg("page_index"), nb::arg("device"), nb::arg("dpi_x"), + nb::arg("dpi_y"), nb::arg("out_dpi_x"), nb::arg("out_dpi_y"), nb::arg("rotation_ccw"), + nb::arg("flags"), nb::arg("max_pixels"), nb::arg("password").none(), nb::arg("output"), + nb::arg("durable"), nb::arg("png_compression"), nb::arg("max_bytes")); } diff --git a/src/pyjpdfium/__init__.py b/src/pyjpdfium/__init__.py index becbf6a..c5cc7f8 100644 --- a/src/pyjpdfium/__init__.py +++ b/src/pyjpdfium/__init__.py @@ -16,6 +16,7 @@ "Document": "pyjpdfium.pdf", "Page": "pyjpdfium.pdf", "Bitmap": "pyjpdfium.pdf", + "rasterize_page": "pyjpdfium.raster", "runtime_info": "pyjpdfium.runtime", "capability": "pyjpdfium.runtime", "quiesce": "pyjpdfium.runtime", @@ -28,6 +29,11 @@ "PageSize": "pyjpdfium.models", "BitmapInfo": "pyjpdfium.models", "RuntimeInfo": "pyjpdfium.models", + "RasterDevice": "pyjpdfium.models", + "RasterResult": "pyjpdfium.models", + "Interpolation": "pyjpdfium.models", + "Border": "pyjpdfium.models", + "RectifyResult": "pyjpdfium.models", "PyJPDFiumError": "pyjpdfium.errors", "NativeLoadError": "pyjpdfium.errors", "InvalidArgumentError": "pyjpdfium.errors", diff --git a/src/pyjpdfium/models.py b/src/pyjpdfium/models.py index 41d6745..dbc9613 100644 --- a/src/pyjpdfium/models.py +++ b/src/pyjpdfium/models.py @@ -80,3 +80,46 @@ class RuntimeInfo: domain_acquisitions: int domain_wait_ns: int domain_hold_ns: int + + +class RasterDevice(IntEnum): + """Native raster devices, named after the Ghostscript devices OCRmyPDF requests.""" + + PNGMONO = 1 + PNGMONOD = 2 + PNGGRAY = 3 + PNG256 = 4 # written as 24-bit RGB; no palette quantization + PNG16M = 5 + PNGALPHA = 6 # opaque white background, so written as RGB like the reference + + +class Interpolation(IntEnum): + NEAREST = 0 + BILINEAR = 1 + BICUBIC = 2 # OpenCV INTER_CUBIC kernel + + +class Border(IntEnum): + WHITE = 0 + REPLICATE = 1 + + +@dataclass(frozen=True) +class RasterResult: + width: int + height: int + bits_per_pixel: int + dpi: tuple[float, float] + + +@dataclass(frozen=True) +class RectifyResult: + """Corners as used (top-left, top-right, bottom-right, bottom-left) and the inverse map.""" + + quad: tuple[tuple[float, float], ...] + to_source: tuple[float, ...] # row-major 3x3, output pixel -> source pixel + + def map_to_source(self, x: float, y: float) -> tuple[float, float]: + h = self.to_source + w = h[6] * x + h[7] * y + h[8] + return (h[0] * x + h[1] * y + h[2]) / w, (h[3] * x + h[4] * y + h[5]) / w diff --git a/src/pyjpdfium/pdf.py b/src/pyjpdfium/pdf.py index 3457d58..7b88141 100644 --- a/src/pyjpdfium/pdf.py +++ b/src/pyjpdfium/pdf.py @@ -5,12 +5,20 @@ import os import threading import weakref -from collections.abc import Callable +from collections.abc import Callable, Sequence from typing import Union from ._loader import native from .errors import InvalidArgumentError, PyJPDFiumError, StaleHandleError -from .models import BitmapInfo, PageSize, PixelFormat, RenderFlags +from .models import ( + BitmapInfo, + Border, + Interpolation, + PageSize, + PixelFormat, + RectifyResult, + RenderFlags, +) StrPath = Union[str, "os.PathLike[str]"] Dpi = Union[float, "tuple[float, float]"] @@ -224,6 +232,37 @@ def save( max_bytes or _NO_LIMIT, ) + def rectify( + self, + quad: "Sequence[tuple[float, float]] | Sequence[float]", + *, + size: tuple[int, int] | None = None, + interpolation: Interpolation | str = Interpolation.BILINEAR, + border: Border | str = Border.WHITE, + max_pixels: int | None = None, + ) -> tuple[Bitmap, RectifyResult]: + """Perspective-corrects a quad (4 corners in source pixels, any order) into a new bitmap. + + Without ``size`` the output takes PaddleOCR's truncated lengths of the longer opposite + edges. The result maps output pixels back to source pixels. + """ + values = list(quad) + if values and isinstance(values[0], (tuple, list)): + flat = [float(c) for point in values for c in point] + else: + flat = [float(v) for v in values] + if isinstance(interpolation, str): + interpolation = Interpolation[interpolation.upper()] + if isinstance(border, str): + border = Border[border.upper()] + width, height = size if size is not None else (0, 0) + token, corners, to_source = native().bitmap_rectify( + self._require_open(), flat, width, height, int(interpolation), int(border), + max_pixels or 0, + ) + points = tuple((corners[i], corners[i + 1]) for i in range(0, 8, 2)) + return Bitmap(token), RectifyResult(points, tuple(to_source)) + def __repr__(self) -> str: state = "closed" if self.closed else "open" return f"" diff --git a/src/pyjpdfium/raster.py b/src/pyjpdfium/raster.py new file mode 100644 index 0000000..34c0742 --- /dev/null +++ b/src/pyjpdfium/raster.py @@ -0,0 +1,83 @@ +"""Page rasterization with OCRmyPDF's rasterize_pdf_page contract, done natively.""" + +from __future__ import annotations + +import os +from typing import Union + +from ._loader import native +from .errors import UnsupportedError +from .models import RasterDevice, RasterResult + +StrPath = Union[str, "os.PathLike[str]"] +Dpi = Union[float, "tuple[float, float]"] + +_USE_CROPBOX = 0x0001 +_FILTER_VECTOR = 0x0002 + +# Ghostscript devices OCRmyPDF may request that this rasterizer does not produce. +_NOT_PRODUCED = {"jpeg", "jpeggray", "jpg", "tiff", "tif"} + + +def _device(device: RasterDevice | str) -> RasterDevice: + if isinstance(device, RasterDevice): + return device + name = device.lower() + if name in _NOT_PRODUCED: + raise UnsupportedError(f"raster device {device!r} needs a JPEG or TIFF encoder") + try: + return RasterDevice[name.upper()] + except KeyError: + raise UnsupportedError(f"unknown raster device {device!r}") from None + + +def _pair(dpi: Dpi) -> tuple[float, float]: + if isinstance(dpi, (int, float)): + return float(dpi), float(dpi) + x, y = dpi + return float(x), float(y) + + +def rasterize_page( + path: StrPath, + page_index: int, + output: StrPath, + *, + device: RasterDevice | str = RasterDevice.PNG16M, + raster_dpi: Dpi = 300.0, + output_dpi: Dpi | None = None, + rotation: int = 0, + use_cropbox: bool = False, + filter_vector: bool = False, + password: str | None = None, + max_pixels: int | None = None, + durable: bool = False, + png_compression: int | None = None, + max_bytes: int | None = None, +) -> RasterResult: + """Rasterizes one page (0-based) to a PNG, published atomically. + + ``rotation`` is a counterclockwise correction in quarter turns, applied on top of the page's + own /Rotate as Ghostscript does. ``output_dpi`` sets only the DPI written to the file. + """ + dpi_x, dpi_y = _pair(raster_dpi) + out_x, out_y = _pair(output_dpi) if output_dpi is not None else (0.0, 0.0) + flags = (_USE_CROPBOX if use_cropbox else 0) | (_FILTER_VECTOR if filter_vector else 0) + width, height, bpp, rx, ry = native().rasterize_page_file( + os.fsdecode(os.fspath(path)), + page_index, + int(_device(device)), + dpi_x, + dpi_y, + out_x, + out_y, + rotation % 360, + flags, + max_pixels or 0, + password, + os.fsdecode(os.fspath(output)), + durable, + -1 if png_compression is None else png_compression, + max_bytes or 0, + ) + return RasterResult(width, height, bpp, (rx, ry)) diff --git a/tests/native/test_raster.py b/tests/native/test_raster.py new file mode 100644 index 0000000..66f3665 --- /dev/null +++ b/tests/native/test_raster.py @@ -0,0 +1,116 @@ +import importlib.util + +import pytest +from pngread import read_png + +import pyjpdfium +from pyjpdfium import errors + +pytestmark = pytest.mark.native + + +def one_page(tmp_path, name, page_extra, content="0 0 0 rg 72 600 100 100 re f"): + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + f"<< /Type /Page /Parent 2 0 R {page_extra} /Contents 4 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + ] + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n{body}\nendobj\n".encode() + xref = len(out) + out += b"xref\n0 5\n0000000000 65535 f \n" + out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets) + out += f"trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + path = tmp_path / name + path.write_bytes(bytes(out)) + return path + + +@pytest.fixture +def letter(tmp_path): + return one_page(tmp_path, "letter.pdf", "/MediaBox [0 0 612 792]") + + +@pytest.mark.parametrize( + ("device", "depth", "color"), + [("png16m", 8, 2), ("pnggray", 8, 0), ("pngmonod", 1, 0), ("pngmono", 1, 0), + ("png256", 8, 2), ("pngalpha", 8, 2)], +) +def test_devices_produce_the_reference_png_modes(letter, tmp_path, device, depth, color): + out = tmp_path / f"{device}.png" + result = pyjpdfium.rasterize_page(letter, 0, out, device=device, raster_dpi=72) + png = read_png(out) + assert (png.width, png.height, png.bit_depth, png.color_type) == (612, 792, depth, color) + assert result.bits_per_pixel == (1 if depth == 1 else 8 * png.channels) + # The square spans x 72-172pt and y 600-700pt; rows run top down. + assert png.pixel(122, 142)[0] == 0 + assert png.pixel(400, 142)[0] in (1, 255) + + +def test_dpi_metadata_follows_pillow(letter, tmp_path): + out = tmp_path / "dpi.png" + pyjpdfium.rasterize_page(letter, 0, out, raster_dpi=300, output_dpi=(200, 100)) + png = read_png(out) + assert (png.width, png.height) == (2550, 3300) + assert png.ppm == (int(200 / 0.0254 + 0.5), int(100 / 0.0254 + 0.5)) + + +def test_quarter_turn_follows_ghostscript(letter, tmp_path): + out = tmp_path / "rot.png" + result = pyjpdfium.rasterize_page(letter, 0, out, device="pnggray", raster_dpi=(300, 150), + rotation=90) + assert (result.width, result.height) == (1650, 2550) + assert result.dpi == (150.0, 300.0) + png = read_png(out) + # A counterclockwise turn moves the square from the top-left to the bottom-left. + assert png.pixel(int(142 * 150 / 72), int((612 - 122) * 300 / 72))[0] == 0 + assert png.pixel(int(142 * 150 / 72), int(122 * 300 / 72))[0] == 255 + + +def test_cropbox_policy(tmp_path): + pdf = one_page(tmp_path, "box.pdf", "/MediaBox [0 0 612 792] /CropBox [100 100 400 500]") + media = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "m.png", device="pnggray", raster_dpi=72) + crop = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "c.png", device="pnggray", raster_dpi=72, + use_cropbox=True) + assert (media.width, media.height) == (612, 792) + assert (crop.width, crop.height) == (300, 400) + + +def test_python_rounding_semantics(tmp_path): + # 5pt at 36 dpi is 2.5 px; Python's round() gives 2. + pdf = one_page(tmp_path, "tiny.pdf", "/MediaBox [0 0 5 5]", "0 g 0 0 5 5 re f") + result = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "t.png", device="pnggray", raster_dpi=36) + assert (result.width, result.height) == (round(2.5), round(2.5)) == (2, 2) + + +def test_unsupported_requests_are_refused(letter, tmp_path): + out = tmp_path / "x.png" + for device in ("jpeggray", "tiff", "nonsense"): + with pytest.raises(errors.UnsupportedError): + pyjpdfium.rasterize_page(letter, 0, out, device=device) + with pytest.raises(errors.UnsupportedError): + pyjpdfium.rasterize_page(letter, 0, out, filter_vector=True) + with pytest.raises(errors.InvalidArgumentError): + pyjpdfium.rasterize_page(letter, 0, out, rotation=45) + with pytest.raises(errors.PageNotFoundError): + pyjpdfium.rasterize_page(letter, 1, out) + with pytest.raises(errors.LimitExceededError): + pyjpdfium.rasterize_page(letter, 0, out, max_pixels=100) + assert not out.exists() + + +@pytest.mark.skipif(importlib.util.find_spec("PIL") is None, reason="Pillow is not installed") +def test_mono_matches_pillow_floyd_steinberg(tmp_path): + from PIL import Image + + # A smooth gradient exercises error diffusion across every gray level. + content = "".join(f"{g / 255:.4f} g {g * 2} 0 2 792 re f " for g in range(256)) + pdf = one_page(tmp_path, "ramp.pdf", "/MediaBox [0 0 512 792]", content) + pyjpdfium.rasterize_page(pdf, 0, tmp_path / "g.png", device="pnggray", raster_dpi=50) + pyjpdfium.rasterize_page(pdf, 0, tmp_path / "m.png", device="pngmonod", raster_dpi=50) + gray, mono = read_png(tmp_path / "g.png"), read_png(tmp_path / "m.png") + expected = Image.frombytes("L", (gray.width, gray.height), b"".join(gray.rows)).convert("1") + assert mono.gray_bytes() == expected.convert("L").tobytes() diff --git a/tests/native/test_rectify.py b/tests/native/test_rectify.py new file mode 100644 index 0000000..147c181 --- /dev/null +++ b/tests/native/test_rectify.py @@ -0,0 +1,56 @@ +import math + +import pytest + +import pyjpdfium +from pyjpdfium import errors + +pytestmark = pytest.mark.native + + +@pytest.fixture +def gray_page(pdf_path): + with pyjpdfium.open_document(pdf_path) as doc, doc.page(0) as page: + bitmap = page.render(dpi=72, pixel_format="gray") + yield bitmap + bitmap.close() + + +def test_identity_quad_in_any_order_is_exact(gray_page): + source = gray_page.tobytes() + corners = [(612, 792), (0, 0), (0, 792), (612, 0)] + for interpolation in ("nearest", "bilinear", "bicubic"): + out, result = gray_page.rectify(corners, interpolation=interpolation) + with out: + assert (out.width, out.height) == (612, 792) + assert out.tobytes() == source + assert result.quad == ((0, 0), (612, 0), (612, 792), (0, 792)) + + +def test_crop_maps_back_to_source(gray_page): + quad = [72, 92, 172, 92, 172, 192, 72, 192] + out, result = gray_page.rectify(quad, interpolation="nearest") + with out: + assert (out.width, out.height) == (100, 100) + assert out.view()[50, 50] == 0 + sx, sy = result.map_to_source(100, 100) + assert math.isclose(sx, 172) and math.isclose(sy, 192) + + +def test_explicit_size_and_budget(gray_page): + quad = [72, 92, 172, 92, 172, 192, 72, 192] + out, _ = gray_page.rectify(quad, size=(320, 48), border="replicate", interpolation="bicubic") + with out: + assert (out.width, out.height) == (320, 48) + with pytest.raises(errors.LimitExceededError): + gray_page.rectify(quad, size=(4000, 4000), max_pixels=1_000_000) + + +@pytest.mark.parametrize( + "quad", + [[0, 0, 10, 0, 20, 0, 30, 0], [0, 0, 10, 0, 2, 2, 0, 10], [0, 0, 10, 0, 10, float("nan"), 0, 10]], + ids=["collinear", "concave", "nan"], +) +def test_degenerate_quads_are_rejected(gray_page, quad): + with pytest.raises(errors.InvalidArgumentError): + gray_page.rectify(quad) diff --git a/tests/parity/fetch_reference.py b/tests/parity/fetch_reference.py new file mode 100644 index 0000000..b4ef8be --- /dev/null +++ b/tests/parity/fetch_reference.py @@ -0,0 +1,47 @@ +"""Downloads OCRmyPDF's source (pinned, hash-checked) for the raster parity tests. + +The wheel is unpacked, never installed: the tests execute only its pypdfium rasterizer and the +pure-Python Resolution class, against stubs for everything else. + + python tests/parity/fetch_reference.py [cache_dir] # prints the source directory +""" + +from __future__ import annotations + +import hashlib +import sys +import urllib.request +import zipfile +from pathlib import Path + +VERSION = "17.13.0" +URL = ( + "https://files.pythonhosted.org/packages/3f/01/" + "37f376f372901878e6166249bb2c15e943b1627294060be19d6f3ba3b849/" + f"ocrmypdf-{VERSION}-py3-none-any.whl" +) +SHA256 = "f4d226f361c4b1498637f9c674daebfaf84223b84d9eba6bfba60a3ba260f9de" +NEEDED = ("ocrmypdf/builtin_plugins/pypdfium.py", "ocrmypdf/helpers.py") + + +def fetch(cache: Path) -> Path: + source = cache / f"ocrmypdf-{VERSION}" + if all((source / name).is_file() for name in NEEDED): + return source + cache.mkdir(parents=True, exist_ok=True) + wheel = cache / URL.rsplit("/", 1)[1] + with urllib.request.urlopen(URL, timeout=60) as response: # noqa: S310 - pinned https URL + data = response.read() + digest = hashlib.sha256(data).hexdigest() + if digest != SHA256: + raise SystemExit(f"ocrmypdf wheel digest mismatch: {digest}") + wheel.write_bytes(data) + with zipfile.ZipFile(wheel) as archive: + for name in NEEDED: + archive.extract(name, source) + return source + + +if __name__ == "__main__": + target = Path(sys.argv[1]) if len(sys.argv) > 1 else Path.home() / ".cache" / "pyjpdfium-parity" + print(fetch(target)) diff --git a/tests/parity/reference_runner.py b/tests/parity/reference_runner.py new file mode 100644 index 0000000..93446ce --- /dev/null +++ b/tests/parity/reference_runner.py @@ -0,0 +1,74 @@ +"""Runs OCRmyPDF's pypdfium rasterize_pdf_page in its own process and reports the PNG it wrote. + +Usage: python -I reference_runner.py SOURCE_DIR REQUEST_JSON +Only pypdfium.py and the Resolution class from helpers.py execute; the rest of OCRmyPDF is stubbed. +""" + +from __future__ import annotations + +import ast +import importlib.util +import json +import sys +import types +from pathlib import Path + + +def resolution_class(helpers: Path) -> type: + tree = ast.parse(helpers.read_text(encoding="utf-8")) + node = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "Resolution") + header = ( + "from __future__ import annotations\n" + "from collections.abc import Iterable\n" + "from math import isclose, isfinite\n" + "from statistics import harmonic_mean\n" + "from typing import Any, Callable, Generic, TypeVar\n" + "T = TypeVar('T', float, int)\n" + ) + namespace: dict = {} + exec(compile(header + ast.unparse(node), str(helpers), "exec"), namespace) # noqa: S102 + return namespace["Resolution"] + + +def load_reference(source: Path): + resolution = resolution_class(source / "ocrmypdf" / "helpers.py") + package = types.ModuleType("ocrmypdf") + package.hookimpl = lambda func: func + exceptions = types.ModuleType("ocrmypdf.exceptions") + exceptions.MissingDependencyError = type("MissingDependencyError", (Exception,), {}) + helpers = types.ModuleType("ocrmypdf.helpers") + helpers.Resolution = resolution + sys.modules.update({"ocrmypdf": package, "ocrmypdf.exceptions": exceptions, + "ocrmypdf.helpers": helpers}) + path = source / "ocrmypdf" / "builtin_plugins" / "pypdfium.py" + spec = importlib.util.spec_from_file_location("ocrmypdf.builtin_plugins.pypdfium", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, resolution + + +def main() -> None: + source, request = Path(sys.argv[1]), json.loads(sys.argv[2]) + module, resolution = load_reference(source) + page_dpi = request.get("page_dpi") + module.rasterize_pdf_page( + input_file=Path(request["input"]), + output_file=Path(request["output"]), + raster_device=request["device"], + raster_dpi=resolution(*request["raster_dpi"]), + pageno=request["pageno"], + page_dpi=resolution(*page_dpi) if page_dpi else None, + rotation=request.get("rotation", 0), + filter_vector=False, + stop_on_soft_error=True, + options=None, + use_cropbox=request.get("use_cropbox", False), + ) + from PIL import Image + + with Image.open(request["output"]) as image: + print(json.dumps({"size": image.size, "mode": image.mode, "dpi": image.info.get("dpi")})) + + +if __name__ == "__main__": + main() diff --git a/tests/parity/requirements.txt b/tests/parity/requirements.txt new file mode 100644 index 0000000..2cccc21 --- /dev/null +++ b/tests/parity/requirements.txt @@ -0,0 +1,6 @@ +# Reference stack for raster parity tests (Linux x86_64, CPython 3.12). Install with +# pip install --require-hashes --only-binary=:all: -r tests/parity/requirements.txt +pillow==12.3.0 \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 +pypdfium2==5.14.0 \ + --hash=sha256:11f281613fa22313d9c7ab89947665e84eccf8ebe40e1198a84a88352305648d diff --git a/tests/parity/test_raster_parity.py b/tests/parity/test_raster_parity.py new file mode 100644 index 0000000..ebe5f71 --- /dev/null +++ b/tests/parity/test_raster_parity.py @@ -0,0 +1,133 @@ +"""Raster parity with OCRmyPDF's pypdfium rasterizer, the reference for jpdf_rasterize_page_file. + +Opt-in: set PYJPDFIUM_PARITY_SRC to the directory printed by fetch_reference.py and install +requirements.txt. The reference runs in a separate process so its PDFium never shares ours. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import pyjpdfium + +SOURCE = os.environ.get("PYJPDFIUM_PARITY_SRC") +RUNNER = Path(__file__).with_name("reference_runner.py") +Image = pytest.importorskip("PIL.Image") if SOURCE else None +pytestmark = [ + pytest.mark.native, + pytest.mark.skipif(not SOURCE, reason="PYJPDFIUM_PARITY_SRC is not set"), +] + +PAGES = { + "square": ("/MediaBox [0 0 612 792]", "0 0 0 rg 72 600 100 100 re f 0.4 g 300 200 150 80 re f"), + "text": ( + "/MediaBox [0 0 612 792] /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 " + "/BaseFont /Helvetica >> >> >>", + "BT /F1 18 Tf 72 700 Td (The quick brown fox jumps over the lazy dog.) Tj " + "0 -40 Td /F1 9 Tf (Small print 0123456789 still has to survive OCR.) Tj ET", + ), + "ramp": ( + "/MediaBox [0 0 512 792]", + "".join(f"{g / 255:.4f} g {g * 2} 0 2 792 re f " for g in range(256)), + ), +} + + +def write_pdf(path: Path, page_extra: str, content: str) -> Path: + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + f"<< /Type /Page /Parent 2 0 R {page_extra} /Contents 4 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + ] + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n{body}\nendobj\n".encode() + xref = len(out) + out += b"xref\n0 5\n0000000000 65535 f \n" + out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets) + out += f"trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + path.write_bytes(bytes(out)) + return path + + +def reference(pdf: Path, out: Path, device: str, dpi, rotation=0, use_cropbox=False) -> dict: + request = {"input": str(pdf), "output": str(out), "device": device, "raster_dpi": list(dpi), + "pageno": 1, "rotation": rotation, "use_cropbox": use_cropbox} + env = {k: v for k, v in os.environ.items() if not k.startswith("PYJPDFIUM")} + done = subprocess.run([sys.executable, "-I", str(RUNNER), SOURCE, json.dumps(request)], + capture_output=True, text=True, env=env, check=False) + assert done.returncode == 0, done.stderr + return json.loads(done.stdout.strip().splitlines()[-1]) + + +def difference(a: Path, b: Path) -> tuple[float, float]: + """Mean absolute difference (0-255) and the fraction of differing samples.""" + from PIL import ImageChops + + with Image.open(a) as x, Image.open(b) as y: + x, y = x.convert("L" if x.mode in ("1", "L") else "RGB"), y.convert( + "L" if y.mode in ("1", "L") else "RGB") + histogram = ImageChops.difference(x, y).histogram() + total = sum(histogram) + bands = len(histogram) // 256 + per_value = [sum(histogram[v + 256 * b] for b in range(bands)) for v in range(256)] + return sum(v * n for v, n in enumerate(per_value)) / total, 1 - per_value[0] / total + + +@pytest.mark.parametrize("page", sorted(PAGES)) +@pytest.mark.parametrize("device", ["pnggray", "png16m", "pngmonod"]) +@pytest.mark.parametrize("dpi", [72, 150, 300]) +@pytest.mark.parametrize("rotation", [0, 90]) +def test_matches_reference(tmp_path, page, device, dpi, rotation): + pdf = write_pdf(tmp_path / "in.pdf", *PAGES[page]) + ref = reference(pdf, tmp_path / "ref.png", device, (dpi, dpi), rotation) + ours = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "ours.png", device=device, + raster_dpi=dpi, rotation=rotation) + with Image.open(tmp_path / "ours.png") as image: + assert list(image.size) == ref["size"] + assert image.mode == ref["mode"] + assert [round(v, 3) for v in image.info["dpi"]] == [round(v, 3) for v in ref["dpi"]] + assert (ours.width, ours.height) == tuple(ref["size"]) + mean, changed = difference(tmp_path / "ours.png", tmp_path / "ref.png") + # Measured against pypdfium2 5.14.0: identical except RGB text (0.27% of samples, mean 0.003), + # where the two PDFium builds anti-alias glyph edges slightly differently. + assert mean < 0.05 and changed < 0.01, f"mean {mean:.4f}, changed {changed:.5f}" + + +def test_deviation_partial_cropbox(tmp_path): + # Reference: expected size comes from the CropBox, one axis is within 2 px, so the MediaBox + # render is cropped from its top-left corner. Ours renders the whole MediaBox. + pdf = write_pdf(tmp_path / "in.pdf", "/MediaBox [0 0 612 792] /CropBox [0 100 612 500]", + "0 g 0 0 612 792 re f") + ref = reference(pdf, tmp_path / "ref.png", "pnggray", (72, 72)) + ours = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "o.png", device="pnggray", raster_dpi=72) + assert ref["size"] == [612, 400] + assert (ours.width, ours.height) == (612, 792) + + +def test_deviation_non_square_dpi(tmp_path): + # Reference renders both axes at the harmonic mean; ours honours each axis like Ghostscript. + pdf = write_pdf(tmp_path / "in.pdf", *PAGES["square"]) + ref = reference(pdf, tmp_path / "ref.png", "pnggray", (300, 150)) + ours = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "o.png", device="pnggray", + raster_dpi=(300, 150)) + assert ref["size"] == [1700, 2200] + assert (ours.width, ours.height) == (2550, 1650) + + +def test_deviation_rotation_on_rotated_page(tmp_path): + # Reference overwrites /Rotate; ours adds the correction to it, as Ghostscript does. + pdf = write_pdf(tmp_path / "in.pdf", "/MediaBox [0 0 612 792] /Rotate 90", "0 g 72 600 100 100 re f") + ref = reference(pdf, tmp_path / "ref.png", "pnggray", (72, 72), rotation=90) + ours = pyjpdfium.rasterize_page(pdf, 0, tmp_path / "o.png", device="pnggray", raster_dpi=72, + rotation=90) + assert ref["size"] == [792, 612] + assert (ours.width, ours.height) == (612, 792) diff --git a/tests/pngread.py b/tests/pngread.py new file mode 100644 index 0000000..71d3f74 --- /dev/null +++ b/tests/pngread.py @@ -0,0 +1,75 @@ +"""Minimal PNG decoder for tests: non-interlaced 1-bit or 8-bit gray, RGB and RGBA.""" + +from __future__ import annotations + +import struct +import zlib +from dataclasses import dataclass +from pathlib import Path + +_CHANNELS = {0: 1, 2: 3, 6: 4} + + +@dataclass +class Png: + width: int + height: int + bit_depth: int + color_type: int + ppm: tuple[int, int] | None + rows: list[bytes] + + @property + def channels(self) -> int: + return _CHANNELS[self.color_type] + + def pixel(self, x: int, y: int) -> tuple[int, ...]: + row = self.rows[y] + if self.bit_depth == 1: + return ((row[x >> 3] >> (7 - (x & 7))) & 1,) + c = self.channels + return tuple(row[x * c : x * c + c]) + + def gray_bytes(self) -> bytes: + """1-bit pixels expanded to 0/255, for comparison with Pillow's "1" mode.""" + return bytes(255 * self.pixel(x, y)[0] for y in range(self.height) for x in range(self.width)) + + +def _paeth(a: int, b: int, c: int) -> int: + p = a + b - c + pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + if pa <= pb and pa <= pc: + return a + return b if pb <= pc else c + + +def read_png(path: Path) -> Png: + data = Path(path).read_bytes() + assert data[:8] == b"\x89PNG\r\n\x1a\n" + pos, idat, ppm = 8, bytearray(), None + while pos < len(data): + (length,) = struct.unpack(">I", data[pos : pos + 4]) + kind, body = data[pos + 4 : pos + 8], data[pos + 8 : pos + 8 + length] + if kind == b"IHDR": + width, height, depth, color = struct.unpack(">IIBB", body[:10]) + elif kind == b"pHYs": + ppm = struct.unpack(">II", body[:8]) + elif kind == b"IDAT": + idat += body + pos += 12 + length + channels = _CHANNELS[color] + row_len = (width + 7) // 8 if depth == 1 else width * channels + bpp = 1 if depth == 1 else channels + raw, rows, prev = zlib.decompress(bytes(idat)), [], bytes(row_len) + for y in range(height): + start = y * (row_len + 1) + kind, line = raw[start], bytearray(raw[start + 1 : start + 1 + row_len]) + for i in range(row_len): + a = line[i - bpp] if i >= bpp else 0 + b = prev[i] + c = prev[i - bpp] if i >= bpp else 0 + predictor = (0, a, b, (a + b) // 2, _paeth(a, b, c))[kind] + line[i] = (line[i] + predictor) & 0xFF + rows.append(bytes(line)) + prev = line + return Png(width, height, depth, color, ppm, rows) From 79c9c84cece5e6d8a8df57593b54422bea2f509b Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:17:14 +0100 Subject: [PATCH 04/10] Reword the in-process note --- docs/ARCHITECTURE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8927446..8c7821b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -49,5 +49,5 @@ pixels, so it stays valid after the bitmap, page or document is closed. ## Isolation -The native core runs in-process. It removes the JVM, not the crash domain: a native fault ends the -interpreter. Untrusted or hard-budget jobs should run in a worker process. +The native core runs in-process, so it shares the interpreter's process. Jobs that need their own +memory or time limits can run in a worker process. From 6ceaf5dab613eab07d361f6a5259ac016181722a Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:14:48 +0100 Subject: [PATCH 05/10] Add text layer writing and page text with OCRmyPDF graft parity tests --- README.md | 18 ++- docs/ARCHITECTURE.md | 2 +- docs/COMPATIBILITY.md | 9 +- src/native/pyjpdfium_native.cpp | 62 ++++++++ src/pyjpdfium/__init__.py | 3 + src/pyjpdfium/models.py | 19 +++ src/pyjpdfium/pdf.py | 4 + src/pyjpdfium/textlayer.py | 65 ++++++++ tests/native/test_text_layer.py | 106 +++++++++++++ tests/parity/fetch_reference.py | 6 +- tests/parity/requirements.txt | 13 +- tests/parity/test_text_layer_parity.py | 204 +++++++++++++++++++++++++ tests/parity/text_layer_runner.py | 164 ++++++++++++++++++++ tests/unit/test_lazy_import.py | 1 + tests/unit/test_text_layer_models.py | 22 +++ 15 files changed, 687 insertions(+), 11 deletions(-) create mode 100644 src/pyjpdfium/textlayer.py create mode 100644 tests/native/test_text_layer.py create mode 100644 tests/parity/test_text_layer_parity.py create mode 100644 tests/parity/text_layer_runner.py create mode 100644 tests/unit/test_text_layer_models.py diff --git a/README.md b/README.md index 03e580c..7edda1b 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ PyJPDFium is a thin [nanobind](https://github.com/wjakob/nanobind) binding over `jpdf_*` C API. The native core, not Python or the GIL, owns PDFium serialization, object lifetimes and handle validation, so the same rules hold for the Java and Python bindings. -Status: early development. Covered so far: documents, pages, rendering, transactional PNG output, -OCRmyPDF-compatible page rasterization and perspective rectification. +Status: early development. Covered so far: documents, pages, rendering, page text, transactional +PNG output, OCRmyPDF-compatible page rasterization, perspective rectification and invisible text +layers. ```python import pyjpdfium @@ -22,6 +23,13 @@ with pyjpdfium.open_document("input.pdf") as document: # OCRmyPDF's rasterize_pdf_page contract, natively (see JPDFium docs/design/RASTER.md). pyjpdfium.rasterize_page("input.pdf", 0, "page.png", device="pnggray", raster_dpi=300) + +# The text-only PDF of OCRmyPDF's generate_pdf hook (see JPDFium docs/design/TEXT_LAYER.md). +from pyjpdfium import TextLine, TextWord + +line = TextLine((300, 300, 1500, 400), (TextWord("Hello", (300, 300, 800, 400)), + TextWord("world", (900, 300, 1500, 400)))) +pyjpdfium.write_text_layer("text.pdf", [line], image_size=(2550, 3300), dpi=300) ``` ## Behaviour @@ -54,8 +62,8 @@ PYJPDFIUM_NATIVE_DIR=$JPDFIUM_NATIVE_DIR pytest `PYJPDFIUM_RENDERER` selects `auto` (default), `agg` or `skia`. Native tests (`-m native`) need the real core and are skipped when the extension is not built. -Raster parity with OCRmyPDF's own rasterizer is opt-in: install `tests/parity/requirements.txt` -(hash-pinned), set `PYJPDFIUM_PARITY_SRC` to the output of `tests/parity/fetch_reference.py`, and -run `pytest tests/parity`. +Parity with OCRmyPDF's own rasterizer and sandwich graft is opt-in: install +`tests/parity/requirements.txt` (hash-pinned), set `PYJPDFIUM_PARITY_SRC` to the output of +`tests/parity/fetch_reference.py`, and run `pytest tests/parity`. See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8c7821b..6bd7f22 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -17,7 +17,7 @@ The full contracts live in JPDFium's `docs/design/` (`ARCHITECTURE.md`, `NATIVE_ ## Layers ```text -pyjpdfium pure Python API: Document, Page, Bitmap, runtime_* +pyjpdfium pure Python API: Document, Page, Bitmap, write_text_layer, runtime_* pyjpdfium._loader lazy load, core pinning, identity checks, runtime init pyjpdfium._native nanobind module: one function per jpdf_* call, GIL released around it libjpdfium the shared native core (never compiled into the extension) diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 3687f90..879ee2b 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -16,9 +16,16 @@ the two disagree. It produces PNG only: `jpeg`, `jpeggray` and `tiff` devices ar `png256` is written as RGB. JPDFium's `docs/design/RASTER.md` lists every deviation and the measured pixel parity. +## Text layers + +`write_text_layer` writes the text-only PDF OCRmyPDF's `generate_pdf` hook returns, laid out as +OCRmyPDF 17.13's renderer lays out invisible text. OCRmyPDF's own sandwich graft accepts it, and +pypdfium2, qpdf and fontTools read it back (JPDFium's `docs/design/TEXT_LAYER.md`). Right-to-left +scripts raise `UnsupportedError` until the multilingual writer lands. + ## Not in this release -- OCR, text-layer writing and the OCRmyPDF plugin. +- OCR and the OCRmyPDF plugin. - PDF/A output: rejected by design. - Opening from bytes or file-like objects, saving or editing documents. - Free-threaded CPython builds are untested. diff --git a/src/native/pyjpdfium_native.cpp b/src/native/pyjpdfium_native.cpp index e659e70..d950b4c 100644 --- a/src/native/pyjpdfium_native.cpp +++ b/src/native/pyjpdfium_native.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -88,6 +89,9 @@ bool struct_sizes_match() { {JPDF_STRUCT_RASTER_RESULT, sizeof(jpdf_raster_result)}, {JPDF_STRUCT_RECTIFY_OPTIONS, sizeof(jpdf_rectify_options)}, {JPDF_STRUCT_RECTIFY_RESULT, sizeof(jpdf_rectify_result)}, + {JPDF_STRUCT_TEXT_WORD, sizeof(jpdf_text_word)}, + {JPDF_STRUCT_TEXT_LINE, sizeof(jpdf_text_line)}, + {JPDF_STRUCT_TEXT_LAYER_OPTIONS, sizeof(jpdf_text_layer_options)}, }; for (const auto& pair : expected) { if (jpdf_abi_struct_size(static_cast(pair[0])) != pair[1]) return false; @@ -131,6 +135,58 @@ nb::bytes bitmap_copy(jpdf_handle bitmap) { return out; } +std::string page_text(jpdf_handle page) { + uint64_t needed = 0; + run([&] { return jpdf_page_get_text(page, nullptr, 0, &needed); }); + std::string text(static_cast(needed), '\0'); + run([&] { return jpdf_page_get_text(page, text.data(), needed, &needed); }); + text.resize(text.empty() ? 0 : text.size() - 1); + return text; +} + +using Box = std::tuple; + +// Owned copies are built with the GIL held; the native call then only reads them. +void text_layer_write(const std::string& output, int32_t image_width, int32_t image_height, + double dpi_x, double dpi_y, + const std::vector>& lines, + const std::vector>& words, bool durable, + uint64_t max_bytes) { + jpdf_text_layer_options o{}; + o.struct_size = sizeof(o); + o.image_width = image_width; + o.image_height = image_height; + o.dpi_x = dpi_x; + o.dpi_y = dpi_y; + o.font = JPDF_FONT_GLYPHLESS; + std::vector native_lines(lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + jpdf_text_line& l = native_lines[i]; + l.struct_size = sizeof(l); + std::tie(l.x0, l.y0, l.x1, l.y1) = std::get<0>(lines[i]); + l.word_count = std::get<1>(lines[i]); + } + std::vector native_words(words.size()); + for (size_t i = 0; i < words.size(); ++i) { + jpdf_text_word& w = native_words[i]; + w.struct_size = sizeof(w); + w.text_utf8 = std::get<0>(words[i]).c_str(); + std::tie(w.x0, w.y0, w.x1, w.y1) = std::get<1>(words[i]); + } + if (native_lines.size() > INT32_MAX || native_words.size() > INT32_MAX) { + throw nb::value_error("too many lines or words"); + } + jpdf_write_options w{}; + w.struct_size = sizeof(w); + w.flags = durable ? JPDF_WRITE_DURABLE : 0u; + w.max_bytes = max_bytes; + run([&] { + return jpdf_text_layer_write(&o, native_lines.data(), + static_cast(native_lines.size()), native_words.data(), + static_cast(native_words.size()), output.c_str(), &w); + }); +} + using PixelArray = nb::ndarray; size_t channels_for(int32_t pixel_format) { @@ -240,6 +296,8 @@ NB_MODULE(_native, m) { return std::make_tuple(info.index, info.width_pt, info.height_pt, info.rotation); }); + m.def("page_text", &page_text); + m.def("page_render", [](jpdf_handle page, uint32_t flags, double dpi_x, double dpi_y, int32_t width, int32_t height, int32_t pixel_format, uint64_t max_pixels) { @@ -294,6 +352,10 @@ NB_MODULE(_native, m) { std::vector(r.to_source, r.to_source + 9)); }); + m.def("text_layer_write", &text_layer_write, nb::arg("output"), nb::arg("image_width"), + nb::arg("image_height"), nb::arg("dpi_x"), nb::arg("dpi_y"), nb::arg("lines"), + nb::arg("words"), nb::arg("durable"), nb::arg("max_bytes")); + m.def("rasterize_page_file", [](const std::string& path, int32_t page_index, int32_t device, double dpi_x, double dpi_y, double out_dpi_x, double out_dpi_y, int32_t rotation_ccw, diff --git a/src/pyjpdfium/__init__.py b/src/pyjpdfium/__init__.py index c5cc7f8..6ec0353 100644 --- a/src/pyjpdfium/__init__.py +++ b/src/pyjpdfium/__init__.py @@ -17,6 +17,7 @@ "Page": "pyjpdfium.pdf", "Bitmap": "pyjpdfium.pdf", "rasterize_page": "pyjpdfium.raster", + "write_text_layer": "pyjpdfium.textlayer", "runtime_info": "pyjpdfium.runtime", "capability": "pyjpdfium.runtime", "quiesce": "pyjpdfium.runtime", @@ -34,6 +35,8 @@ "Interpolation": "pyjpdfium.models", "Border": "pyjpdfium.models", "RectifyResult": "pyjpdfium.models", + "TextWord": "pyjpdfium.models", + "TextLine": "pyjpdfium.models", "PyJPDFiumError": "pyjpdfium.errors", "NativeLoadError": "pyjpdfium.errors", "InvalidArgumentError": "pyjpdfium.errors", diff --git a/src/pyjpdfium/models.py b/src/pyjpdfium/models.py index dbc9613..a7d8fec 100644 --- a/src/pyjpdfium/models.py +++ b/src/pyjpdfium/models.py @@ -112,6 +112,25 @@ class RasterResult: dpi: tuple[float, float] +Box = tuple[float, float, float, float] + + +@dataclass(frozen=True) +class TextWord: + """A word and its box ``(x0, y0, x1, y1)`` in image pixels, origin top-left.""" + + text: str + box: Box + + +@dataclass(frozen=True) +class TextLine: + """Words in reading order. The line box height sets the font size; it must be positive.""" + + box: Box + words: tuple[TextWord, ...] + + @dataclass(frozen=True) class RectifyResult: """Corners as used (top-left, top-right, bottom-right, bottom-left) and the inverse map.""" diff --git a/src/pyjpdfium/pdf.py b/src/pyjpdfium/pdf.py index 7b88141..c97fd26 100644 --- a/src/pyjpdfium/pdf.py +++ b/src/pyjpdfium/pdf.py @@ -180,6 +180,10 @@ def render( ) return None + def text(self) -> str: + """The page text in PDFium's extraction order, with its generated spaces and line breaks.""" + return native().page_text(self._require_open()) + def __repr__(self) -> str: state = "closed" if self.closed else "open" return f"" diff --git a/src/pyjpdfium/textlayer.py b/src/pyjpdfium/textlayer.py new file mode 100644 index 0000000..5065975 --- /dev/null +++ b/src/pyjpdfium/textlayer.py @@ -0,0 +1,65 @@ +"""Invisible text layers, the text-only PDF of OCRmyPDF's generate_pdf hook, written natively.""" + +from __future__ import annotations + +import os +from collections.abc import Iterable +from typing import Union + +from ._loader import native +from .errors import InvalidArgumentError +from .models import Box, TextLine + +StrPath = Union[str, "os.PathLike[str]"] +Dpi = Union[float, "tuple[float, float]"] + + +def _box(box: Box) -> Box: + x0, y0, x1, y1 = box + return float(x0), float(y0), float(x1), float(y1) + + +def _text(text: str) -> str: + # The native side takes NUL-terminated UTF-8, so these could not reach it intact. + if "\x00" in text: + raise InvalidArgumentError("word text contains a NUL character") + try: + text.encode("utf-8") + except UnicodeEncodeError: + raise InvalidArgumentError("word text is not valid Unicode") from None + return text + + +def write_text_layer( + output: StrPath, + lines: Iterable[TextLine], + *, + image_size: tuple[int, int], + dpi: Dpi, + durable: bool = False, + max_bytes: int | None = None, +) -> None: + """Publishes a one-page PDF sized to the image, holding only invisible, extractable text. + + Each word is stretched to its box and gets a trailing space, except between CJK words, as + OCRmyPDF's renderer does. Right-to-left text raises UnsupportedError for now. On any failure + ``output`` is left untouched. + """ + native_lines: list[tuple[Box, int]] = [] + native_words: list[tuple[str, Box]] = [] + for line in lines: + native_lines.append((_box(line.box), len(line.words))) + native_words.extend((_text(word.text), _box(word.box)) for word in line.words) + width, height = image_size + dpi_x, dpi_y = (float(dpi), float(dpi)) if isinstance(dpi, (int, float)) else map(float, dpi) + native().text_layer_write( + os.fsdecode(os.fspath(output)), + int(width), + int(height), + dpi_x, + dpi_y, + native_lines, + native_words, + durable, + max_bytes or 0, + ) diff --git a/tests/native/test_text_layer.py b/tests/native/test_text_layer.py new file mode 100644 index 0000000..22a1eee --- /dev/null +++ b/tests/native/test_text_layer.py @@ -0,0 +1,106 @@ +import threading + +import pytest + +import pyjpdfium +from pyjpdfium import TextLine, TextWord, errors + +pytestmark = pytest.mark.native + +LINES = [ + TextLine((300, 300, 1500, 400), (TextWord("Hello", (300, 300, 800, 400)), + TextWord("world", (900, 300, 1500, 400)))), + TextLine((300, 500, 1200, 600), (TextWord("second", (300, 500, 900, 600)), + TextWord("line", (950, 500, 1200, 600)))), +] +LETTER = {"image_size": (2550, 3300), "dpi": 300} + + +def squash(text): + return " ".join(text.split()) + + +def write(path, lines=LINES, **kwargs): + pyjpdfium.write_text_layer(path, lines, **{**LETTER, **kwargs}) + return path + + +def test_round_trip_through_our_own_reader(tmp_path): + path = write(tmp_path / "layer.pdf") + with pyjpdfium.open_document(path) as document, document.page(0) as page: + assert page.size.width == pytest.approx(612) and page.size.height == pytest.approx(792) + assert squash(page.text()) == "Hello world second line" + with page.render(dpi=100, pixel_format="gray") as bitmap: + assert min(bitmap.tobytes()) == 255, "the layer must not draw anything" + + +def test_empty_layer_is_a_blank_page(tmp_path): + path = write(tmp_path / "empty.pdf", lines=[]) + with pyjpdfium.open_document(path) as document, document.page(0) as page: + assert page.text() == "" + + +def test_writing_never_enters_the_pdfium_domain(tmp_path): + before = pyjpdfium.runtime_info().domain_acquisitions + for i in range(20): + write(tmp_path / f"layer{i}.pdf") + assert pyjpdfium.runtime_info().domain_acquisitions == before + + +def test_concurrent_writers_produce_identical_files(tmp_path): + failures = [] + + def worker(n): + try: + write(tmp_path / f"t{n}.pdf") + except Exception as exc: # noqa: BLE001 - reported below + failures.append(exc) + + threads = [threading.Thread(target=worker, args=(n,)) for n in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not failures + first = (tmp_path / "t0.pdf").read_bytes() + assert all((tmp_path / f"t{n}.pdf").read_bytes() == first for n in range(8)) + + +def test_failed_write_keeps_the_destination(tmp_path): + path = tmp_path / "kept.pdf" + path.write_bytes(b"previous") + with pytest.raises(errors.LimitExceededError): + write(path, max_bytes=64) + assert path.read_bytes() == b"previous" + + +def one_word(text, box=(0, 0, 100, 50)): + return [TextLine(box, (TextWord(text, box),))] + + +@pytest.mark.parametrize( + ("lines", "error"), + [ + (one_word("שלום"), errors.UnsupportedError), # right-to-left is not laid out yet + (one_word("a\tb"), errors.InvalidArgumentError), + (one_word("a\x00b"), errors.InvalidArgumentError), + (one_word("\ud800"), errors.InvalidArgumentError), + (one_word(""), errors.InvalidArgumentError), + (one_word("a", box=(0, 50, 100, 50)), errors.InvalidArgumentError), + ([TextLine((0, 0, 100, 50), ())], errors.InvalidArgumentError), + ], +) +def test_rejections(tmp_path, lines, error): + path = tmp_path / "rejected.pdf" + with pytest.raises(error): + write(path, lines=lines) + assert not path.exists() + + +def test_page_text_needs_an_open_page(tmp_path): + path = write(tmp_path / "layer.pdf") + with pyjpdfium.open_document(path) as document: + page = document.page(0) + page.close() + with pytest.raises(errors.StaleHandleError): + page.text() diff --git a/tests/parity/fetch_reference.py b/tests/parity/fetch_reference.py index b4ef8be..4bd59ab 100644 --- a/tests/parity/fetch_reference.py +++ b/tests/parity/fetch_reference.py @@ -1,7 +1,7 @@ """Downloads OCRmyPDF's source (pinned, hash-checked) for the raster parity tests. -The wheel is unpacked, never installed: the tests execute only its pypdfium rasterizer and the -pure-Python Resolution class, against stubs for everything else. +The wheel is unpacked, never installed: the tests execute only its pypdfium rasterizer, the +pure-Python Resolution class and its sandwich graft, against stubs for everything else. python tests/parity/fetch_reference.py [cache_dir] # prints the source directory """ @@ -21,7 +21,7 @@ f"ocrmypdf-{VERSION}-py3-none-any.whl" ) SHA256 = "f4d226f361c4b1498637f9c674daebfaf84223b84d9eba6bfba60a3ba260f9de" -NEEDED = ("ocrmypdf/builtin_plugins/pypdfium.py", "ocrmypdf/helpers.py") +NEEDED = ("ocrmypdf/builtin_plugins/pypdfium.py", "ocrmypdf/helpers.py", "ocrmypdf/_graft.py") def fetch(cache: Path) -> Path: diff --git a/tests/parity/requirements.txt b/tests/parity/requirements.txt index 2cccc21..bd5a200 100644 --- a/tests/parity/requirements.txt +++ b/tests/parity/requirements.txt @@ -1,6 +1,17 @@ -# Reference stack for raster parity tests (Linux x86_64, CPython 3.12). Install with +# Reference stack for the parity tests (Linux x86_64, CPython 3.12). Install with # pip install --require-hashes --only-binary=:all: -r tests/parity/requirements.txt pillow==12.3.0 \ --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 pypdfium2==5.14.0 \ --hash=sha256:11f281613fa22313d9c7ab89947665e84eccf8ebe40e1198a84a88352305648d +# Text layers: OCRmyPDF 17.13's own graft needs pikepdf >= 10.15; fontTools parses the font. +pikepdf==10.15.0 \ + --hash=sha256:203053af0efa8777ebed505f7cc4677e66c0068cf25e67144a782c0e360f18b4 +lxml==6.1.2 \ + --hash=sha256:9477e14217c212e6023c994a71a1a349db19b0e10fd5bf189666b281ae63b1fd \ + --hash=sha256:04cf9e3f4ee9cab9d9ba05401bef8668840fa9620fcd4d8e85a2d2fd0b0fa960 +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e +fonttools==4.64.0 \ + --hash=sha256:66a83f93579fb3493e458c4449d1d566a7b2a1c7b19915cd0fa3c9b8b5a8540b \ + --hash=sha256:4a05783ff54ce4c7a28f18e5772efdf63c219374bd9ffc55452182e1cef8be60 diff --git a/tests/parity/test_text_layer_parity.py b/tests/parity/test_text_layer_parity.py new file mode 100644 index 0000000..6f814c8 --- /dev/null +++ b/tests/parity/test_text_layer_parity.py @@ -0,0 +1,204 @@ +"""Text layers checked by tools independent of JPDFium, and grafted by OCRmyPDF's sandwich code. + +Opt-in like the raster parity tests: set PYJPDFIUM_PARITY_SRC and install requirements.txt. Each +tool runs in its own process, so pypdfium2's PDFium never shares ours. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import pyjpdfium +from pyjpdfium import TextLine, TextWord + +SOURCE = os.environ.get("PYJPDFIUM_PARITY_SRC") +RUNNER = Path(__file__).with_name("text_layer_runner.py") +pytestmark = [ + pytest.mark.native, + pytest.mark.skipif(not SOURCE, reason="PYJPDFIUM_PARITY_SRC is not set"), +] + +DPI = 300.0 +SCALE = 72.0 / DPI +IMAGE = (2550, 3300) # US Letter at 300 dpi +LINES = [ + TextLine((150, 300, 1650, 400), ( + TextWord("The", (150, 300, 420, 400)), + TextWord("quick", (470, 300, 860, 400)), + TextWord("brown", (910, 300, 1330, 400)), + TextWord("fox", (1380, 300, 1650, 400)), + )), + TextLine((150, 500, 1500, 580), ( + TextWord("naïve", (150, 500, 560, 580)), + TextWord("café", (610, 500, 960, 580)), + TextWord("Ünïcödé", (1010, 500, 1500, 580)), + )), + TextLine((150, 700, 900, 800), ( + TextWord("日本", (150, 700, 350, 800)), + TextWord("語", (360, 700, 460, 800)), + TextWord("テキスト", (470, 700, 900, 800)), + )), + # A lone one-glyph word: newer PDFium drops text whose glyph box is empty. + TextLine((150, 900, 250, 1000), (TextWord("字", (150, 900, 250, 1000)),)), +] +EXPECTED = "The quick brown fox naïve café Ünïcödé 日本語テキスト 字" +TERMS = ["quick brown", "café Ünïcödé", "日本語", "fox naïve"] + + +def run(*args: str) -> dict: + env = {k: v for k, v in os.environ.items() if not k.startswith("PYJPDFIUM")} + done = subprocess.run([sys.executable, "-I", str(RUNNER), *args], capture_output=True, + text=True, env=env, check=False) + assert done.returncode == 0, done.stderr + return json.loads(done.stdout.strip().splitlines()[-1]) + + +def squash(text: str) -> str: + return " ".join(text.split()) + + +def write_pdf(path: Path, page_extra: str, content: str) -> Path: + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + f"<< /Type /Page /Parent 2 0 R {page_extra} /Contents 4 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + ] + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n{body}\nendobj\n".encode() + xref = len(out) + out += b"xref\n0 5\n0000000000 65535 f \n" + out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets) + out += f"trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + path.write_bytes(bytes(out)) + return path + + +def is_cjk(word: TextWord) -> bool: + return all(0x3000 <= ord(c) <= 0x9FFF for c in word.text) + + +def expected_boxes(lines, to_page): + """Each non-space character's word box, in page points; CJK runs stretch to the next word.""" + boxes = [] + for line in lines: + for i, word in enumerate(line.words): + x0, _, x1, _ = word.box + following = line.words[i + 1] if i + 1 < len(line.words) else None + if following is not None and is_cjk(word) and is_cjk(following): + x1 = following.box[0] + box = to_page(x0, line.box[1], x1, line.box[3]) + boxes.extend([(c, box) for c in word.text]) + return boxes + + +def upright(x0, y0, x1, y1, height=IMAGE[1]): + return x0 * SCALE, (height - y1) * SCALE, x1 * SCALE, (height - y0) * SCALE + + +# PDFium rounds glyph boxes: tops come out about 0.012 em high in pypdfium2's build. +EM_SLACK = 0.02 + + +def check_placement(chars, expected, tolerance=0.05): + """Characters sit inside their word box, and each word's characters span it edge to edge.""" + found = [c for c in chars if not c[1] and not c[0].isspace()] + assert [c[0] for c in found] == [c for c, _ in expected] + spans = {} + for (char, _, left, bottom, right, top), (_, (bl, bb, br, bt)) in zip(found, expected): + slack = EM_SLACK * (bt - bb) + assert bl - tolerance <= left <= right <= br + tolerance, char + assert abs(bottom - bb) <= slack and abs(top - bt) <= slack, char + span = spans.setdefault((bl, bb, br, bt), [left, right]) + span[0], span[1] = min(span[0], left), max(span[1], right) + for (bl, _, br, _), (left, right) in spans.items(): + assert abs(left - bl) <= tolerance and abs(right - br) <= tolerance + + +@pytest.fixture +def layer(tmp_path: Path) -> Path: + path = tmp_path / "layer.pdf" + pyjpdfium.write_text_layer(path, LINES, image_size=IMAGE, dpi=DPI) + return path + + +def test_independent_pdfium_reads_text_search_and_placement(layer): + result = run("probe", str(layer), *TERMS) + assert result["size"] == pytest.approx([612, 792], abs=1e-3) + assert squash(result["text"]) == EXPECTED + # Word spaces are real characters, so extractors without layout analysis keep them too: + # three between words and one closing each of the two Latin lines. + assert sum(1 for c in result["chars"] if c[0] == " " and not c[1]) == 7 + assert result["search"] == {term: 1 for term in TERMS} + assert result["blank"], "the layer must not draw anything" + check_placement(result["chars"], expected_boxes(LINES, upright)) + + +def test_qpdf_and_fonttools_accept_the_file(layer): + result = run("font", str(layer)) + assert result["problems"] == [] + assert result["whole_font_checksum_ok"] + assert result["contours"] == [2, 2], "blank point contours, as in OCRmyPDF's Occulta" + assert result["units_per_em"] == 1000 + assert (result["ascent"], result["descent"]) == (800, -200) + assert {"cmap", "glyf", "head", "hhea", "hmtx", "loca", "maxp", "name", "post"} <= set( + result["tables"]) + + +def base_page(tmp_path: Path, rotate: int = 0) -> Path: + # Visible artwork only, so all extracted text comes from the grafted layer. + content = "0 0 0 rg 72 600 100 100 re f 0.5 g 300 200 150 80 re f" + extra = "/MediaBox [0 0 612 792]" + (f" /Rotate {rotate}" if rotate else "") + return write_pdf(tmp_path / f"base{rotate}.pdf", extra, content) + + +def graft(tmp_path: Path, base: Path, text: Path, **extra) -> Path: + output = tmp_path / f"grafted-{base.stem}.pdf" + request = {"base": str(base), "text": str(text), "output": str(output), **extra} + assert run("graft", SOURCE, json.dumps(request))["grafted"] + return output + + +def test_ocrmypdf_sandwich_graft_adds_text_and_nothing_visible(tmp_path, layer): + base = base_page(tmp_path) + stripped = tmp_path / "stripped.pdf" + grafted = graft(tmp_path, base, layer, stripped=str(stripped)) + before, after = run("probe", str(base)), run("probe", str(grafted), *TERMS) + assert after["render"] == before["render"], "grafting changed what the page shows" + assert squash(after["text"]) == EXPECTED + assert after["search"] == {term: 1 for term in TERMS} + check_placement(after["chars"], expected_boxes(LINES, upright)) + # OCRmyPDF's --redo-ocr strips render-mode-3 text; ours must come out cleanly. + cleaned = run("probe", str(stripped)) + assert squash(cleaned["text"]) == "" + assert cleaned["render"] == before["render"] + + +def test_graft_onto_a_rotated_page_follows_the_displayed_image(tmp_path): + # OCRmyPDF rasterizes /Rotate 90 pages as displayed, so the image is landscape. + layer = tmp_path / "landscape.pdf" + pyjpdfium.write_text_layer(layer, LINES, image_size=(IMAGE[1], IMAGE[0]), dpi=DPI) + grafted = graft(tmp_path, base_page(tmp_path, rotate=90), layer) + result = run("probe", str(grafted), *TERMS) + assert result["rotation"] == 90 + assert squash(result["text"]) == EXPECTED + + def rotated(x0, y0, x1, y1): + # Displayed top-left (x, y) is unrotated page point (u = y, v = x) under /Rotate 90. + return y0 * SCALE, x0 * SCALE, y1 * SCALE, x1 * SCALE + + found = [c for c in result["chars"] if not c[1] and not c[0].isspace()] + for (char, _, left, bottom, right, top), (_, box) in zip(found, + expected_boxes(LINES, rotated)): + bl, bb, br, bt = box + slack = EM_SLACK * (br - bl) # the line height runs along x once rotated + assert bl - slack <= left and right <= br + slack, char + assert bb - 0.05 <= bottom and top <= bt + 0.05, char diff --git a/tests/parity/text_layer_runner.py b/tests/parity/text_layer_runner.py new file mode 100644 index 0000000..0ca51f1 --- /dev/null +++ b/tests/parity/text_layer_runner.py @@ -0,0 +1,164 @@ +"""Reads or grafts a text layer with tools independent of JPDFium, in its own process. + +Usage: python -I text_layer_runner.py probe PDF [TERM ...] # pypdfium2: text, boxes, render + python -I text_layer_runner.py font PDF # qpdf check, fontTools font parse + python -I text_layer_runner.py graft SOURCE_DIR REQUEST # OCRmyPDF's own sandwich graft +Prints one JSON object. graft executes OCRmyPDF's _graft.py with the rest of OCRmyPDF stubbed. +""" + +from __future__ import annotations + +import enum +import hashlib +import importlib.util +import io +import json +import sys +import types +from pathlib import Path + + +def probe(path: str, terms: list[str]) -> dict: + import pypdfium2 as pdfium + import pypdfium2.raw as raw + + pdf = pdfium.PdfDocument(path) + page = pdf[0] + text = page.get_textpage() + chars = [] + for i in range(text.count_chars()): + left, bottom, right, top = text.get_charbox(i, loose=True) + code = raw.FPDFText_GetUnicode(text.raw, i) + generated = bool(raw.FPDFText_IsGenerated(text.raw, i)) + chars.append([chr(code), generated, left, bottom, right, top]) + hits = {} + for term in terms: + searcher = text.search(term) + count = 0 + while searcher.get_next() is not None: + count += 1 + hits[term] = count + pixels = page.render(scale=100 / 72).to_pil().convert("L").tobytes() + result = { + "size": list(page.get_size()), + "rotation": page.get_rotation(), + "text": text.get_text_range(), + "chars": chars, + "search": hits, + "blank": min(pixels) == 255, + "render": hashlib.sha256(pixels).hexdigest(), + } + text.close() + page.close() + pdf.close() + return result + + +def font(path: str) -> dict: + import pikepdf + from fontTools.ttLib import TTFont + + with pikepdf.Pdf.open(path) as pdf: + problems = [str(p) for p in pdf.check_pdf_syntax()] + type0 = pdf.pages[0].Resources.Font.F1 + data = type0.DescendantFonts[0].FontDescriptor.FontFile2.read_bytes() + parsed = TTFont(io.BytesIO(data), checkChecksums=2) + for tag in parsed.keys(): + parsed[tag] # decompile every table, so a malformed one raises here + order = parsed.getGlyphOrder() + words = (int.from_bytes(data[i:i + 4].ljust(4, b"\0"), "big") for i in range(0, len(data), 4)) + return { + "problems": problems, + "tables": sorted(parsed.keys()), + "contours": [parsed["glyf"][name].numberOfContours for name in order], + "units_per_em": parsed["head"].unitsPerEm, + "ascent": parsed["hhea"].ascent, + "descent": parsed["hhea"].descent, + "whole_font_checksum_ok": sum(words) & 0xFFFFFFFF == 0xB1B0AFBA, + } + + +class ProcessingMode(str, enum.Enum): + default = "default" + force = "force" + force_ocr_no_links = "force-ocr-no-links" + skip = "skip" + redo = "redo" + strip_text = "strip" + + +def load_graft(source: Path): + package = types.ModuleType("ocrmypdf") + package.__path__ = [] + annots = types.ModuleType("ocrmypdf._annots") + annots.link_annotations = lambda page: [] + annots.transfer_link_annotations = lambda *args, **kwargs: None + jobcontext = types.ModuleType("ocrmypdf._jobcontext") + jobcontext.PdfContext = object + options = types.ModuleType("ocrmypdf._options") + options.ProcessingMode = ProcessingMode + pipeline = types.ModuleType("ocrmypdf._pipeline") + pipeline.VECTOR_PAGE_DPI = 400 + sys.modules.update({"ocrmypdf": package, "ocrmypdf._annots": annots, + "ocrmypdf._jobcontext": jobcontext, "ocrmypdf._options": options, + "ocrmypdf._pipeline": pipeline}) + path = source / "ocrmypdf" / "_graft.py" + spec = importlib.util.spec_from_file_location("ocrmypdf._graft", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # dataclasses resolve their module while it executes + spec.loader.exec_module(module) + return module + + +def graft(source: str, request: dict) -> dict: + import pikepdf + + module = load_graft(Path(source)) + base = pikepdf.Pdf.open(request["base"], conversion_mode="explicit") + # Only the attributes OcrGrafter.graft_page reads on the sandwich path. + grafter = types.SimpleNamespace( + pdf_base=base, + path_base=Path(request["base"]).resolve(), + context=types.SimpleNamespace( + options=types.SimpleNamespace(mode=ProcessingMode(request.get("mode", "default"))) + ), + pdfinfo=[types.SimpleNamespace(rotation=int(base.pages[0].obj.get("/Rotate", 0)))], + use_sandwich_renderer=True, + render_mode=module.RenderMode.UNDERNEATH, + emplacements=1, + ) + grafter._graft_sandwich_text_layer = types.MethodType( + module.OcrGrafter._graft_sandwich_text_layer, grafter + ) + module.OcrGrafter.graft_page( + grafter, + pageno=0, + image=None, + ocr_output=Path(request["text"]), + ocr_tree=None, + autorotate_correction=request.get("autorotate", 0), + ) + base.save(request["output"]) + base.close() + if request.get("stripped"): + with pikepdf.Pdf.open(request["output"]) as pdf: + module.strip_invisible_text(pdf, pdf.pages[0]) + pdf.save(request["stripped"]) + return {"grafted": True} + + +def main() -> None: + command, args = sys.argv[1], sys.argv[2:] + if command == "probe": + result = probe(args[0], args[1:]) + elif command == "font": + result = font(args[0]) + elif command == "graft": + result = graft(args[0], json.loads(args[1])) + else: + raise SystemExit(f"unknown command {command!r}") + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_lazy_import.py b/tests/unit/test_lazy_import.py index 13abfcc..88dee7d 100644 --- a/tests/unit/test_lazy_import.py +++ b/tests/unit/test_lazy_import.py @@ -5,6 +5,7 @@ import sys import pyjpdfium import pyjpdfium.errors, pyjpdfium.models, pyjpdfium.pdf, pyjpdfium.runtime +import pyjpdfium.raster, pyjpdfium.textlayer assert pyjpdfium.PixelFormat.RGB == 2 loaded = [m for m in sys.modules if m.startswith("pyjpdfium._native")] assert not loaded, loaded diff --git a/tests/unit/test_text_layer_models.py b/tests/unit/test_text_layer_models.py new file mode 100644 index 0000000..79b7474 --- /dev/null +++ b/tests/unit/test_text_layer_models.py @@ -0,0 +1,22 @@ +import pytest + +from pyjpdfium import TextLine, TextWord, errors +from pyjpdfium.textlayer import write_text_layer + + +def one_word(text): + box = (0.0, 0.0, 100.0, 50.0) + return [TextLine(box, (TextWord(text, box),))] + + +# Both are refused before the native library is needed: it would truncate or reject them. +@pytest.mark.parametrize("text", ["a\x00b", "\ud800"]) +def test_text_that_cannot_cross_the_c_abi_is_refused_in_python(tmp_path, text): + with pytest.raises(errors.InvalidArgumentError): + write_text_layer(tmp_path / "x.pdf", one_word(text), image_size=(100, 100), dpi=300) + + +def test_models_are_frozen(): + word = TextWord("a", (0, 0, 1, 1)) + with pytest.raises(AttributeError): + word.text = "b" # type: ignore[misc] From f5150631b7a603af7af708ee04977d4df0a39259 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:01:12 +0100 Subject: [PATCH 06/10] Add OCR engine binding with parity tests against PaddleOCR --- .github/workflows/ci.yml | 23 ++++- CMakeLists.txt | 24 +++++ README.md | 14 ++- docs/ARCHITECTURE.md | 10 +- docs/COMPATIBILITY.md | 16 ++- src/native/pyjpdfium_native.cpp | 10 ++ src/native/pyjpdfium_ocr_native.cpp | 141 +++++++++++++++++++++++++++ src/pyjpdfium/__init__.py | 5 + src/pyjpdfium/_loader.py | 48 +++++++-- src/pyjpdfium/models.py | 39 ++++++++ src/pyjpdfium/ocr.py | 62 ++++++++++++ src/pyjpdfium/pdf.py | 5 + tests/native/test_ocr.py | 140 ++++++++++++++++++++++++++ tests/parity/ocr-requirements.txt | 136 ++++++++++++++++++++++++++ tests/parity/ocr_reference_runner.py | 55 +++++++++++ tests/parity/test_ocr_parity.py | 126 ++++++++++++++++++++++++ tests/unit/test_lazy_import.py | 6 +- tests/unit/test_ocr_models.py | 24 +++++ 18 files changed, 862 insertions(+), 22 deletions(-) create mode 100644 src/native/pyjpdfium_ocr_native.cpp create mode 100644 src/pyjpdfium/ocr.py create mode 100644 tests/native/test_ocr.py create mode 100644 tests/parity/ocr-requirements.txt create mode 100644 tests/parity/ocr_reference_runner.py create mode 100644 tests/parity/test_ocr_parity.py create mode 100644 tests/unit/test_ocr_models.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1449223..a686b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake ninja-build pkg-config \ libpcre2-dev libfreetype6-dev libharfbuzz-dev libicu-dev \ - libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf + libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf libgl1 - name: Fetch pinned PDFium and verify its published digest env: @@ -76,10 +76,19 @@ jobs: mkdir -p jpdfium/native/pdfium tar -xzf "$RUNNER_TEMP/$ASSET" -C jpdfium/native/pdfium + # The optional OCR module: ONNX Runtime and the model profile, each pinned by digest. + - name: Fetch ONNX Runtime and the OCR profile (pinned) + run: | + bash jpdfium/native/ocr/fetch-onnxruntime.sh linux-x64 + python jpdfium/native/ocr/tools/fetch_profile.py \ + jpdfium/native/ocr/profiles/pp-ocrv6-small.profile "$RUNNER_TEMP/ocr-profile" + - name: Build the JPDFium core run: bash jpdfium/native/build-real.sh - name: Native core tests + env: + JPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile run: ctest --test-dir jpdfium/native/build-real --output-on-failure - name: Build pyjpdfium @@ -93,6 +102,7 @@ jobs: - name: Python tests (unit + native) env: PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + PYJPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile run: | # Native tests skip when the extension is missing; fail loudly here instead. python -c "import pyjpdfium; print(pyjpdfium.runtime_info())" @@ -105,3 +115,14 @@ jobs: python -m pip install --require-hashes --only-binary=:all: -r tests/parity/requirements.txt export PYJPDFIUM_PARITY_SRC="$(python tests/parity/fetch_reference.py "$RUNNER_TEMP/parity")" python -m pytest tests/parity -q -rs + + - name: OCR parity with the official PaddleOCR pipeline + env: + PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + PYJPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile + PYJPDFIUM_OCR_REFERENCE: ${{ runner.temp }}/ocr-reference/bin/python + run: | + python -m venv "$RUNNER_TEMP/ocr-reference" + "$RUNNER_TEMP/ocr-reference/bin/pip" install --require-hashes --only-binary=:all: \ + -r tests/parity/ocr-requirements.txt + python -m pytest tests/parity/test_ocr_parity.py -q -rs diff --git a/CMakeLists.txt b/CMakeLists.txt index 1090a26..2998809 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,30 @@ endif() install(TARGETS _native LIBRARY DESTINATION pyjpdfium) +# Optional OCR extension: built when the staged natives include libjpdfium_ocr. +set(JPDFIUM_OCR_INCLUDE_DIR "${JPDFIUM_INCLUDE_DIR}/../../ocr/include" CACHE PATH + "Directory containing jpdfium_ocr.h") +find_library(JPDFIUM_OCR_LIBRARY NAMES jpdfium_ocr PATHS "${JPDFIUM_NATIVE_DIR}" NO_DEFAULT_PATH) +if(JPDFIUM_OCR_LIBRARY AND EXISTS "${JPDFIUM_OCR_INCLUDE_DIR}/jpdfium_ocr.h") + message(STATUS "JPDFium OCR module: ${JPDFIUM_OCR_LIBRARY}") + nanobind_add_module(_ocr_native NB_STATIC src/native/pyjpdfium_ocr_native.cpp) + target_include_directories(_ocr_native PRIVATE "${JPDFIUM_INCLUDE_DIR}" "${JPDFIUM_OCR_INCLUDE_DIR}") + target_link_libraries(_ocr_native PRIVATE "${JPDFIUM_OCR_LIBRARY}" "${JPDFIUM_LIBRARY}") + if(MSVC) + target_compile_options(_ocr_native PRIVATE /W4 /WX) + else() + target_compile_options(_ocr_native PRIVATE -Wall -Wextra -Werror) + endif() + if(APPLE) + set_target_properties(_ocr_native PROPERTIES INSTALL_RPATH "@loader_path/_libs") + elseif(UNIX) + set_target_properties(_ocr_native PROPERTIES INSTALL_RPATH "$ORIGIN/_libs") + endif() + install(TARGETS _ocr_native LIBRARY DESTINATION pyjpdfium) +else() + message(STATUS "JPDFium OCR module not staged; pyjpdfium.ocr will be unavailable") +endif() + if(PYJPDFIUM_BUNDLE_NATIVES) install(DIRECTORY "${JPDFIUM_NATIVE_DIR}/" DESTINATION pyjpdfium/_libs FILES_MATCHING PATTERN "*.so*" PATTERN "*.dylib" PATTERN "*.dll" diff --git a/README.md b/README.md index 7edda1b..de078b5 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ PyJPDFium is a thin [nanobind](https://github.com/wjakob/nanobind) binding over lifetimes and handle validation, so the same rules hold for the Java and Python bindings. Status: early development. Covered so far: documents, pages, rendering, page text, transactional -PNG output, OCRmyPDF-compatible page rasterization, perspective rectification and invisible text -layers. +PNG output, OCRmyPDF-compatible page rasterization, perspective rectification, invisible text +layers and OCR with PP-OCR models. ```python import pyjpdfium @@ -30,6 +30,12 @@ from pyjpdfium import TextLine, TextWord line = TextLine((300, 300, 1500, 400), (TextWord("Hello", (300, 300, 800, 400)), TextWord("world", (900, 300, 1500, 400)))) pyjpdfium.write_text_layer("text.pdf", [line], image_size=(2550, 3300), dpi=300) + +# OCR, when the natives include the OCR module (see JPDFium docs/design/OCR.md). +with pyjpdfium.OcrEngine("pp-ocrv6-small/") as engine: + result = engine.recognize("page.png") + pyjpdfium.write_text_layer("text.pdf", result.text_lines(), image_size=result.image_size, + dpi=300) ``` ## Behaviour @@ -64,6 +70,8 @@ PYJPDFIUM_NATIVE_DIR=$JPDFIUM_NATIVE_DIR pytest Native tests (`-m native`) need the real core and are skipped when the extension is not built. Parity with OCRmyPDF's own rasterizer and sandwich graft is opt-in: install `tests/parity/requirements.txt` (hash-pinned), set `PYJPDFIUM_PARITY_SRC` to the output of -`tests/parity/fetch_reference.py`, and run `pytest tests/parity`. +`tests/parity/fetch_reference.py`, and run `pytest tests/parity`. OCR tests need +`PYJPDFIUM_OCR_PROFILE`; OCR parity also needs `PYJPDFIUM_OCR_REFERENCE`, a Python with +`tests/parity/ocr-requirements.txt` installed. See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6bd7f22..ac5f419 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -17,10 +17,12 @@ The full contracts live in JPDFium's `docs/design/` (`ARCHITECTURE.md`, `NATIVE_ ## Layers ```text -pyjpdfium pure Python API: Document, Page, Bitmap, write_text_layer, runtime_* -pyjpdfium._loader lazy load, core pinning, identity checks, runtime init -pyjpdfium._native nanobind module: one function per jpdf_* call, GIL released around it -libjpdfium the shared native core (never compiled into the extension) +pyjpdfium pure Python API: Document, Page, Bitmap, OcrEngine, runtime_* ... +pyjpdfium._loader lazy load, core pinning, identity checks, runtime init +pyjpdfium._native nanobind module: one function per jpdf_* call, GIL released around it +pyjpdfium._ocr_native nanobind module over jpdfium_ocr.h, loaded on the first OcrEngine +libjpdfium the shared native core (never compiled into the extension) +libjpdfium_ocr the optional OCR module, linked to the core, with ONNX Runtime ``` `import pyjpdfium` and its pure modules load nothing native. The first call that needs the core diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 879ee2b..8129be4 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -23,9 +23,23 @@ OCRmyPDF 17.13's renderer lays out invisible text. OCRmyPDF's own sandwich graft pypdfium2, qpdf and fontTools read it back (JPDFium's `docs/design/TEXT_LAYER.md`). Right-to-left scripts raise `UnsupportedError` until the multilingual writer lands. +## OCR + +`OcrEngine` runs JPDFium's optional OCR module: PP-OCRv6 small on ONNX Runtime 1.30.0, with the +models of a profile directory fetched by JPDFium's `native/ocr/tools/fetch_profile.py`. On the +parity fixtures it reproduces the official `paddleocr` 3.7.0 pipeline exactly: the same polygons, +the same text, and confidences within 1.2e-7 (JPDFium's `docs/design/OCR.md`). + +- PNG input is read as OpenCV's `imread(IMREAD_COLOR)` reads it: alpha dropped, 16-bit samples + cut to their high byte. +- Lines only: `OcrResult.word_geometry` is `"absent"`, and `text_lines()` keeps each line whole. +- No text-line orientation model, so vertical text is not read correctly, as in the reference + configuration. +- The OCR extension is built only when the staged natives include `libjpdfium_ocr`. + ## Not in this release -- OCR and the OCRmyPDF plugin. +- The OCRmyPDF plugin. - PDF/A output: rejected by design. - Opening from bytes or file-like objects, saving or editing documents. - Free-threaded CPython builds are untested. diff --git a/src/native/pyjpdfium_native.cpp b/src/native/pyjpdfium_native.cpp index d950b4c..eb0e5f6 100644 --- a/src/native/pyjpdfium_native.cpp +++ b/src/native/pyjpdfium_native.cpp @@ -92,6 +92,7 @@ bool struct_sizes_match() { {JPDF_STRUCT_TEXT_WORD, sizeof(jpdf_text_word)}, {JPDF_STRUCT_TEXT_LINE, sizeof(jpdf_text_line)}, {JPDF_STRUCT_TEXT_LAYER_OPTIONS, sizeof(jpdf_text_layer_options)}, + {JPDF_STRUCT_IMAGE_LOAD_OPTIONS, sizeof(jpdf_image_load_options)}, }; for (const auto& pair : expected) { if (jpdf_abi_struct_size(static_cast(pair[0])) != pair[1]) return false; @@ -317,6 +318,15 @@ NB_MODULE(_native, m) { run([&] { return jpdf_page_render_to_file(page, &o, path.c_str(), &w); }); }); + m.def("image_load", [](const std::string& path, uint64_t max_pixels) { + jpdf_image_load_options o{}; + o.struct_size = sizeof(o); + o.max_pixels = max_pixels; + jpdf_handle bitmap = 0; + run([&] { return jpdf_image_load_file(path.c_str(), &o, &bitmap); }); + return bitmap; + }); + m.def("bitmap_info", [](jpdf_handle bitmap) { jpdf_bitmap_info info{}; info.struct_size = sizeof(info); diff --git a/src/native/pyjpdfium_ocr_native.cpp b/src/native/pyjpdfium_ocr_native.cpp new file mode 100644 index 0000000..123476e --- /dev/null +++ b/src/native/pyjpdfium_ocr_native.cpp @@ -0,0 +1,141 @@ +// pyjpdfium_ocr_native.cpp - nanobind binding over jpdfium_ocr.h, loaded only when OCR is used. +// Native calls run with the GIL released; errors come from the core's per-thread detail. + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "jpdfium_api.h" +#include "jpdfium_ocr.h" + +namespace nb = nanobind; + +namespace { + +std::string last_error_detail() { + uint64_t needed = 0; + jpdf_last_error_message(nullptr, 0, &needed); + if (needed <= 1) return {}; + std::string detail(static_cast(needed), '\0'); + if (jpdf_last_error_message(detail.data(), needed, &needed) != JPDF_OK) return {}; + detail.pop_back(); + return detail; +} + +[[noreturn]] void raise_status(jpdf_status status, const std::string& detail) { + nb::object errors = nb::module_::import_("pyjpdfium.errors"); + nb::object exc = errors.attr("_from_status")(status, jpdf_status_name(status), detail); + PyErr_SetObject(reinterpret_cast(Py_TYPE(exc.ptr())), exc.ptr()); + throw nb::python_error(); +} + +template +void run(Call&& call) { + jpdf_status status; + std::string detail; + { + nb::gil_scoped_release release; + status = call(); + if (status != JPDF_OK) detail = last_error_detail(); + } + if (status != JPDF_OK) raise_status(status, detail); +} + +bool struct_sizes_match() { + const int64_t expected[][2] = { + {JPDF_OCR_STRUCT_OPTIONS, sizeof(jpdf_ocr_options)}, + {JPDF_OCR_STRUCT_ENGINE_INFO, sizeof(jpdf_ocr_engine_info)}, + {JPDF_OCR_STRUCT_RECOGNIZE_OPTIONS, sizeof(jpdf_ocr_recognize_options)}, + {JPDF_OCR_STRUCT_RESULT_INFO, sizeof(jpdf_ocr_result_info)}, + {JPDF_OCR_STRUCT_LINE, sizeof(jpdf_ocr_line)}, + }; + for (const auto& pair : expected) { + if (jpdf_ocr_abi_struct_size(static_cast(pair[0])) != pair[1]) return false; + } + return true; +} + +using Line = std::tuple, double, double>; +using Recognized = std::tuple>; + +// Reads a whole result; on failure the detail is captured before the result is closed. +jpdf_status read_result(jpdf_handle result, jpdf_ocr_result_info* info, std::vector* lines) { + jpdf_status st = jpdf_ocr_result_get_info(result, info); + for (int32_t i = 0; st == JPDF_OK && i < info->line_count; ++i) { + jpdf_ocr_line line{}; + line.struct_size = sizeof(line); + uint64_t needed = 0; + st = jpdf_ocr_result_get_line(result, i, &line, nullptr, 0, &needed); + if (st != JPDF_OK) break; + std::string text(static_cast(needed), '\0'); + st = jpdf_ocr_result_get_line(result, i, &line, text.data(), needed, &needed); + if (st != JPDF_OK) break; + text.pop_back(); + lines->emplace_back(std::move(text), std::vector(line.polygon, line.polygon + 8), + line.confidence, line.detection_score); + } + return st; +} + +// Recognizes and reads the result in one released-GIL section, then closes it. +Recognized recognize(jpdf_handle engine, jpdf_handle bitmap, uint64_t max_pixels) { + jpdf_ocr_recognize_options o{}; + o.struct_size = sizeof(o); + o.max_pixels = max_pixels; + jpdf_ocr_result_info info{}; + info.struct_size = sizeof(info); + std::vector lines; + jpdf_status status; + std::string detail; + { + nb::gil_scoped_release release; + jpdf_handle result = 0; + status = jpdf_ocr_recognize(engine, bitmap, &o, &result); + if (status == JPDF_OK) { + status = read_result(result, &info, &lines); + if (status != JPDF_OK) detail = last_error_detail(); + jpdf_ocr_result_close(result); + } else { + detail = last_error_detail(); + } + } + if (status != JPDF_OK) raise_status(status, detail); + return {info.image_width, info.image_height, info.dpi_x, info.dpi_y, std::move(lines)}; +} + +} // namespace + +NB_MODULE(_ocr_native, m) { + m.doc() = "Low-level binding over jpdfium_ocr.h. Use pyjpdfium.ocr instead."; + m.attr("COMPILED_API_VERSION") = (JPDF_OCR_API_VERSION_MAJOR << 16) | JPDF_OCR_API_VERSION_MINOR; + m.def("api_version", [] { return jpdf_ocr_api_version(); }); + m.def("struct_sizes_match", &struct_sizes_match); + m.def("capability", [](const std::string& name) { + int32_t level = 0; + run([&] { return jpdf_ocr_capability(name.c_str(), &level); }); + return level; + }); + m.def("engine_open", [](const std::string& profile_dir, int32_t threads) { + jpdf_ocr_options o{}; + o.struct_size = sizeof(o); + o.profile_dir_utf8 = profile_dir.c_str(); + o.intra_op_threads = threads; + jpdf_handle engine = 0; + run([&] { return jpdf_ocr_engine_open(&o, &engine); }); + return engine; + }); + m.def("engine_info", [](jpdf_handle engine) { + jpdf_ocr_engine_info info{}; + info.struct_size = sizeof(info); + run([&] { return jpdf_ocr_engine_get_info(engine, &info); }); + return std::make_tuple(std::string(info.profile_id), std::string(info.runtime_version)); + }); + m.def("engine_close", [](jpdf_handle engine) { run([&] { return jpdf_ocr_engine_close(engine); }); }); + m.def("recognize", &recognize); +} diff --git a/src/pyjpdfium/__init__.py b/src/pyjpdfium/__init__.py index 6ec0353..321d12f 100644 --- a/src/pyjpdfium/__init__.py +++ b/src/pyjpdfium/__init__.py @@ -13,6 +13,9 @@ _EXPORTS = { "open_document": "pyjpdfium.pdf", + "load_image": "pyjpdfium.pdf", + "OcrEngine": "pyjpdfium.ocr", + "recognize_image": "pyjpdfium.ocr", "Document": "pyjpdfium.pdf", "Page": "pyjpdfium.pdf", "Bitmap": "pyjpdfium.pdf", @@ -37,6 +40,8 @@ "RectifyResult": "pyjpdfium.models", "TextWord": "pyjpdfium.models", "TextLine": "pyjpdfium.models", + "OcrLine": "pyjpdfium.models", + "OcrResult": "pyjpdfium.models", "PyJPDFiumError": "pyjpdfium.errors", "NativeLoadError": "pyjpdfium.errors", "InvalidArgumentError": "pyjpdfium.errors", diff --git a/src/pyjpdfium/_loader.py b/src/pyjpdfium/_loader.py index 2b0ea7b..db18c34 100644 --- a/src/pyjpdfium/_loader.py +++ b/src/pyjpdfium/_loader.py @@ -18,6 +18,7 @@ _lock = threading.Lock() _module: ModuleType | None = None +_ocr_module: ModuleType | None = None # Keeps Windows DLL directory registrations and preloaded cores alive for the process. _pins: list[object] = [] @@ -34,6 +35,30 @@ def native() -> ModuleType: return _module +def ocr_native() -> ModuleType: + """Returns the OCR extension, loading the core first and then the OCR module beside it.""" + global _ocr_module + native() # the OCR module links the core, so the core loads and starts first + if _ocr_module is not None: + return _ocr_module + with _lock: + if _ocr_module is None: + directory = native_dir() + if directory is not None: + _pin_library(directory / _library_name("jpdfium_ocr"), "the JPDFium OCR module") + try: + module = importlib.import_module("pyjpdfium._ocr_native") + except ImportError as exc: + raise NativeLoadError(f"the JPDFium OCR module is not available: {exc}") from exc + library, compiled = module.api_version(), module.COMPILED_API_VERSION + if library >> 16 != compiled >> 16 or (library & 0xFFFF) < (compiled & 0xFFFF): + raise NativeLoadError("JPDFium OCR module API is incompatible with this build") + if not module.struct_sizes_match(): + raise NativeLoadError("JPDFium OCR struct layouts differ from the compiled header") + _ocr_module = module + return _ocr_module + + def native_dir() -> Path | None: configured = os.environ.get(NATIVE_DIR_ENV) if configured: @@ -42,23 +67,26 @@ def native_dir() -> Path | None: return bundled if bundled.is_dir() else None -def _core_name() -> str: +def _library_name(stem: str) -> str: if sys.platform == "win32": - return "jpdfium.dll" + return f"{stem}.dll" if sys.platform == "darwin": - return "libjpdfium.dylib" - return "libjpdfium.so" + return f"lib{stem}.dylib" + return f"lib{stem}.so" -def _pin_core(directory: Path) -> None: - core = directory / _core_name() - if not core.is_file(): - raise NativeLoadError(f"{core} not found; set {NATIVE_DIR_ENV} to staged JPDFium natives") +def _pin_library(path: Path, what: str) -> None: + if not path.is_file(): + raise NativeLoadError(f"{what} not found at {path}; set {NATIVE_DIR_ENV} to staged natives") if sys.platform == "win32": - _pins.append(os.add_dll_directory(str(directory))) + _pins.append(os.add_dll_directory(str(path.parent))) return # Loading this exact file first makes the extension bind to it rather than another copy. - _pins.append(ctypes.CDLL(str(core), mode=ctypes.RTLD_GLOBAL)) + _pins.append(ctypes.CDLL(str(path), mode=ctypes.RTLD_GLOBAL)) + + +def _pin_core(directory: Path) -> None: + _pin_library(directory / _library_name("jpdfium"), "the JPDFium core") def _loaded_core_paths() -> set[str]: diff --git a/src/pyjpdfium/models.py b/src/pyjpdfium/models.py index a7d8fec..aff8949 100644 --- a/src/pyjpdfium/models.py +++ b/src/pyjpdfium/models.py @@ -131,6 +131,45 @@ class TextLine: words: tuple[TextWord, ...] +@dataclass(frozen=True) +class OcrLine: + """One recognized line. Its polygon runs clockwise from the top-left, in image pixels.""" + + text: str + polygon: tuple[tuple[float, float], ...] + confidence: float + detection_score: float + + @property + def box(self) -> Box: + xs = [x for x, _ in self.polygon] + ys = [y for _, y in self.polygon] + return min(xs), min(ys), max(xs), max(ys) + + +@dataclass(frozen=True) +class OcrResult: + """Lines in reading order. The models report lines only, so word geometry is absent.""" + + image_size: tuple[int, int] + dpi: tuple[float, float] + profile: str + lines: tuple[OcrLine, ...] + word_geometry: str = "absent" + + @property + def text(self) -> str: + return "\n".join(line.text for line in self.lines) + + def text_lines(self) -> tuple[TextLine, ...]: + """For write_text_layer: each line becomes one word spanning its box, never split.""" + return tuple( + TextLine(line.box, (TextWord(line.text, line.box),)) + for line in self.lines + if line.text.strip() + ) + + @dataclass(frozen=True) class RectifyResult: """Corners as used (top-left, top-right, bottom-right, bottom-left) and the inverse map.""" diff --git a/src/pyjpdfium/ocr.py b/src/pyjpdfium/ocr.py new file mode 100644 index 0000000..e863972 --- /dev/null +++ b/src/pyjpdfium/ocr.py @@ -0,0 +1,62 @@ +"""OCR with PP-OCR models on ONNX Runtime, in JPDFium's optional native OCR module. + +The module and its models load on first use. A profile directory holds the models it pins, as +fetched by JPDFium's native/ocr/tools/fetch_profile.py; every file is checked against its +digest when an engine opens. +""" + +from __future__ import annotations + +import os +from typing import Union + +from ._loader import ocr_native +from .errors import InvalidArgumentError +from .models import OcrLine, OcrResult +from .pdf import Bitmap, _Handle, load_image + +StrPath = Union[str, "os.PathLike[str]"] + + +class OcrEngine(_Handle): + """A verified profile with its two models loaded. Safe to share across threads.""" + + def __init__(self, profile: StrPath, *, threads: int = 0) -> None: + if threads < 0: + raise InvalidArgumentError("threads must not be negative") + lib = ocr_native() + token = lib.engine_open(os.fsdecode(os.fspath(profile)), threads) + super().__init__(token, lib.engine_close) + self.profile, self.runtime_version = lib.engine_info(self._require_open()) + + def recognize(self, image: Bitmap | StrPath, *, max_pixels: int | None = None) -> OcrResult: + """Recognizes a Bitmap, or a PNG file loaded as OpenCV's imread would read it.""" + self._require_open() + if isinstance(image, Bitmap): + return self._recognize(image, max_pixels) + with load_image(image) as bitmap: + return self._recognize(bitmap, max_pixels) + + def _recognize(self, bitmap: Bitmap, max_pixels: int | None) -> OcrResult: + width, height, dpi_x, dpi_y, lines = ocr_native().recognize( + self._require_open(), bitmap._require_open(), max_pixels or 0 + ) + return OcrResult( + image_size=(width, height), + dpi=(dpi_x, dpi_y), + profile=self.profile, + lines=tuple( + OcrLine(text, tuple(zip(poly[0::2], poly[1::2])), confidence, score) + for text, poly, confidence, score in lines + ), + ) + + def __repr__(self) -> str: + state = "closed" if self.closed else "open" + return f"" + + +def recognize_image(path: StrPath, *, profile: StrPath, threads: int = 0) -> OcrResult: + """One-off recognition; keep an OcrEngine for many images, since opening one loads models.""" + with OcrEngine(profile, threads=threads) as engine: + return engine.recognize(path) diff --git a/src/pyjpdfium/pdf.py b/src/pyjpdfium/pdf.py index c97fd26..5d2d645 100644 --- a/src/pyjpdfium/pdf.py +++ b/src/pyjpdfium/pdf.py @@ -78,6 +78,11 @@ def open_document(path: StrPath, *, password: str | None = None) -> Document: return Document(lib.doc_open(_fspath(path), password), _fspath(path)) +def load_image(path: StrPath, *, max_pixels: int | None = None) -> Bitmap: + """Decodes a PNG into a Bitmap: gray stays gray, colour becomes RGB, alpha is dropped.""" + return Bitmap(native().image_load(_fspath(path), max_pixels or 0)) + + class Document(_Handle): def __init__(self, token: int, path: str) -> None: super().__init__(token, native().doc_close) diff --git a/tests/native/test_ocr.py b/tests/native/test_ocr.py new file mode 100644 index 0000000..1503ac6 --- /dev/null +++ b/tests/native/test_ocr.py @@ -0,0 +1,140 @@ +import importlib.util +import os +import shutil +import subprocess +import sys +import threading + +import pytest + +import pyjpdfium +from pyjpdfium import errors + +PROFILE = os.environ.get("PYJPDFIUM_OCR_PROFILE") +pytestmark = [ + pytest.mark.native, + pytest.mark.skipif(not PROFILE, reason="PYJPDFIUM_OCR_PROFILE is not set"), + pytest.mark.skipif(importlib.util.find_spec("pyjpdfium._ocr_native") is None, + reason="pyjpdfium was built without the OCR module"), +] + +LINES = ["The quick brown fox jumps over the lazy dog.", "Invoice 2026-10-11 total 1,234.56 EUR"] + + +def page_pdf(tmp_path): + content = (f"BT /F1 18 Tf 72 700 Td ({LINES[0]}) Tj 0 -40 Td /F1 12 Tf ({LINES[1]}) Tj ET") + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 << " + "/Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> /Contents 4 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + ] + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n{body}\nendobj\n".encode() + xref = len(out) + out += b"xref\n0 5\n0000000000 65535 f \n" + out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets) + out += f"trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + path = tmp_path / "page.pdf" + path.write_bytes(bytes(out)) + return path + + +@pytest.fixture(scope="module") +def engine(): + with pyjpdfium.OcrEngine(PROFILE) as e: + yield e + + +def test_engine_identity(engine): + assert engine.profile == "pp-ocrv6-small" + assert engine.runtime_version == "1.30.0" + + +def test_png_file_is_read_like_opencv(tmp_path, engine): + png = tmp_path / "page.png" + pyjpdfium.rasterize_page(page_pdf(tmp_path), 0, png, device="pnggray", raster_dpi=300) + result = engine.recognize(png) + assert [line.text for line in result.lines] == LINES + assert result.image_size == (2550, 3300) + assert result.dpi[0] == pytest.approx(300, abs=0.01) + assert result.word_geometry == "absent" + assert all(0.9 < line.confidence <= 1 for line in result.lines) + + +def test_bitmap_input_and_independent_lifetimes(tmp_path, engine): + with pyjpdfium.open_document(page_pdf(tmp_path)) as document, document.page(0) as page: + bitmap = page.render(dpi=300, pixel_format="rgb") + result = engine.recognize(bitmap) + bitmap.close() # the result owns copies; nothing depends on the bitmap any more + assert result.text.splitlines() == LINES + + +def test_ocr_feeds_the_text_layer(tmp_path, engine): + png = tmp_path / "page.png" + pyjpdfium.rasterize_page(page_pdf(tmp_path), 0, png, device="pnggray", raster_dpi=300) + result = engine.recognize(png) + layer = tmp_path / "layer.pdf" + pyjpdfium.write_text_layer(layer, result.text_lines(), image_size=result.image_size, dpi=300) + with pyjpdfium.open_document(layer) as document, document.page(0) as page: + assert " ".join(page.text().split()) == " ".join(LINES) + + +def test_concurrent_recognitions_agree(tmp_path, engine): + png = tmp_path / "page.png" + pyjpdfium.rasterize_page(page_pdf(tmp_path), 0, png, device="pnggray", raster_dpi=150) + texts, failures = [], [] + + def worker(): + try: + texts.append(engine.recognize(png).text) + except Exception as exc: # noqa: BLE001 - reported below + failures.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not failures and len(set(texts)) == 1 + + +def test_tampered_profile_is_refused(tmp_path): + copy = tmp_path / "profile" + shutil.copytree(PROFILE, copy) + with (copy / "dictionary.txt").open("a", encoding="utf-8") as f: + f.write("x\n") + with pytest.raises(errors.PdfFormatError, match="digest"): + pyjpdfium.OcrEngine(copy) + + +def test_closed_engine_is_stale(tmp_path): + engine = pyjpdfium.OcrEngine(PROFILE) + engine.close() + with pytest.raises(errors.StaleHandleError): + engine.recognize(tmp_path / "unused.png") + + +PROBE = """ +import pyjpdfium +with pyjpdfium.open_document(r"{pdf}") as d: + pass +maps = open("/proc/self/maps").read() +assert "onnxruntime" not in maps and "jpdfium_ocr" not in maps, "OCR loaded too early" +engine = pyjpdfium.OcrEngine(r"{profile}") +maps = open("/proc/self/maps").read() +assert "onnxruntime" in maps and "jpdfium_ocr" in maps +print("lazy-ok") +""" + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="reads /proc/self/maps") +def test_ocr_module_loads_only_when_used(tmp_path): + probe = PROBE.format(pdf=page_pdf(tmp_path), profile=PROFILE) + done = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, + check=False) + assert done.returncode == 0, done.stderr + assert "lazy-ok" in done.stdout diff --git a/tests/parity/ocr-requirements.txt b/tests/parity/ocr-requirements.txt new file mode 100644 index 0000000..ac3c358 --- /dev/null +++ b/tests/parity/ocr-requirements.txt @@ -0,0 +1,136 @@ +# Official PaddleOCR 3.7 pipeline for the OCR parity tests (Linux x86_64, CPython 3.12), in its own +# environment: python -m venv ref && ref/bin/pip install --require-hashes --only-binary=:all: \ +# -r tests/parity/ocr-requirements.txt +# Then set PYJPDFIUM_OCR_REFERENCE=ref/bin/python. opencv-contrib-python needs libGL. +PyYAML==6.0.2 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 +aiohappyeyeballs==2.7.1 \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 +aiohttp==3.14.5 \ + --hash=sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584 +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e +aistudio_sdk==0.3.9 \ + --hash=sha256:41621d052ddb7a58643438a7ca541f58ef0ab94a2acdfff0a71db81bf68dc027 +annotated-types==0.8.0 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +bce-python-sdk==0.9.79 \ + --hash=sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 +cffi==2.1.1 \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf +chardet==7.6.0 \ + --hash=sha256:2cf0adaca8b1c4bacfade9d0a1e4f8f70b1bb122833d6f07ab90e3adc84eb13a +charset-normalizer==3.5.2 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 +colorlog==6.12.0 \ + --hash=sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e +crc32c==2.9.post0 \ + --hash=sha256:0a56531e7e965eb3382a8a89e9cf3f134059c53ba1d59788bf27d27ad16cc378 +cryptography==50.0.2 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 +filelock==4.1.0 \ + --hash=sha256:2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1 +flatbuffers==25.12.19 \ + --hash=sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4 +frozenlist==1.8.0 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 +fsspec==2026.9.0 \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f +future==1.0.0 \ + --hash=sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216 +h11==0.16.0 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +hf-xet==1.7.0 \ + --hash=sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8 +httpcore2==2.13.1 \ + --hash=sha256:e1e05d4f25f7d7d496bfb96748f6f4b67657b03da069b3a68c36069f3db73d0a +httpx2==2.13.1 \ + --hash=sha256:6dff50fabc270ee5fd25d845d0b078ed20564579744d6d962850975996d2f9a4 +huggingface_hub==2.2.0 \ + --hash=sha256:1667f145dc56dc210d60966069397df9ecfca9607a5d43db88b308c89dae56b3 +idna==3.20 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +imagesize==2.0.1 \ + --hash=sha256:ea0c9a0384df69ed86a943a15cde37d0360b82491b3910dc2215e202e62b5b02 +modelscope==1.40.2 \ + --hash=sha256:ea03e0ec4a73398a488d3d240cda4f0c5b62390377a134756d4074a3036cb909 +modelscope-hub==0.4.6 \ + --hash=sha256:361bcec229bfbed645a9291ec5b167d76cd94b795207dc2eceba628108f4023a +multidict==7.1.0 \ + --hash=sha256:fed6b7705d49dd07e5e0dd5f5c873fc44047e92d714299b13245b5fecac49d01 +numpy==2.3.5 \ + --hash=sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28 +onnxruntime==1.30.0 \ + --hash=sha256:fa688e7891a6aa206636fe7372e27ee75fd17713289f6b4fc7b190e0a7de9328 +opencv-contrib-python==4.10.0.84 \ + --hash=sha256:a261223db41f6e512d76deaf21c8fcfb4fbbcbc2de62ca7f74a05f2c9ee489ef +packaging==26.3 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +paddleocr==3.7.0 \ + --hash=sha256:c0f0a81ad4112727f30c6fcf986ac0ef6a120d31ee0991a01fae0357ee32d338 +paddlex==3.7.2 \ + --hash=sha256:f1678bf650bbaccfd8f0d4e49d0ae631b4685c829fdae6e802ccd90d4fcb9a7f +pandas==3.0.6 \ + --hash=sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d +pillow==12.3.0 \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 +prettytable==3.18.0 \ + --hash=sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a +propcache==0.5.4 \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b +protobuf==7.36.2 \ + --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 +psutil==7.2.2 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 +py-cpuinfo==9.0.0 \ + --hash=sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5 +pyclipper==1.4.0 \ + --hash=sha256:d1f807e2b4760a8e5c6d6b4e8c1d71ef52b7fe1946ff088f4fa41e16a881a5ca +pycparser==3.11 \ + --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 +pycryptodome==3.24.0 \ + --hash=sha256:cf975cc3a0822a662ec2cdae85b38ad6f67654f9b48fbe02c5baae5999a6c18d +pydantic==2.14.0 \ + --hash=sha256:15fab1bea6f1dc5003b54fc2ecab230c1fd1dbade2acd4addc52d81e32416d4b +pydantic_core==2.50.0 \ + --hash=sha256:f187030fc3d62c668feb0f09e92852e0eb414d7fcefc4748f2e67d245aade37e +pypdfium2==5.14.0 \ + --hash=sha256:11f281613fa22313d9c7ab89947665e84eccf8ebe40e1198a84a88352305648d +python-bidi==0.6.11 \ + --hash=sha256:3611d13b53d4c899c4f2a7cd8eb897064e8b5546c3c5d1037dd6209c82858a27 +python-dateutil==2.9.0.post0 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +ruamel.yaml==0.19.1 \ + --hash=sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93 +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 +shapely==2.2.0 \ + --hash=sha256:650a5f4d8a8e3c96982079d8c99b6ddbe6602bbd1e34c75c2b95dbc0d28ac997 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 +truststore==0.10.4 \ + --hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981 +typing_extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.4 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 +ujson==6.0.0 \ + --hash=sha256:e9359bfd0efd12593f0db40ccb2d1497284401da207f1d6a1783718313201b21 +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 +wcwidth==0.9.2 \ + --hash=sha256:751bef0ab404b6a1dc028b56b4b85d46486be1c55833f80da533e42dc691f389 +yarl==1.25.1 \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d diff --git a/tests/parity/ocr_reference_runner.py b/tests/parity/ocr_reference_runner.py new file mode 100644 index 0000000..a34eb02 --- /dev/null +++ b/tests/parity/ocr_reference_runner.py @@ -0,0 +1,55 @@ +"""Runs the official PaddleOCR 3.7 pipeline (ONNX Runtime engine) on images, offline. + +Usage: REFERENCE_PYTHON -I ocr_reference_runner.py PROFILE_DIR IMAGE [IMAGE ...] +Prints one JSON object per image. The models come from the profile directory, never from a hub. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path + + +def model_dir(profile: Path, work: Path, role: str, stem: str) -> Path: + # PaddleOCR expects inference.onnx and inference.yml; link the pinned files under those names. + target = work / role + target.mkdir() + (target / "inference.onnx").symlink_to(profile / f"{stem}.onnx") + (target / "inference.yml").symlink_to(profile / f"{stem}.yml") + return target + + +def main() -> None: + profile, images = Path(sys.argv[1]), sys.argv[2:] + os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True" + os.environ["HF_HUB_OFFLINE"] = "1" + from paddleocr import PaddleOCR + + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + ocr = PaddleOCR( + text_detection_model_name="PP-OCRv6_small_det", + text_detection_model_dir=str(model_dir(profile, work, "det", "det")), + text_recognition_model_name="PP-OCRv6_small_rec", + text_recognition_model_dir=str(model_dir(profile, work, "rec", "rec")), + engine="onnxruntime", + use_doc_orientation_classify=False, + use_doc_unwarping=False, + use_textline_orientation=False, + ) + for image in images: + for res in ocr.predict(image): + r = res.json["res"] + print(json.dumps({ + "image": image, + "polys": [[[int(v) for v in p] for p in poly] for poly in r["rec_polys"]], + "texts": r["rec_texts"], + "scores": [float(s) for s in r["rec_scores"]], + })) + + +if __name__ == "__main__": + main() diff --git a/tests/parity/test_ocr_parity.py b/tests/parity/test_ocr_parity.py new file mode 100644 index 0000000..101cdaf --- /dev/null +++ b/tests/parity/test_ocr_parity.py @@ -0,0 +1,126 @@ +"""OCR parity with the official PaddleOCR 3.7 pipeline on the same PNG files. + +Opt-in: set PYJPDFIUM_OCR_PROFILE to a profile fetched with JPDFium's fetch_profile.py and +PYJPDFIUM_OCR_REFERENCE to a Python with paddleocr 3.7.0 and onnxruntime 1.30.0 (see +tests/parity/ocr-requirements.txt). The reference runs in its own process and never downloads. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import pytest + +import pyjpdfium + +PROFILE = os.environ.get("PYJPDFIUM_OCR_PROFILE") +REFERENCE = os.environ.get("PYJPDFIUM_OCR_REFERENCE") +RUNNER = Path(__file__).with_name("ocr_reference_runner.py") +pytestmark = [ + pytest.mark.native, + pytest.mark.skipif(not (PROFILE and REFERENCE), + reason="PYJPDFIUM_OCR_PROFILE and PYJPDFIUM_OCR_REFERENCE are not set"), +] + +FONTS = ("/Resources << /Font << " + "/F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> " + "/F2 << /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >> " + "/F3 << /Type /Font /Subtype /Type1 /BaseFont /Courier /Encoding /WinAnsiEncoding >> " + "/F4 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold " + "/Encoding /WinAnsiEncoding >> >> >>") + +PAGES = { + "letters": ("/MediaBox [0 0 612 792]", + "BT /F1 16 Tf 72 720 Td (The quick brown fox jumps over the lazy dog.) Tj ET " + "BT /F2 14 Tf 72 690 Td (Pack my box with five dozen liquor jugs.) Tj ET " + "BT /F3 12 Tf 72 660 Td (Invoice No. 2026-0042 Total: 1,234.56 EUR) Tj ET " + "BT /F4 20 Tf 72 620 Td (Grand total due) Tj ET " + "BT /F1 11 Tf 72 590 Td (Caf\\351 na\\357ve r\\351sum\\351 \\374ber Stra\\337e) Tj ET"), + "columns": ("/MediaBox [0 0 612 792]", + "".join(f"BT /F1 12 Tf {x} {700 - 24 * row} Td ({cell}) Tj ET " + for row, cells in enumerate([("Item", "Qty", "Price"), + ("Paper A4", "12", "4.99"), + ("Toner", "2", "61.20"), + ("Stapler", "1", "8.75")]) + for x, cell in zip((72, 300, 450), cells))), + "rotated": ("/MediaBox [0 0 612 792]", + "BT /F1 16 Tf 0 1 -1 0 300 200 Tm (Vertical heading text) Tj ET " + "BT /F1 14 Tf 72 720 Td (A horizontal line beside it) Tj ET"), + "skewed": ("/MediaBox [0 0 612 792]", + "BT /F1 18 Tf 0.99863 0.05234 -0.05234 0.99863 72 500 Tm " + "(A slightly skewed line of text) Tj ET"), + "tiny": ("/MediaBox [0 0 150 30]", "BT /F1 14 Tf 5 10 Td (Tiny label) Tj ET"), +} + +CASES = [ + ("letters", "pnggray", 300), + ("letters", "png16m", 200), + ("letters", "pngmono", 300), + ("letters", "pnggray", 600), # 5100 x 6600: the detector scales it under 4000 px + ("columns", "pnggray", 300), + ("rotated", "pnggray", 300), + ("skewed", "png16m", 300), + ("tiny", "pnggray", 72), # 150 x 30: upscaled to the 64 px minimum side +] + + +def write_pdf(path: Path, page_extra: str, content: str) -> Path: + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + f"<< /Type /Page /Parent 2 0 R {page_extra} {FONTS} /Contents 4 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + ] + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{number} 0 obj\n{body}\nendobj\n".encode("latin-1") + xref = len(out) + out += b"xref\n0 5\n0000000000 65535 f \n" + out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets) + out += f"trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + path.write_bytes(bytes(out)) + return path + + +@pytest.fixture(scope="module") +def images(tmp_path_factory) -> list[Path]: + work = tmp_path_factory.mktemp("ocr") + out = [] + for page, device, dpi in CASES: + pdf = write_pdf(work / f"{page}.pdf", *PAGES[page]) + png = work / f"{page}-{device}-{dpi}.png" + pyjpdfium.rasterize_page(pdf, 0, png, device=device, raster_dpi=dpi) + out.append(png) + return out + + +@pytest.fixture(scope="module") +def reference(images) -> dict[str, dict]: + env = {k: v for k, v in os.environ.items() if not k.startswith("PYJPDFIUM")} + done = subprocess.run([REFERENCE, "-I", str(RUNNER), PROFILE, *map(str, images)], + capture_output=True, text=True, env=env, check=False, timeout=900) + assert done.returncode == 0, done.stderr[-4000:] + results = [json.loads(line) for line in done.stdout.splitlines() if line.startswith("{")] + return {r["image"]: r for r in results} + + +@pytest.fixture(scope="module") +def engine(): + with pyjpdfium.OcrEngine(PROFILE) as e: + yield e + + +@pytest.mark.parametrize("case", range(len(CASES)), ids=["-".join(map(str, c)) for c in CASES]) +def test_matches_reference(case, images, reference, engine): + image = images[case] + want = reference[str(image)] + got = engine.recognize(image) + assert [line.text for line in got.lines] == want["texts"] + assert [[[int(x), int(y)] for x, y in line.polygon] for line in got.lines] == want["polys"] + for line, score in zip(got.lines, want["scores"]): + assert line.confidence == pytest.approx(score, abs=1e-6), line.text + assert got.lines, "every fixture has text" diff --git a/tests/unit/test_lazy_import.py b/tests/unit/test_lazy_import.py index 88dee7d..5742218 100644 --- a/tests/unit/test_lazy_import.py +++ b/tests/unit/test_lazy_import.py @@ -5,15 +5,15 @@ import sys import pyjpdfium import pyjpdfium.errors, pyjpdfium.models, pyjpdfium.pdf, pyjpdfium.runtime -import pyjpdfium.raster, pyjpdfium.textlayer +import pyjpdfium.ocr, pyjpdfium.raster, pyjpdfium.textlayer assert pyjpdfium.PixelFormat.RGB == 2 -loaded = [m for m in sys.modules if m.startswith("pyjpdfium._native")] +loaded = [m for m in sys.modules if m.startswith(("pyjpdfium._native", "pyjpdfium._ocr_native"))] assert not loaded, loaded try: maps = open("/proc/self/maps").read() except OSError: maps = "" -assert "libjpdfium" not in maps and "libpdfium" not in maps +assert "libjpdfium" not in maps and "libpdfium" not in maps and "onnxruntime" not in maps print("lazy-ok") """ diff --git a/tests/unit/test_ocr_models.py b/tests/unit/test_ocr_models.py new file mode 100644 index 0000000..1a20b6a --- /dev/null +++ b/tests/unit/test_ocr_models.py @@ -0,0 +1,24 @@ +from pyjpdfium import OcrLine, OcrResult + + +def result(*lines): + return OcrResult(image_size=(1000, 800), dpi=(300.0, 300.0), profile="p", lines=lines) + + +def line(text, polygon): + return OcrLine(text, polygon, 0.99, 0.9) + + +def test_box_bounds_a_rotated_polygon(): + assert line("a", ((10, 20), (110, 15), (112, 45), (12, 50))).box == (10, 15, 112, 50) + + +def test_text_lines_keep_each_line_whole(): + r = result(line("hello world", ((10, 10), (200, 10), (200, 40), (10, 40))), + line(" ", ((10, 50), (20, 50), (20, 60), (10, 60))), + line("next", ((10, 70), (60, 70), (60, 90), (10, 90)))) + text_lines = r.text_lines() + # Word geometry is absent: a line is one word over its box, never split evenly. + assert [tl.words[0].text for tl in text_lines] == ["hello world", "next"] + assert text_lines[0].words[0].box == text_lines[0].box == (10, 10, 200, 40) + assert r.text == "hello world\n \nnext" From e1b26f9234ad11d79003802c1e5e59b5b7cb37c4 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:36:21 +0100 Subject: [PATCH 07/10] Add right-to-left and rotated text layers with multilingual parity --- .github/workflows/ci.yml | 4 +- README.md | 2 +- docs/COMPATIBILITY.md | 38 ++- src/native/pyjpdfium_native.cpp | 14 +- src/pyjpdfium/models.py | 37 ++- src/pyjpdfium/textlayer.py | 39 ++- tests/native/test_text_layer.py | 46 ++- tests/parity/requirements.txt | 6 + tests/parity/test_multilingual_parity.py | 339 +++++++++++++++++++++++ tests/parity/text_layer_runner.py | 39 +++ tests/unit/test_ocr_models.py | 3 + tests/unit/test_text_layer_models.py | 13 + 12 files changed, 557 insertions(+), 23 deletions(-) create mode 100644 tests/parity/test_multilingual_parity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a686b23..131f9ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake ninja-build pkg-config \ libpcre2-dev libfreetype6-dev libharfbuzz-dev libicu-dev \ - libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf libgl1 + libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf libgl1 poppler-utils - name: Fetch pinned PDFium and verify its published digest env: @@ -108,7 +108,7 @@ jobs: python -c "import pyjpdfium; print(pyjpdfium.runtime_info())" python -m pytest -q -rs - - name: Raster parity with OCRmyPDF's pypdfium rasterizer + - name: Parity with OCRmyPDF and independent text extractors env: PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 run: | diff --git a/README.md b/README.md index de078b5..0a980ae 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ lifetimes and handle validation, so the same rules hold for the Java and Python Status: early development. Covered so far: documents, pages, rendering, page text, transactional PNG output, OCRmyPDF-compatible page rasterization, perspective rectification, invisible text -layers and OCR with PP-OCR models. +layers in any script and direction, and OCR with PP-OCR models. ```python import pyjpdfium diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 8129be4..c4eb129 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -20,8 +20,42 @@ measured pixel parity. `write_text_layer` writes the text-only PDF OCRmyPDF's `generate_pdf` hook returns, laid out as OCRmyPDF 17.13's renderer lays out invisible text. OCRmyPDF's own sandwich graft accepts it, and -pypdfium2, qpdf and fontTools read it back (JPDFium's `docs/design/TEXT_LAYER.md`). Right-to-left -scripts raise `UnsupportedError` until the multilingual writer lands. +pypdfium2, qpdf and fontTools read it back (JPDFium's `docs/design/TEXT_LAYER.md`). + +Right-to-left and mixed-direction words are laid out by the Unicode bidi algorithm, and a +`quad` on a `TextLine` or `TextWord` places rotated or skewed lines. Text is always passed in +logical order. Explicit bidi embeddings, overrides and isolates raise `UnsupportedError`. + +### Multilingual qualification + +`tests/parity/test_multilingual_parity.py` writes 23 samples, each flat, rotated 15, 90 and 180 +degrees, and skewed. Every character box lands in its expected cell. Four extractors then read +the text, and each must reach at least the level below. The levels are, from strongest to +weakest: + +- exact: the same text; +- NFC: equal after NFC normalization; +- words: the same words in another order; +- characters: the same characters, regrouped. + +| Extractor | Left-to-right | Right-to-left | Mixed directions | +|---|---|---|---| +| PDFium (JPDFium's and pypdfium2's) | exact; words when upside down | NFC when horizontal; words when rotated | characters | +| pdfminer.six 20260107 | exact; characters at 90 degrees | exact; characters at 90 degrees | words; characters at 90 and 180 degrees | +| poppler 24.02 `pdftotext` | exact; characters at 15 degrees | exact; characters at 15 and 180 degrees | characters | + +Most results beat their floor; flat right-to-left lines come back exact from all four. The +weaker results come from the extractors' own layout analysis: + +- PDFium orders the words of upside-down lines, and of rotated right-to-left lines, by page + position. +- PDFium can reorder Hebrew points, which NFC undoes. +- At a direction change, PDFium and poppler may drop or move a space. PDFium also swaps paired + brackets in right-to-left text. +- pdfminer.six splits vertical text into single characters. +- poppler splits words on slanted baselines. + +Search in viewers built on these extractors follows the same pattern. ## OCR diff --git a/src/native/pyjpdfium_native.cpp b/src/native/pyjpdfium_native.cpp index eb0e5f6..60e283f 100644 --- a/src/native/pyjpdfium_native.cpp +++ b/src/native/pyjpdfium_native.cpp @@ -3,11 +3,14 @@ #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -146,13 +149,14 @@ std::string page_text(jpdf_handle page) { } using Box = std::tuple; +using Quad = std::array; // Owned copies are built with the GIL held; the native call then only reads them. void text_layer_write(const std::string& output, int32_t image_width, int32_t image_height, double dpi_x, double dpi_y, - const std::vector>& lines, - const std::vector>& words, bool durable, - uint64_t max_bytes) { + const std::vector>& lines, + const std::vector>& words, + bool durable, uint64_t max_bytes) { jpdf_text_layer_options o{}; o.struct_size = sizeof(o); o.image_width = image_width; @@ -166,6 +170,8 @@ void text_layer_write(const std::string& output, int32_t image_width, int32_t im l.struct_size = sizeof(l); std::tie(l.x0, l.y0, l.x1, l.y1) = std::get<0>(lines[i]); l.word_count = std::get<1>(lines[i]); + l.flags = std::get<2>(lines[i]); + std::copy(std::get<3>(lines[i]).begin(), std::get<3>(lines[i]).end(), l.quad); } std::vector native_words(words.size()); for (size_t i = 0; i < words.size(); ++i) { @@ -173,6 +179,8 @@ void text_layer_write(const std::string& output, int32_t image_width, int32_t im w.struct_size = sizeof(w); w.text_utf8 = std::get<0>(words[i]).c_str(); std::tie(w.x0, w.y0, w.x1, w.y1) = std::get<1>(words[i]); + w.flags = std::get<2>(words[i]); + std::copy(std::get<3>(words[i]).begin(), std::get<3>(words[i]).end(), w.quad); } if (native_lines.size() > INT32_MAX || native_words.size() > INT32_MAX) { throw nb::value_error("too many lines or words"); diff --git a/src/pyjpdfium/models.py b/src/pyjpdfium/models.py index aff8949..331b3b0 100644 --- a/src/pyjpdfium/models.py +++ b/src/pyjpdfium/models.py @@ -113,22 +113,36 @@ class RasterResult: Box = tuple[float, float, float, float] +Point = tuple[float, float] +Quad = tuple[Point, Point, Point, Point] @dataclass(frozen=True) class TextWord: - """A word and its box ``(x0, y0, x1, y1)`` in image pixels, origin top-left.""" + """A word and its box ``(x0, y0, x1, y1)`` in image pixels, origin top-left. + + ``quad``, when given, places the word instead of the box: its corners top-left, top-right, + bottom-right, bottom-left as read. Text is in logical order, also for right-to-left scripts. + """ text: str box: Box + quad: Quad | None = None @dataclass(frozen=True) class TextLine: - """Words in reading order. The line box height sets the font size; it must be positive.""" + """Words in logical order. The line box height sets the font size; it must be positive. + + ``quad`` replaces the box for rotated or skewed lines: text runs along its top-left to + top-right edge, and its height sets the font size. ``direction`` is ``"auto"`` (from the + first strong character), ``"ltr"`` or ``"rtl"``. + """ box: Box words: tuple[TextWord, ...] + quad: Quad | None = None + direction: str = "auto" @dataclass(frozen=True) @@ -146,6 +160,11 @@ def box(self) -> Box: ys = [y for _, y in self.polygon] return min(xs), min(ys), max(xs), max(ys) + @property + def quad(self) -> Quad: + p = self.polygon + return (p[0], p[1], p[2], p[3]) + @dataclass(frozen=True) class OcrResult: @@ -162,12 +181,14 @@ def text(self) -> str: return "\n".join(line.text for line in self.lines) def text_lines(self) -> tuple[TextLine, ...]: - """For write_text_layer: each line becomes one word spanning its box, never split.""" - return tuple( - TextLine(line.box, (TextWord(line.text, line.box),)) - for line in self.lines - if line.text.strip() - ) + """For write_text_layer: each line becomes one word on its polygon, never split.""" + out = [] + for line in self.lines: + if not line.text.strip(): + continue + quad = line.quad + out.append(TextLine(line.box, (TextWord(line.text, line.box, quad),), quad)) + return tuple(out) @dataclass(frozen=True) diff --git a/src/pyjpdfium/textlayer.py b/src/pyjpdfium/textlayer.py index 5065975..41a3277 100644 --- a/src/pyjpdfium/textlayer.py +++ b/src/pyjpdfium/textlayer.py @@ -8,7 +8,7 @@ from ._loader import native from .errors import InvalidArgumentError -from .models import Box, TextLine +from .models import Box, Quad, TextLine StrPath = Union[str, "os.PathLike[str]"] Dpi = Union[float, "tuple[float, float]"] @@ -19,6 +19,30 @@ def _box(box: Box) -> Box: return float(x0), float(y0), float(x1), float(y1) +_QUAD = 0x1 +_LTR = 0x2 +_RTL = 0x4 +_DIRECTIONS = {"auto": 0, "ltr": _LTR, "rtl": _RTL} +_NO_QUAD = (0.0,) * 8 + + +def _quad(quad: Quad | None) -> tuple[int, tuple[float, ...]]: + if quad is None: + return 0, _NO_QUAD + points = tuple(quad) + if len(points) != 4: + raise InvalidArgumentError("a quad has four corners") + return _QUAD, tuple(float(v) for point in points for v in point) + + +def _line_flags(line: TextLine) -> tuple[int, tuple[float, ...]]: + flags, quad = _quad(line.quad) + try: + return flags | _DIRECTIONS[line.direction], quad + except KeyError: + raise InvalidArgumentError("direction must be 'auto', 'ltr' or 'rtl'") from None + + def _text(text: str) -> str: # The native side takes NUL-terminated UTF-8, so these could not reach it intact. if "\x00" in text: @@ -42,14 +66,17 @@ def write_text_layer( """Publishes a one-page PDF sized to the image, holding only invisible, extractable text. Each word is stretched to its box and gets a trailing space, except between CJK words, as - OCRmyPDF's renderer does. Right-to-left text raises UnsupportedError for now. On any failure + OCRmyPDF's renderer does. Right-to-left and mixed-direction words stay in logical order and + are placed by the Unicode bidi algorithm; quads place rotated or skewed lines. On any failure ``output`` is left untouched. """ - native_lines: list[tuple[Box, int]] = [] - native_words: list[tuple[str, Box]] = [] + native_lines: list[tuple[Box, int, int, tuple[float, ...]]] = [] + native_words: list[tuple[str, Box, int, tuple[float, ...]]] = [] for line in lines: - native_lines.append((_box(line.box), len(line.words))) - native_words.extend((_text(word.text), _box(word.box)) for word in line.words) + native_lines.append((_box(line.box), len(line.words), *_line_flags(line))) + native_words.extend( + (_text(word.text), _box(word.box), *_quad(word.quad)) for word in line.words + ) width, height = image_size dpi_x, dpi_y = (float(dpi), float(dpi)) if isinstance(dpi, (int, float)) else map(float, dpi) native().text_layer_write( diff --git a/tests/native/test_text_layer.py b/tests/native/test_text_layer.py index 22a1eee..fc5d1a6 100644 --- a/tests/native/test_text_layer.py +++ b/tests/native/test_text_layer.py @@ -1,3 +1,4 @@ +import math import threading import pytest @@ -74,14 +75,57 @@ def test_failed_write_keeps_the_destination(tmp_path): assert path.read_bytes() == b"previous" +def test_right_to_left_and_mixed_lines_read_back_in_logical_order(tmp_path): + lines = [ + TextLine((900, 300, 1600, 400), (TextWord("שלום", (1300, 300, 1600, 400)), + TextWord("עולם", (900, 300, 1250, 400)))), + TextLine((900, 500, 1600, 600), (TextWord("مَرْحَبًا", (1100, 500, 1600, 600)), + TextWord("يَا", (900, 500, 1050, 600)))), + TextLine((300, 700, 950, 800), (TextWord("see", (300, 700, 450, 800)), + TextWord("שלום", (500, 700, 700, 800)), + TextWord("here", (750, 700, 950, 800)))), + ] + path = write(tmp_path / "rtl.pdf", lines=lines) + with pyjpdfium.open_document(path) as document, document.page(0) as page: + assert squash(page.text()) == "שלום עולם مَرْحَبًا يَا see שלום here" + + +def rotated(box, degrees, centre): + a = math.radians(degrees) + cx, cy = centre + x0, y0, x1, y1 = box + return tuple((cx + (x - cx) * math.cos(a) - (y - cy) * math.sin(a), + cy + (x - cx) * math.sin(a) + (y - cy) * math.cos(a)) + for x, y in ((x0, y0), (x1, y0), (x1, y1), (x0, y1))) + + +@pytest.mark.parametrize("degrees", [15, 90, -90]) +def test_rotated_quad_lines_read_back(tmp_path, degrees): + centre = (1200, 1050) + words = [("rotated", (1000, 1000, 1350, 1100)), ("text", (1400, 1000, 1600, 1100))] + line = TextLine((1000, 1000, 1600, 1100), + tuple(TextWord(t, b, rotated(b, degrees, centre)) for t, b in words), + rotated((1000, 1000, 1600, 1100), degrees, centre)) + path = write(tmp_path / "rotated.pdf", lines=[line]) + with pyjpdfium.open_document(path) as document, document.page(0) as page: + assert squash(page.text()) == "rotated text" + with page.render(dpi=100, pixel_format="gray") as bitmap: + assert min(bitmap.tobytes()) == 255 + + def one_word(text, box=(0, 0, 100, 50)): return [TextLine(box, (TextWord(text, box),))] +MIRRORED = ((0, 50), (100, 50), (100, 0), (0, 0)) # anticlockwise: a mirror image + + @pytest.mark.parametrize( ("lines", "error"), [ - (one_word("שלום"), errors.UnsupportedError), # right-to-left is not laid out yet + (one_word("a\u202eb"), errors.UnsupportedError), # explicit overrides are refused + ([TextLine((0, 0, 100, 50), (TextWord("a", (0, 0, 100, 50)),), MIRRORED)], + errors.InvalidArgumentError), (one_word("a\tb"), errors.InvalidArgumentError), (one_word("a\x00b"), errors.InvalidArgumentError), (one_word("\ud800"), errors.InvalidArgumentError), diff --git a/tests/parity/requirements.txt b/tests/parity/requirements.txt index bd5a200..85e8420 100644 --- a/tests/parity/requirements.txt +++ b/tests/parity/requirements.txt @@ -15,3 +15,9 @@ packaging==26.2 \ fonttools==4.64.0 \ --hash=sha256:66a83f93579fb3493e458c4449d1d566a7b2a1c7b19915cd0fa3c9b8b5a8540b \ --hash=sha256:4a05783ff54ce4c7a28f18e5772efdf63c219374bd9ffc55452182e1cef8be60 +# Multilingual qualification: pdfminer.six reads text in content-stream order. +pdfminer.six==20260107 --hash=sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9 +charset-normalizer==3.5.2 --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 +cryptography==50.0.2 --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 +cffi==2.1.1 --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf +pycparser==3.11 --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 diff --git a/tests/parity/test_multilingual_parity.py b/tests/parity/test_multilingual_parity.py new file mode 100644 index 0000000..a5bbc4c --- /dev/null +++ b/tests/parity/test_multilingual_parity.py @@ -0,0 +1,339 @@ +"""Multilingual qualification: scripts, directions and rotations read back by independent tools. + +Opt-in like the other parity tests. Each layer is read by pypdfium2's PDFium, pdfminer.six and +poppler's pdftotext when present, and by our own PDFium. Text must reach the floor recorded for +that tool and layout in docs/COMPATIBILITY.md; character boxes must land in the writer's cells. +""" + +from __future__ import annotations + +import json +import math +import os +import shutil +import subprocess +import sys +import unicodedata +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +import pytest + +import pyjpdfium +from pyjpdfium import TextLine, TextWord + +from test_text_layer_parity import write_pdf + +SOURCE = os.environ.get("PYJPDFIUM_PARITY_SRC") +RUNNER = Path(__file__).with_name("text_layer_runner.py") +pytestmark = [ + pytest.mark.native, + pytest.mark.skipif(not SOURCE, reason="PYJPDFIUM_PARITY_SRC is not set"), +] + +DPI = 300.0 +SCALE = 72.0 / DPI +IMAGE = (2550, 3300) +CELL = 50.0 # px per code point, the glyphless font's own width on a 100 px line +GAP = 40.0 +HEIGHT = 100.0 + + +@dataclass(frozen=True) +class Sample: + name: str + text: str + kind: str # "ltr", "rtl" or "mixed" + rtl_line: bool = False + + +SAMPLES = [ + Sample("latin", "The quick brown fox", "ltr"), + Sample("greek", "Καλημέρα κόσμε", "ltr"), + Sample("cyrillic", "Привет мир", "ltr"), + Sample("chinese", "中文 文本", "ltr"), + Sample("hangul", "안녕하세요 세계", "ltr"), + Sample("kana", "ひらがな カタカナ", "ltr"), + Sample("devanagari", "नमस्ते दुनिया", "ltr"), + Sample("thai", "สวัสดี ชาวโลก", "ltr"), + Sample("latin_combining", "cafe\u0301 nai\u0308ve", "ltr"), + Sample("emoji", "\U0001f44d\U0001f3fd \U0001f468\u200d\U0001f469\u200d\U0001f467 " + "\U0001f1e9\U0001f1ea", "ltr"), + Sample("digits", "3.14 $100 (a)", "ltr"), + Sample("hebrew", "שלום עולם", "rtl", True), + Sample("hebrew_niqqud", "שָׁלוֹם עוֹלָם", "rtl", True), + Sample("arabic", "مرحبا بالعالم", "rtl", True), + Sample("arabic_harakat", "مَرْحَبًا يَا", "rtl", True), + Sample("persian_zwnj", "می\u200cخواهم بروم", "rtl", True), + Sample("hebrew_digits", "עמוד 12 מתוך 30", "mixed", True), + Sample("hebrew_word_digits", "עמוד12", "mixed", True), + Sample("arabic_indic_digits", "سنة ٢٠٢٤", "mixed", True), + Sample("rtl_brackets", "(שלום) [עולם]", "mixed", True), + Sample("ltr_with_hebrew", "see שלום here", "mixed"), + Sample("rtl_with_latin", "שלום world עולם", "mixed", True), + Sample("rtl_with_cjk", "שלום 日本 語", "mixed", True), # unspaced CJK in a right-to-left line +] +LAYOUTS = {"flat": (0.0, 0.0), "rot15": (15.0, 0.0), "rot90": (90.0, 0.0), + "rot180": (180.0, 0.0), "skew": (0.0, 30.0)} +RANK = {"lost": 0, "chars": 1, "words": 2, "nfc": 3, "exact": 4} + + +def cjk(text: str) -> bool: + return all(0x3000 <= ord(c) <= 0x9FFF or 0xAC00 <= ord(c) <= 0xD7AF for c in text) + + +def expected_text(sample: Sample) -> str: + words = sample.text.split(" ") + out = words[0] + for previous, word in zip(words, words[1:]): + out += ("" if cjk(previous) and cjk(word) else " ") + word + return out + + +def corners(box): + x0, y0, x1, y1 = box + return (x0, y0), (x1, y0), (x1, y1), (x0, y1) + + +def build(sample: Sample, angle: float, skew: float) -> TextLine: + """One line, words placed in reading order; rotated or skewed lines become quads.""" + words = sample.text.split(" ") + widths = [CELL * len(w) for w in words] + total = sum(widths) + GAP * (len(words) - 1) + x0, y0 = 400.0, 1500.0 + cx, cy = x0 + total / 2, y0 + HEIGHT / 2 + a = math.radians(angle) + + def quad(box): + bx0, by0, bx1, by1 = box + points = ((bx0 + skew, by0), (bx1 + skew, by0), (bx1, by1), (bx0, by1)) + return tuple((cx + (x - cx) * math.cos(a) - (y - cy) * math.sin(a), + cy + (x - cx) * math.sin(a) + (y - cy) * math.cos(a)) for x, y in points) + + placed = [] + x = x0 + total if sample.rtl_line else x0 + for text, width in zip(words, widths): + left = x - width if sample.rtl_line else x + box = (left, y0, left + width, y0 + HEIGHT) + x += -(width + GAP) if sample.rtl_line else width + GAP + placed.append((text, box)) + line_box = (x0, y0, x0 + total, y0 + HEIGHT) + if angle == 0 and skew == 0: + return TextLine(line_box, tuple(TextWord(t, b) for t, b in placed)) + return TextLine(line_box, tuple(TextWord(t, b, quad(b)) for t, b in placed), quad(line_box)) + + +def to_page(point): + return point[0] * SCALE, (IMAGE[1] - point[1]) * SCALE + + +def expected_cells(line: TextLine, rtl: bool): + """Where each emitted character's box centre must land, in page points: the writer contract. + + Words of one strong direction are checked character by character; mixed words, which the full + bidi algorithm lays out, are checked to stay inside their word. + """ + q = [to_page(p) for p in (line.quad or corners(line.box))] + u = ((q[1][0] - q[0][0] + q[2][0] - q[3][0]) / 2, (q[1][1] - q[0][1] + q[2][1] - q[3][1]) / 2) + w = ((q[0][0] - q[3][0] + q[1][0] - q[2][0]) / 2, (q[0][1] - q[3][1] + q[1][1] - q[2][1]) / 2) + size = math.hypot(*w) + u = (u[0] / math.hypot(*u), u[1] / math.hypot(*u)) + w = (w[0] / size, w[1] / size) + det = u[0] * w[1] - u[1] * w[0] + base = q[3] + + def along(p): + return ((p[0] - base[0]) * w[1] - (p[1] - base[1]) * w[0]) / det + + def point(s): + return base[0] + s * u[0] + size / 2 * w[0], base[1] + s * u[1] + size / 2 * w[1] + + spans = [] + for word in line.words: + values = [along(to_page(p)) for p in (word.quad or corners(word.box))] + spans.append((min(values), max(values))) + cells, loose = [], [] + for i, word in enumerate(line.words): + lo, hi = spans[i] + n = len(word.text) + cell = (hi - lo) / n + if i + 1 < len(line.words): + step = hi - spans[i + 1][1] if rtl else spans[i + 1][0] - lo + both = cjk(word.text) and cjk(line.words[i + 1].text) + space = step > 0 and not both + if both and 0 < step and step - (hi - lo) <= 2 * size: + cell *= step / (hi - lo) + else: + space = not word.text.isspace() and not cjk(word.text) + start = hi - n * cell if rtl else lo + classes = {unicodedata.bidirectional(c) for c in word.text} - {"NSM"} + if "R" in classes or "AL" in classes or "AN" in classes: + order = range(n - 1, -1, -1) if classes <= {"R", "AL"} else None + else: + order = range(n) if not rtl or classes <= {"L"} else None + if order is None: + loose.extend((c, point(start + n * cell / 2), n * cell / 2) for c in word.text) + else: + cells.extend((word.text[k], point(start + (slot + 0.5) * cell), cell) + for k, slot in zip(range(n), order)) + if space: + s = start - cell / 2 if rtl else start + (n + 0.5) * cell + cells.append((" ", point(s), cell)) + return cells, loose + + +def reached(got: str, sample: Sample) -> str: + """exact, then equal after NFC, then the same words in any order, then the same characters.""" + got, want = " ".join(got.split()), expected_text(sample) + if got == want: + return "exact" + nfc_got, nfc_want = (unicodedata.normalize("NFC", s) for s in (got, want)) + if nfc_got == nfc_want: + return "nfc" + words = [unicodedata.normalize("NFC", w) for w in sample.text.split(" ")] + if sorted(nfc_got.split()) in (sorted(nfc_want.split()), sorted(words)): + return "words" + nfd_got, nfd_want = (unicodedata.normalize("NFD", s).replace(" ", "") for s in (got, want)) + if Counter(nfd_got) == Counter(nfd_want): + return "chars" + return "lost" + + +def floor(tool: str, sample: Sample, layout: str) -> str: + """The weakest result each tool showed in the qualification run (docs/COMPATIBILITY.md).""" + rotated = layout in ("rot15", "rot90", "rot180") + if tool == "pdfminer": + # pdfminer reads stream order, but splits vertical text into single characters. + if layout == "rot90" or (sample.kind == "mixed" and layout == "rot180"): + return "chars" + return "words" if sample.kind == "mixed" else "exact" + if sample.kind == "mixed": + return "chars" # PDFium and poppler re-derive order and spaces at a direction change + if tool == "poppler": + if layout == "rot15" or (layout == "rot180" and sample.kind == "rtl"): + return "chars" # poppler splits words on slanted or upside-down right-to-left lines + return "exact" + # PDFium orders the words of upside-down lines, and of rotated right-to-left ones, by page + # position; it can also reorder Hebrew points, which NFC undoes. + if sample.kind == "rtl": + return "words" if rotated else "nfc" + return "words" if layout == "rot180" else "exact" + + +def run(*args: str): + env = {k: v for k, v in os.environ.items() if not k.startswith("PYJPDFIUM")} + done = subprocess.run([sys.executable, "-I", str(RUNNER), *args], capture_output=True, + text=True, env=env, check=False) + assert done.returncode == 0, done.stderr + return json.loads(done.stdout.strip().splitlines()[-1]) + + +@dataclass +class Case: + sample: Sample + layout: str + line: TextLine + path: Path + texts: dict + chars: list + + +@pytest.fixture(scope="module") +def cases(tmp_path_factory) -> list[Case]: + root = tmp_path_factory.mktemp("multilingual") + built = [] + for sample in SAMPLES: + for layout, (angle, skew) in LAYOUTS.items(): + line = build(sample, angle, skew) + path = root / f"{sample.name}-{layout}.pdf" + pyjpdfium.write_text_layer(path, [line], image_size=IMAGE, dpi=DPI) + built.append((sample, layout, line, path)) + extracted = run("extract", *(str(path) for *_, path in built)) + pdftotext = shutil.which("pdftotext") + out = [] + for (sample, layout, line, path), found in zip(built, extracted): + with pyjpdfium.open_document(path) as document, document.page(0) as page: + texts = {"ours": page.text(), "pypdfium2": found["pdfium"]} + if found["pdfminer"] is not None: + texts["pdfminer"] = found["pdfminer"] + if pdftotext: + raw = subprocess.run([pdftotext, "-enc", "UTF-8", str(path), "-"], capture_output=True, + text=True, check=True).stdout + texts["poppler"] = "".join(c for c in raw if c not in "\u202a\u202b\u202c\u202d\u202e") + out.append(Case(sample, layout, line, path, texts, found["chars"])) + return out + + +@pytest.mark.parametrize("tool", ["ours", "pypdfium2", "pdfminer", "poppler"]) +def test_extracted_text_reaches_its_floor(cases, tool): + if tool not in cases[0].texts: + pytest.skip(f"{tool} is not installed") + misses = [] + for case in cases: + got = reached(case.texts[tool], case.sample) + need = floor(tool, case.sample, case.layout) + if RANK[got] < RANK[need]: + misses.append(f"{case.sample.name}/{case.layout}: {got} < {need}: " + f"{case.texts[tool]!r}") + assert not misses, "\n".join(misses) + + +def test_horizontal_right_to_left_lines_are_exact_where_it_matters(cases): + # The gate's core claim: logical text from PDFium and stream-order readers alike. + for case in cases: + if case.sample.kind == "rtl" and case.layout in ("flat", "skew"): + for tool in ("ours", "pypdfium2", "pdfminer"): + if tool in case.texts: + assert reached(case.texts[tool], case.sample) in ("exact", "nfc"), ( + tool, case.sample.name, case.layout) + + +def test_characters_land_in_their_cells(cases): + misses = [] + for case in cases: + cells, loose = expected_cells(case.line, case.sample.rtl_line) + found = [(c[0], ((c[2] + c[4]) / 2, (c[3] + c[5]) / 2)) for c in case.chars if not c[1]] + # A cell's centre within a quarter cell; a mixed word's characters within the word. + targets = [(c, p, cell / 4) for c, p, cell in cells] + loose + if case.sample.kind == "mixed": + # PDFium merges or drops spaces at a direction change, so only letters are held. + found = [f for f in found if not f[0].isspace()] + targets = [t for t in targets if not t[0].isspace()] + if len(found) != len(targets): + misses.append(f"{case.sample.name}/{case.layout}: {len(found)} characters, " + f"want {len(targets)}") + continue + unused = list(found) + for char, (x, y), limit in targets: + hit = next((f for f in unused if f[0] == char and + math.hypot(f[1][0] - x, f[1][1] - y) <= limit), None) + if hit is None: + misses.append(f"{case.sample.name}/{case.layout}: {char!r} not at its cell") + else: + unused.remove(hit) + assert not misses, "\n".join(misses) + + +def test_multilingual_layers_draw_nothing_and_survive_ocrmypdfs_graft(tmp_path): + lines = [build(SAMPLES[11], 0, 0), build(SAMPLES[0], 90, 0), build(SAMPLES[13], 0, 30)] + # Spread the three lines down the page so they never overlap. + moved = [] + for n, line in enumerate(lines): + dy = (n - 1) * 600.0 + shift = (lambda q: tuple((x, y + dy) for x, y in q)) if line.quad else None + words = tuple(TextWord(w.text, (w.box[0], w.box[1] + dy, w.box[2], w.box[3] + dy), + shift(w.quad) if shift else None) for w in line.words) + box = (line.box[0], line.box[1] + dy, line.box[2], line.box[3] + dy) + moved.append(TextLine(box, words, shift(line.quad) if shift else None)) + layer = tmp_path / "layer.pdf" + pyjpdfium.write_text_layer(layer, moved, image_size=IMAGE, dpi=DPI) + before = run("probe", str(layer)) + assert before["blank"], "the layer must not draw anything" + base = write_pdf(tmp_path / "base.pdf", "/MediaBox [0 0 612 792]", "0.5 g 0 0 150 80 re f") + output = tmp_path / "grafted.pdf" + request = {"base": str(base), "text": str(layer), "output": str(output)} + assert run("graft", SOURCE, json.dumps(request))["grafted"] + after = run("probe", str(output)) + assert " ".join(after["text"].split()) == " ".join(before["text"].split()) + assert after["render"] == run("probe", str(base))["render"], "the graft changed the page" diff --git a/tests/parity/text_layer_runner.py b/tests/parity/text_layer_runner.py index 0ca51f1..6133916 100644 --- a/tests/parity/text_layer_runner.py +++ b/tests/parity/text_layer_runner.py @@ -1,6 +1,7 @@ """Reads or grafts a text layer with tools independent of JPDFium, in its own process. Usage: python -I text_layer_runner.py probe PDF [TERM ...] # pypdfium2: text, boxes, render + python -I text_layer_runner.py extract PDF [PDF ...] # pypdfium2 and pdfminer.six text python -I text_layer_runner.py font PDF # qpdf check, fontTools font parse python -I text_layer_runner.py graft SOURCE_DIR REQUEST # OCRmyPDF's own sandwich graft Prints one JSON object. graft executes OCRmyPDF's _graft.py with the rest of OCRmyPDF stubbed. @@ -54,6 +55,42 @@ def probe(path: str, terms: list[str]) -> dict: return result +def extract(paths: list[str]) -> list[dict]: + """pypdfium2's text and character boxes, and pdfminer.six's text when it is installed.""" + import pypdfium2 as pdfium + import pypdfium2.raw as raw + + try: + from pdfminer.high_level import extract_text + except ImportError: + extract_text = None + results = [] + for path in paths: + pdf = pdfium.PdfDocument(path) + page = pdf[0] + text = page.get_textpage() + chars, count, i = [], text.count_chars(), 0 + while i < count: + code = raw.FPDFText_GetUnicode(text.raw, i) + generated = bool(raw.FPDFText_IsGenerated(text.raw, i)) + left, bottom, right, top = text.get_charbox(i, loose=True) + if 0xD800 <= code < 0xDC00 and i + 1 < count: # PDFium indexes UTF-16 units + low = raw.FPDFText_GetUnicode(text.raw, i + 1) + code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00) + i += 1 + chars.append([chr(code), generated, left, bottom, right, top]) + i += 1 + results.append({ + "pdfium": text.get_text_range(), + "chars": chars, + "pdfminer": extract_text(path) if extract_text else None, + }) + text.close() + page.close() + pdf.close() + return results + + def font(path: str) -> dict: import pikepdf from fontTools.ttLib import TTFont @@ -151,6 +188,8 @@ def main() -> None: command, args = sys.argv[1], sys.argv[2:] if command == "probe": result = probe(args[0], args[1:]) + elif command == "extract": + result = extract(args) elif command == "font": result = font(args[0]) elif command == "graft": diff --git a/tests/unit/test_ocr_models.py b/tests/unit/test_ocr_models.py index 1a20b6a..109b0cd 100644 --- a/tests/unit/test_ocr_models.py +++ b/tests/unit/test_ocr_models.py @@ -21,4 +21,7 @@ def test_text_lines_keep_each_line_whole(): # Word geometry is absent: a line is one word over its box, never split evenly. assert [tl.words[0].text for tl in text_lines] == ["hello world", "next"] assert text_lines[0].words[0].box == text_lines[0].box == (10, 10, 200, 40) + # The polygon places the line, so rotated or skewed lines keep their slope. + quad = ((10, 10), (200, 10), (200, 40), (10, 40)) + assert text_lines[0].quad == text_lines[0].words[0].quad == quad assert r.text == "hello world\n \nnext" diff --git a/tests/unit/test_text_layer_models.py b/tests/unit/test_text_layer_models.py index 79b7474..fca55a0 100644 --- a/tests/unit/test_text_layer_models.py +++ b/tests/unit/test_text_layer_models.py @@ -16,6 +16,19 @@ def test_text_that_cannot_cross_the_c_abi_is_refused_in_python(tmp_path, text): write_text_layer(tmp_path / "x.pdf", one_word(text), image_size=(100, 100), dpi=300) +@pytest.mark.parametrize( + "line", + [ + TextLine((0, 0, 1, 1), (TextWord("a", (0, 0, 1, 1)),), direction="up"), + TextLine((0, 0, 1, 1), (TextWord("a", (0, 0, 1, 1)),), quad=((0, 0), (1, 0), (1, 1))), + TextLine((0, 0, 1, 1), (TextWord("a", (0, 0, 1, 1), ((0, 0),)),)), + ], +) +def test_bad_directions_and_quads_are_refused_in_python(tmp_path, line): + with pytest.raises(errors.InvalidArgumentError): + write_text_layer(tmp_path / "x.pdf", [line], image_size=(100, 100), dpi=300) + + def test_models_are_frozen(): word = TextWord("a", (0, 0, 1, 1)) with pytest.raises(AttributeError): From 74d61bb0bf013bc479a153be43fe39c29cb9f4fe Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:05:38 +0100 Subject: [PATCH 08/10] Add the pyjpdfium command and an OCRmyPDF plugin with end-to-end tests --- .github/workflows/ci.yml | 17 +- README.md | 22 +- docs/ARCHITECTURE.md | 6 +- docs/COMPATIBILITY.md | 6 +- docs/OCRMYPDF.md | 74 +++++++ pyproject.toml | 4 + src/pyjpdfium/__main__.py | 7 + src/pyjpdfium/_loader.py | 47 +++- src/pyjpdfium/cli.py | 268 +++++++++++++++++++++++ src/pyjpdfium/ocrmypdf.py | 284 +++++++++++++++++++++++++ src/pyjpdfium/profiles.py | 76 +++++++ tests/native/test_cli.py | 101 +++++++++ tests/native/test_coexistence.py | 52 +++++ tests/ocrmypdf/mask_plugin.py | 12 ++ tests/ocrmypdf/requirements.txt | 65 ++++++ tests/ocrmypdf/test_ocrmypdf_plugin.py | 259 ++++++++++++++++++++++ tests/unit/test_lazy_import.py | 2 +- tests/unit/test_profiles_and_cli.py | 50 +++++ tools/stage_natives.py | 103 +++++++++ 19 files changed, 1438 insertions(+), 17 deletions(-) create mode 100644 docs/OCRMYPDF.md create mode 100644 src/pyjpdfium/__main__.py create mode 100644 src/pyjpdfium/cli.py create mode 100644 src/pyjpdfium/ocrmypdf.py create mode 100644 src/pyjpdfium/profiles.py create mode 100644 tests/native/test_cli.py create mode 100644 tests/native/test_coexistence.py create mode 100644 tests/ocrmypdf/mask_plugin.py create mode 100644 tests/ocrmypdf/requirements.txt create mode 100644 tests/ocrmypdf/test_ocrmypdf_plugin.py create mode 100644 tests/unit/test_profiles_and_cli.py create mode 100644 tools/stage_natives.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 131f9ad..a457217 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,8 @@ jobs: JPDFIUM_INCLUDE_DIR: ${{ github.workspace }}/jpdfium/native/bridge/include run: | python -m pip install "scikit-build-core>=0.11" "nanobind>=2.9" "pytest>=8" - python -m pip install --no-build-isolation . + python -m pip wheel --no-build-isolation --no-deps . -w "$RUNNER_TEMP/wheel" + python -m pip install --no-deps "$RUNNER_TEMP"/wheel/pyjpdfium-*.whl - name: Python tests (unit + native) env: @@ -126,3 +127,17 @@ jobs: "$RUNNER_TEMP/ocr-reference/bin/pip" install --require-hashes --only-binary=:all: \ -r tests/parity/ocr-requirements.txt python -m pytest tests/parity/test_ocr_parity.py -q -rs + + - name: OCRmyPDF end to end with the pyjpdfium plugin + env: + PYJPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile + PYJPDFIUM_OCRMYPDF_PYTHON: ${{ runner.temp }}/ocrmypdf/bin/python + run: | + # OCRmyPDF imports pypdfium2, so our natives need their own sonames in that process. + python tools/stage_natives.py jpdfium/native/dist/linux-x64 "$RUNNER_TEMP/natives" + export PYJPDFIUM_NATIVE_DIR="$RUNNER_TEMP/natives" + python -m venv "$RUNNER_TEMP/ocrmypdf" + "$RUNNER_TEMP/ocrmypdf/bin/pip" install --require-hashes --only-binary=:all: \ + -r tests/ocrmypdf/requirements.txt + "$RUNNER_TEMP/ocrmypdf/bin/pip" install --no-deps "$RUNNER_TEMP"/wheel/pyjpdfium-*.whl + python -m pytest tests/ocrmypdf tests/native/test_cli.py -q -rs diff --git a/README.md b/README.md index 0a980ae..c89d2bc 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ lifetimes and handle validation, so the same rules hold for the Java and Python Status: early development. Covered so far: documents, pages, rendering, page text, transactional PNG output, OCRmyPDF-compatible page rasterization, perspective rectification, invisible text -layers in any script and direction, and OCR with PP-OCR models. +layers in any script and direction, OCR with PP-OCR models, a command line, and an OCRmyPDF +plugin that needs neither Tesseract nor Ghostscript. ```python import pyjpdfium @@ -38,6 +39,21 @@ with pyjpdfium.OcrEngine("pp-ocrv6-small/") as engine: dpi=300) ``` +## Command line and OCRmyPDF + +```bash +pyjpdfium info [input.pdf] # runtime identity and capabilities, or a PDF's pages +pyjpdfium render input.pdf page.png --page 1 --dpi 300 +pyjpdfium ocr input.pdf --profile pp-ocrv6-small/ --layer text.pdf +pyjpdfium fonts # text layer fonts +pyjpdfium models pp-ocrv6-small/ # check a profile against its pinned digests +pyjpdfium doctor # check the installation; recognizes nothing + +ocrmypdf --plugin pyjpdfium.ocrmypdf --output-type pdf input.pdf output.pdf +``` + +The plugin, what it refuses and why are in [docs/OCRMYPDF.md](docs/OCRMYPDF.md). + ## Behaviour - `import pyjpdfium` loads no native library. The core loads and starts on first use. @@ -72,6 +88,8 @@ Parity with OCRmyPDF's own rasterizer and sandwich graft is opt-in: install `tests/parity/requirements.txt` (hash-pinned), set `PYJPDFIUM_PARITY_SRC` to the output of `tests/parity/fetch_reference.py`, and run `pytest tests/parity`. OCR tests need `PYJPDFIUM_OCR_PROFILE`; OCR parity also needs `PYJPDFIUM_OCR_REFERENCE`, a Python with -`tests/parity/ocr-requirements.txt` installed. +`tests/parity/ocr-requirements.txt` installed. The OCRmyPDF tests need +`PYJPDFIUM_OCRMYPDF_PYTHON`, a Python with `tests/ocrmypdf/requirements.txt` and pyjpdfium +installed, and natives staged by `tools/stage_natives.py`. See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ac5f419..3fb8e12 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -21,6 +21,8 @@ pyjpdfium pure Python API: Document, Page, Bitmap, OcrEngine, runti pyjpdfium._loader lazy load, core pinning, identity checks, runtime init pyjpdfium._native nanobind module: one function per jpdf_* call, GIL released around it pyjpdfium._ocr_native nanobind module over jpdfium_ocr.h, loaded on the first OcrEngine +pyjpdfium.cli the pyjpdfium command; pyjpdfium.profiles checks model profiles in Python +pyjpdfium.ocrmypdf OCRmyPDF plugin, imported only by ocrmypdf --plugin libjpdfium the shared native core (never compiled into the extension) libjpdfium_ocr the optional OCR module, linked to the core, with ONNX Runtime ``` @@ -29,7 +31,9 @@ libjpdfium_ocr the optional OCR module, linked to the core, with ONNX Ru runs `_loader.native()`, which: 1. Loads the core from `PYJPDFIUM_NATIVE_DIR` or the bundled `pyjpdfium/_libs`, by exact path, so - the extension binds to that copy. + the extension binds to that copy. It loads with `RTLD_LOCAL`, and refuses natives whose + `libpdfium.so` or `libonnxruntime.so.1` would collide with one already loaded (pypdfium2, + onnxruntime); `tools/stage_natives.py` renames ours. 2. Checks the core's API version and struct layouts against the compiled header, and refuses to continue if more than one `libjpdfium` is mapped into the process. 3. Calls `jpdf_runtime_init` (explicit lazy initialization). diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index c4eb129..2938345 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -71,9 +71,13 @@ the same text, and confidences within 1.2e-7 (JPDFium's `docs/design/OCR.md`). configuration. - The OCR extension is built only when the staged natives include `libjpdfium_ocr`. +## OCRmyPDF + +`pyjpdfium.ocrmypdf` supports OCRmyPDF 17.13.x on Linux with staged natives, `--output-type pdf` +and English. [OCRMYPDF.md](OCRMYPDF.md) lists every refused option. + ## Not in this release -- The OCRmyPDF plugin. - PDF/A output: rejected by design. - Opening from bytes or file-like objects, saving or editing documents. - Free-threaded CPython builds are untested. diff --git a/docs/OCRMYPDF.md b/docs/OCRMYPDF.md new file mode 100644 index 0000000..6e2cb0c --- /dev/null +++ b/docs/OCRMYPDF.md @@ -0,0 +1,74 @@ +# OCRmyPDF plugin + +`pyjpdfium.ocrmypdf` runs OCRmyPDF 17.13 with JPDFium's native rasterizer, its PP-OCR engine +and its invisible text layer writer. Tesseract and Ghostscript are not needed. + +```bash +pip install "pyjpdfium[ocrmypdf]" +export PYJPDFIUM_OCR_PROFILE=/path/to/pp-ocrv6-small # fetched with fetch_profile.py +ocrmypdf --plugin pyjpdfium.ocrmypdf --output-type pdf input.pdf output.pdf +``` + +The plugin is loaded only by `--plugin`; installing PyJPDFium never changes how OCRmyPDF runs +without it. + +## What it does + +- Rasterization: with `--rasterizer auto`, pages are rasterized by `rasterize_page`, which matches + OCRmyPDF's own pypdfium rasterizer (JPDFium `docs/design/RASTER.md`). `--rasterizer pypdfium` + or `ghostscript` hands rasterization back to OCRmyPDF. +- Engine: the plugin is the OCR engine for `--ocr-engine auto` and for `--ocr-engine pyjpdfium`, + a value it registers. `--ocr-engine none` keeps OCRmyPDF's null engine. +- Text: with `--pdf-renderer auto` or `sandwich`, `generate_pdf` recognizes the image and writes + the text layer natively. `--pdf-renderer fpdf2` passes the same lines to OCRmyPDF's own + renderer through `generate_ocr`, as a reference. +- Exact image: the engine reads only the image OCRmyPDF hands it, after every filter, never the + source page. A test plugin that blanks part of the OCR image proves the hidden text is not + found while the visible page keeps it. +- Engines load once per worker process and are shared by its threads. ONNX Runtime uses one + thread per page by default (`--pyjpdfium-threads`), since OCRmyPDF already runs pages in + parallel. + +The plugin blocks OCRmyPDF's built-in Tesseract plugin, so Tesseract's checks and options do not +apply. OCRmyPDF's own validation still reads `options.tesseract`, so the plugin registers that +option model with its defaults. + +## Refused + +Anything not yet checked end to end is refused before any page is processed: + +| Request | Why | +|---|---| +| `--output-type` other than `pdf` or `none` (the default `auto` included) | PDF/A output is not produced | +| `--deskew`, `--rotate-pages` | The profile has no skew or orientation model | +| `--clean`, `--clean-final`, `--unpaper-args`, `--remove-background` | Not checked with this engine | +| `--ocr-engine tesseract`, any Tesseract option | Tesseract is not the engine | +| `-l` other than `eng` | Only checked languages are reported; the models read more | +| An unfetched or modified profile | Every file must match its pinned digest | +| An OCRmyPDF release other than 17.13.x | The plugin API is checked against that release only | + +## Sharing a process with pypdfium2 + +OCRmyPDF imports pypdfium2, whose `libpdfium.so` has the same soname as JPDFium's PDFium. +`tools/stage_natives.py` stages JPDFium's natives with their own sonames +(`libjpdfium_pdfium.so`, `libjpdfium_onnxruntime.so.1`), and the loader loads the core with +`RTLD_LOCAL`, so neither PDFium's symbols reach the other. Unstaged natives are refused with an +explicit error when a same-named library is already loaded. + +```bash +python tools/stage_natives.py /path/to/JPDFium/native/dist/linux-x64 natives +export PYJPDFIUM_NATIVE_DIR=$PWD/natives +``` + +## Evidence + +`tests/ocrmypdf` runs OCRmyPDF's real command line, installed from hash-pinned wheels into its +own environment, with no network and neither Tesseract nor Ghostscript installed: + +- a scanned page gains exactly its text, renders identically, and gets its `--sidecar` text; +- a `/Rotate 90` page reads upright; +- on a page that already has text, the default mode refuses it, `--skip-text` leaves it + untouched, and `--force-ocr` rasterizes and reads it; +- `--redo-ocr` replaces an earlier layer without doubling it; +- the exact-image check above, and the fpdf2 reference renderer reading the same words; +- every refusal in the table, and a tampered profile. diff --git a/pyproject.toml b/pyproject.toml index b75753f..769921c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,10 @@ dependencies = [] [project.optional-dependencies] test = ["pytest>=8"] +ocrmypdf = ["ocrmypdf>=17.13,<17.14"] + +[project.scripts] +pyjpdfium = "pyjpdfium.cli:main" [project.urls] Source = "https://github.com/Stirling-Tools/PyJPDFium" diff --git a/src/pyjpdfium/__main__.py b/src/pyjpdfium/__main__.py new file mode 100644 index 0000000..010c001 --- /dev/null +++ b/src/pyjpdfium/__main__.py @@ -0,0 +1,7 @@ +"""Runs the pyjpdfium command: python -m pyjpdfium.""" + +import sys + +from .cli import main + +sys.exit(main()) diff --git a/src/pyjpdfium/_loader.py b/src/pyjpdfium/_loader.py index db18c34..ff43e4a 100644 --- a/src/pyjpdfium/_loader.py +++ b/src/pyjpdfium/_loader.py @@ -15,6 +15,9 @@ NATIVE_DIR_ENV = "PYJPDFIUM_NATIVE_DIR" RENDERER_ENV = "PYJPDFIUM_RENDERER" +# Sonames other wheels also ship (pypdfium2, onnxruntime); tools/stage_natives.py renames ours. +SHARED_SONAMES = {"libpdfium.so": "libjpdfium_pdfium.so", + "libonnxruntime.so.1": "libjpdfium_onnxruntime.so.1"} _lock = threading.Lock() _module: ModuleType | None = None @@ -75,14 +78,44 @@ def _library_name(stem: str) -> str: return f"lib{stem}.so" +def _mapped_libraries() -> list[str]: + maps = Path("/proc/self/maps") + if not maps.exists(): + return [] + paths = set() + for line in maps.read_text().splitlines(): + parts = line.split(maxsplit=5) + if len(parts) == 6 and parts[5].startswith("/"): + paths.add(os.path.realpath(parts[5])) + return sorted(paths) + + +def _check_soname_clash(directory: Path) -> None: + """Refuses unstaged natives when another package already loaded a same-named library.""" + for soname in SHARED_SONAMES: + if not (directory / soname).exists(): + continue # staged, or not part of this build + ours = os.path.realpath(directory / soname) + mapped = [p for p in _mapped_libraries() if Path(p).name.startswith(soname)] + # Ours mapped means it loaded first, so later dependencies still resolve to it. + foreign = [p for p in mapped if p != ours] + if foreign and ours not in mapped: + raise NativeLoadError( + f"{foreign[0]} is already loaded and shares the soname {soname} with " + f"{directory}; stage the natives with tools/stage_natives.py" + ) + + def _pin_library(path: Path, what: str) -> None: if not path.is_file(): raise NativeLoadError(f"{what} not found at {path}; set {NATIVE_DIR_ENV} to staged natives") if sys.platform == "win32": _pins.append(os.add_dll_directory(str(path.parent))) return - # Loading this exact file first makes the extension bind to it rather than another copy. - _pins.append(ctypes.CDLL(str(path), mode=ctypes.RTLD_GLOBAL)) + _check_soname_clash(path.parent) + # Loading this file first binds the extension to it. RTLD_LOCAL keeps our PDFium's symbols + # out of the global scope, where another PDFium in the process could resolve to them. + _pins.append(ctypes.CDLL(str(path), mode=ctypes.RTLD_LOCAL)) def _pin_core(directory: Path) -> None: @@ -90,15 +123,7 @@ def _pin_core(directory: Path) -> None: def _loaded_core_paths() -> set[str]: - maps = Path("/proc/self/maps") - if not maps.exists(): - return set() - paths = set() - for line in maps.read_text().splitlines(): - parts = line.split(maxsplit=5) - if len(parts) == 6 and Path(parts[5]).name.startswith("libjpdfium.so"): - paths.add(os.path.realpath(parts[5])) - return paths + return {p for p in _mapped_libraries() if Path(p).name.startswith("libjpdfium.so")} def _check_identity(module: ModuleType) -> None: diff --git a/src/pyjpdfium/cli.py b/src/pyjpdfium/cli.py new file mode 100644 index 0000000..bb20166 --- /dev/null +++ b/src/pyjpdfium/cli.py @@ -0,0 +1,268 @@ +"""The ``pyjpdfium`` command: runtime identity, page rendering, OCR, profiles and a doctor. + +Exit status is 0 on success, 1 when a command fails or ``doctor`` finds a problem, and 2 for +usage errors. +""" + +from __future__ import annotations + +import argparse +import importlib +import json +import os +import platform +import shutil +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from . import __version__ +from .errors import PyJPDFiumError + +CORE_CAPABILITIES = ( + "runtime.lifecycle", "pdf.open", "pdf.render", "pdf.text", "pdf.raster", "image.load", + "image.png_write", "image.rectify", "output.transaction", "legacy.guards", + "text_layer.write", "text_layer.bidi", "text_layer.quad", + "fonts.shape_explicit", "fonts.subset", "pdf.optimize", "svg.render", +) +OCR_CAPABILITIES = ("ocr.recognize", "ocr.word_geometry") +TEXT_LAYER_FONTS = ( + ("glyphless", "built in", "invisible text in any script and direction; draws nothing"), +) + + +def _out(line: str = "") -> None: + print(line, flush=True) + + +def cmd_info(args: argparse.Namespace) -> int: + from . import capability, open_document, runtime_info + + info = runtime_info() + if args.pdf is None: + _out(f"pyjpdfium {__version__}") + _out(f"api {info.api_version[0]}.{info.api_version[1]} build {info.build_id}") + _out(f"renderer {info.renderer.name.lower() if info.renderer else 'auto'}") + for name in CORE_CAPABILITIES: + _out(f" {name:22s} {capability(name).name.lower()}") + return 0 + with open_document(args.pdf, password=args.password) as document: + _out(f"{args.pdf}: {document.page_count} pages") + for index in range(document.page_count): + with document.page(index) as page: + size = page.size + _out(f" page {index + 1}: {size.width:.2f} x {size.height:.2f} pt") + return 0 + + +def cmd_render(args: argparse.Namespace) -> int: + from . import rasterize_page + + result = rasterize_page(args.pdf, args.page - 1, args.output, device=args.device, + raster_dpi=args.dpi, password=args.password) + _out(f"{args.output}: {result.width} x {result.height} px") + return 0 + + +def _profile(args: argparse.Namespace) -> Path: + from .profiles import PROFILE_ENV, default_profile_dir + + directory = Path(args.profile) if args.profile else default_profile_dir() + if directory is None: + raise SystemExit(f"pyjpdfium: pass --profile DIR or set {PROFILE_ENV}") + return directory + + +def cmd_ocr(args: argparse.Namespace) -> int: + from . import OcrEngine, rasterize_page, write_text_layer + + source = Path(args.input) + with tempfile.TemporaryDirectory(prefix="pyjpdfium-ocr-") as work: + image = source + if source.suffix.lower() == ".pdf": + image = Path(work) / "page.png" + rasterize_page(source, args.page - 1, image, device="png16m", raster_dpi=args.dpi) + with OcrEngine(_profile(args), threads=args.threads) as engine: + result = engine.recognize(image) + if args.layer: + write_text_layer(args.layer, result.text_lines(), image_size=result.image_size, + dpi=result.dpi) + if args.json: + lines = [{"text": line.text, "polygon": line.polygon, "confidence": line.confidence} + for line in result.lines] + _out(json.dumps({"image_size": result.image_size, "dpi": result.dpi, + "profile": result.profile, "lines": lines}, ensure_ascii=False)) + else: + _out(result.text) + return 0 + + +def cmd_fonts(args: argparse.Namespace) -> int: + for name, source, description in TEXT_LAYER_FONTS: + _out(f"{name:10s} {source:9s} {description}") + return 0 + + +def cmd_models(args: argparse.Namespace) -> int: + from .profiles import default_profile_dir, verify_profile + + directories = [Path(d) for d in args.dirs] or [d for d in [default_profile_dir()] if d] + if not directories: + _out("no profile given and PYJPDFIUM_OCR_PROFILE is not set") + return 1 + failed = False + for directory in directories: + try: + profile = verify_profile(directory) + except PyJPDFiumError as exc: + _out(f"FAIL {directory}: {exc}") + failed = True + continue + _out(f"ok {profile.id} at {directory}") + for item in profile.files: + _out(f" {item.role:18s} {item.name:16s} sha256 {item.sha256[:16]}") + return 1 if failed else 0 + + +class _Report: + def __init__(self) -> None: + self.problems = 0 + + def ok(self, what: str, detail: str = "") -> None: + _out(f"ok {what}{': ' + detail if detail else ''}") + + def note(self, what: str, detail: str) -> None: + _out(f"note {what}: {detail}") + + def problem(self, what: str, detail: str) -> None: + self.problems += 1 + _out(f"PROBLEM {what}: {detail}") + + +def _loaded_libraries() -> list[str]: + from ._loader import _mapped_libraries + + names = ("libjpdfium", "libpdfium", "libonnxruntime") + return [p for p in _mapped_libraries() if Path(p).name.startswith(names)] + + +def cmd_doctor(args: argparse.Namespace) -> int: + """Reports identity, capabilities, profiles and dependencies; recognizes no document.""" + from . import _loader + from .profiles import default_profile_dir, sha256_file, verify_profile + + report = _Report() + report.ok("pyjpdfium", f"{__version__} on Python {platform.python_version()} " + f"({platform.system()} {platform.machine()})") + directory = _loader.native_dir() + report.ok("natives", str(directory) if directory else "found through the loader path") + try: + from . import capability, runtime_info + + info = runtime_info() + except PyJPDFiumError as exc: + report.problem("core", str(exc)) + return 1 + report.ok("core", f"api {info.api_version[0]}.{info.api_version[1]}, build {info.build_id}, " + f"renderer {info.renderer.name.lower() if info.renderer else 'auto'}") + for name in CORE_CAPABILITIES: + report.ok(f"capability {name}", capability(name).name.lower()) + try: + ocr = _loader.ocr_native() + except PyJPDFiumError as exc: + report.note("ocr module", f"not available ({exc})") + ocr = None + if ocr is not None: + version = ocr.api_version() + report.ok("ocr module", f"api {version >> 16}.{version & 0xFFFF}") + for name in OCR_CAPABILITIES: + level = {0: "absent", 1: "implemented", 2: "qualified"}.get(ocr.capability(name), "?") + report.ok(f"capability {name}", level) + # Listed once both modules are in, so the identities cover everything that loaded. + for path in _loaded_libraries(): + report.ok("loaded", f"{path} sha256 {sha256_file(Path(path))[:16]}") + profile_dir = Path(args.profile) if args.profile else default_profile_dir() + if profile_dir is None: + report.note("ocr profile", "none given; pass --profile or set PYJPDFIUM_OCR_PROFILE") + else: + try: + profile = verify_profile(profile_dir) + except PyJPDFiumError as exc: + report.problem("ocr profile", str(exc)) + else: + report.ok("ocr profile", f"{profile.id} at {profile_dir}, every digest matches") + for name, source, _ in TEXT_LAYER_FONTS: + report.ok(f"font {name}", source) + try: + plugin = importlib.import_module("pyjpdfium.ocrmypdf") + plugin.check_compatible() + except ImportError as exc: + report.note("ocrmypdf", f"not installed; the plugin is optional ({exc.name})") + except Exception as exc: # noqa: BLE001 - OCRmyPDF's own exception types + report.problem("ocrmypdf", str(exc)) + else: + import ocrmypdf + + report.ok("ocrmypdf", f"{ocrmypdf.__version__}, supported by pyjpdfium.ocrmypdf") + for tool in ("tesseract", "gs"): + found = shutil.which(tool) + report.note(tool, f"{'found' if found else 'not found'}; not needed by pyjpdfium") + _out(f"{report.problems} problem(s)") + return 1 if report.problems else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="pyjpdfium", description=__doc__.splitlines()[0]) + parser.add_argument("--version", action="version", version=f"pyjpdfium {__version__}") + sub = parser.add_subparsers(dest="command", required=True) + + info = sub.add_parser("info", help="runtime identity and capabilities, or a PDF's pages") + info.add_argument("pdf", nargs="?") + info.add_argument("--password") + info.set_defaults(run=cmd_info) + + render = sub.add_parser("render", help="rasterize one page to PNG") + render.add_argument("pdf") + render.add_argument("output") + render.add_argument("--page", type=int, default=1, help="1-based (default 1)") + render.add_argument("--dpi", type=float, default=300.0) + render.add_argument("--device", default="png16m", + choices=["png16m", "pnggray", "pngmono", "pngmonod", "png256"]) + render.add_argument("--password") + render.set_defaults(run=cmd_render) + + ocr = sub.add_parser("ocr", help="recognize a PNG, or one page of a PDF") + ocr.add_argument("input") + ocr.add_argument("--page", type=int, default=1, help="1-based, for PDF input (default 1)") + ocr.add_argument("--dpi", type=float, default=300.0, help="raster DPI for PDF input") + ocr.add_argument("--profile", help="profile directory (default $PYJPDFIUM_OCR_PROFILE)") + ocr.add_argument("--threads", type=int, default=0, help="ONNX Runtime threads (0: auto)") + ocr.add_argument("--layer", metavar="PDF", help="also write an invisible text layer") + ocr.add_argument("--json", action="store_true", help="lines with polygons and confidences") + ocr.set_defaults(run=cmd_ocr) + + fonts = sub.add_parser("fonts", help="fonts available to text layers") + fonts.set_defaults(run=cmd_fonts) + + models = sub.add_parser("models", help="verify OCR profile directories against their digests") + models.add_argument("dirs", nargs="*", metavar="DIR") + models.set_defaults(run=cmd_models) + + doctor = sub.add_parser("doctor", help="check this installation without recognizing anything") + doctor.add_argument("--profile", help="profile directory (default $PYJPDFIUM_OCR_PROFILE)") + doctor.set_defaults(run=cmd_doctor) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return args.run(args) + except PyJPDFiumError as exc: + print(f"pyjpdfium {args.command}: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/pyjpdfium/ocrmypdf.py b/src/pyjpdfium/ocrmypdf.py new file mode 100644 index 0000000..0182270 --- /dev/null +++ b/src/pyjpdfium/ocrmypdf.py @@ -0,0 +1,284 @@ +"""OCRmyPDF plugin: JPDFium's rasterizer, PP-OCR engine and native text layer writer. + +Load it explicitly; it is never picked up on its own: + + ocrmypdf --plugin pyjpdfium.ocrmypdf --output-type pdf in.pdf out.pdf + +It replaces Tesseract, so Tesseract need not be installed, and refuses every mode it has not +been checked in rather than falling back. docs/OCRMYPDF.md lists what is supported. +""" + +from __future__ import annotations + +import logging +import math +import os +import threading +from pathlib import Path +from typing import TYPE_CHECKING + +import ocrmypdf +from ocrmypdf import hookimpl +from ocrmypdf.exceptions import BadArgsError, MissingDependencyError +from ocrmypdf.hocrtransform import BoundingBox, OcrClass, OcrElement +from ocrmypdf.pluginspec import OcrEngine, OrientationConfidence +from pydantic import BaseModel + +import pyjpdfium + +from .errors import InvalidArgumentError +from .profiles import PROFILE_ENV, verify_profile + +if TYPE_CHECKING: + from ocrmypdf._options import OcrOptions + +log = logging.getLogger(__name__) + +TESTED_OCRMYPDF = (17, 13) # every 17.13.x; other releases are refused +ENGINE_NAME = "pyjpdfium" +# Only languages checked end to end are reported; the models read more, unqualified. +LANGUAGES = frozenset({"eng"}) +_TESSERACT = "ocrmypdf.builtin_plugins.tesseract_ocr" +_TESSERACT_FIELDS = ( + "tesseract_config", "tesseract_pagesegmode", "tesseract_oem", "tesseract_thresholding", + "tesseract_timeout", "tesseract_non_ocr_timeout", "tesseract_downsample_above", + "tesseract_downsample_large_images", "user_words", "user_patterns", +) +# Preprocessing and orientation run outside the engine or need models this profile lacks. +_REFUSED_FLAGS = { + "deskew": "--deskew needs a skew estimate, which this engine does not qualify", + "rotate_pages": "--rotate-pages needs an orientation model, which this profile lacks", + "clean": "--clean (unpaper) has not been checked with this engine", + "clean_final": "--clean-final (unpaper) has not been checked with this engine", + "remove_background": "--remove-background has not been checked with this engine", +} + + +class PyjpdfiumOptions(BaseModel): + """Options under ``options.pyjpdfium``; on the command line, ``--pyjpdfium-*``.""" + + profile: str | None = None + threads: int = 1 # ONNX Runtime threads per page; OCRmyPDF already runs pages in parallel + + +def ocrmypdf_version() -> tuple[int, ...]: + return tuple(int(part) for part in ocrmypdf.__version__.split(".")[:3] if part.isdigit()) + + +def check_compatible() -> None: + if ocrmypdf_version()[:2] != TESTED_OCRMYPDF: + raise MissingDependencyError( + f"pyjpdfium.ocrmypdf supports OCRmyPDF {TESTED_OCRMYPDF[0]}.{TESTED_OCRMYPDF[1]}.x, " + f"not {ocrmypdf.__version__}" + ) + + +def profile_dir(options: OcrOptions) -> Path | None: + configured = options.pyjpdfium.profile or os.environ.get(PROFILE_ENV) + return Path(configured) if configured else None + + +# --- Registration and preflight. These run in OCRmyPDF's main process, before workers start, +# so they never load native code: forked workers then start from a clean runtime. + + +@hookimpl +def initialize(plugin_manager): + check_compatible() + # This plugin is the OCR engine, so Tesseract's checks and options must not apply. + plugin_manager.set_blocked(_TESSERACT) + + +@hookimpl +def register_options(): + from ocrmypdf.builtin_plugins.tesseract_ocr import TesseractOptions + + # OCRmyPDF's validation reads options.tesseract even when Tesseract is blocked. + return {"pyjpdfium": PyjpdfiumOptions, "tesseract": TesseractOptions} + + +@hookimpl +def add_options(parser): + group = parser.add_argument_group("PyJPDFium", "OCR with JPDFium's native PP-OCR engine") + group.add_argument( + "--pyjpdfium-profile", dest="pyjpdfium_profile", metavar="DIR", + help=f"Fetched OCR model profile directory (default: ${PROFILE_ENV})", + ) + group.add_argument( + "--pyjpdfium-threads", dest="pyjpdfium_threads", type=int, default=1, metavar="N", + help="ONNX Runtime threads per page (default 1; 0 lets ONNX Runtime choose)", + ) + for action in parser._actions: # noqa: SLF001 - argparse exposes no public lookup + if "--ocr-engine" in action.option_strings and ENGINE_NAME not in action.choices: + action.choices = [*action.choices, ENGINE_NAME] + + +@hookimpl +def check_options(options): + if options.ocr_engine == "tesseract": + raise BadArgsError("--ocr-engine tesseract cannot run with pyjpdfium.ocrmypdf loaded") + if options.output_type not in ("pdf", "none"): + raise BadArgsError( + "pyjpdfium.ocrmypdf does not produce PDF/A; pass --output-type pdf " + f"(got {options.output_type!r})" + ) + for flag, reason in _REFUSED_FLAGS.items(): + if getattr(options, flag): + raise BadArgsError(reason) + if options.unpaper_args: + raise BadArgsError("--unpaper-args has not been checked with this engine") + defaults = type(options).model_fields + changed = [f for f in _TESSERACT_FIELDS if getattr(options, f) != defaults[f].default] + if changed: + raise BadArgsError(f"Tesseract options do not apply to this engine: {', '.join(changed)}") + if options.pyjpdfium.threads < 0: + raise BadArgsError("--pyjpdfium-threads must not be negative") + if options.ocr_engine == "none": + return # OCRmyPDF's null engine; only this plugin's rasterizer is used + if options.pdf_renderer == "auto": + # The native writer is the qualified renderer; fpdf2 stays available on request. + options.pdf_renderer = "sandwich" + directory = profile_dir(options) + if directory is None: + raise BadArgsError(f"pass --pyjpdfium-profile DIR or set {PROFILE_ENV}") + try: + log.debug("pyjpdfium OCR profile %s verified", verify_profile(directory).id) + except InvalidArgumentError as exc: + raise BadArgsError(str(exc)) from None + + +# --- Work in OCRmyPDF's workers, which may be threads or processes. + + +@hookimpl +def rasterize_pdf_page( + input_file, output_file, raster_device, raster_dpi, pageno, page_dpi, rotation, + filter_vector, stop_on_soft_error, options, use_cropbox, +): + if options is not None and options.rasterizer != "auto": + return None # OCRmyPDF's own pypdfium or Ghostscript rasterizer was chosen explicitly + # Only --rotate-pages previews ask for JPEG, and that option is refused above. + pyjpdfium.rasterize_page( + input_file, + pageno - 1, + output_file, + device=str(raster_device), + raster_dpi=(float(raster_dpi.x), float(raster_dpi.y)), + output_dpi=(float(page_dpi.x), float(page_dpi.y)) if page_dpi else None, + rotation=rotation or 0, + use_cropbox=use_cropbox, + filter_vector=filter_vector, + ) + return output_file + + +@hookimpl +def filter_ocr_image(page, image): + # Keeps Tesseract's size limits, still registered in workers, off this engine's input. + return image + + +_engine_lock = threading.Lock() +_engine: tuple[int, tuple[str, int], pyjpdfium.OcrEngine] | None = None + + +def _get_engine(options: OcrOptions) -> pyjpdfium.OcrEngine: + """One engine per process, shared by its threads; a forked child opens its own.""" + global _engine + directory = profile_dir(options) + if directory is None: + raise BadArgsError(f"pass --pyjpdfium-profile DIR or set {PROFILE_ENV}") + key = (os.fspath(directory), options.pyjpdfium.threads) + with _engine_lock: + if _engine is None or _engine[0] != os.getpid() or _engine[1] != key: + _engine = (os.getpid(), key, pyjpdfium.OcrEngine(key[0], threads=key[1])) + return _engine[2] + + +def _recognize(input_file: Path, options: OcrOptions) -> pyjpdfium.OcrResult: + # The exact image OCRmyPDF supplies, never a re-render of the source page. + result = _get_engine(options).recognize(input_file) + if not all(d > 0 for d in result.dpi): + raise BadArgsError(f"{input_file} carries no resolution; OCRmyPDF always sets one") + return result + + +def _sidecar(result: pyjpdfium.OcrResult) -> str: + text = "\n".join(line.text for line in result.lines if line.text.strip()) + return text + "\n" if text else "" + + +class PyjpdfiumOcrEngine(OcrEngine): + @staticmethod + def version() -> str: + return pyjpdfium.__version__ + + @staticmethod + def creator_tag(options) -> str: + return f"PyJPDFium {pyjpdfium.__version__} PP-OCR" + + def __str__(self) -> str: + return f"PyJPDFium PP-OCR {pyjpdfium.__version__}" + + @staticmethod + def languages(options) -> frozenset[str]: + return LANGUAGES + + @staticmethod + def get_orientation(input_file, options) -> OrientationConfidence: + raise BadArgsError(_REFUSED_FLAGS["rotate_pages"]) + + @staticmethod + def get_deskew(input_file, options) -> float: + raise BadArgsError(_REFUSED_FLAGS["deskew"]) + + @staticmethod + def generate_hocr(input_file, output_hocr, output_text, options) -> None: + raise BadArgsError("pyjpdfium writes no hOCR; use --pdf-renderer sandwich or fpdf2") + + @staticmethod + def generate_pdf(input_file, output_pdf, output_text, options) -> None: + result = _recognize(Path(input_file), options) + pyjpdfium.write_text_layer( + output_pdf, result.text_lines(), image_size=result.image_size, dpi=result.dpi + ) + Path(output_text).write_text(_sidecar(result), encoding="utf-8") + + @staticmethod + def supports_generate_ocr() -> bool: + return True + + @staticmethod + def generate_ocr(input_file, options, page_number: int = 0): + """The same recognition as an OcrElement tree, for OCRmyPDF's own fpdf2 renderer.""" + result = _recognize(Path(input_file), options) + width, height = result.image_size + page = OcrElement( + ocr_class=OcrClass.PAGE, + bbox=BoundingBox(left=0, top=0, right=width, bottom=height), + dpi=float(result.dpi[0]), + page_number=page_number, + ) + for line in result.lines: + if not line.text.strip(): + continue + left, top, right, bottom = line.box + box = BoundingBox(left=left, top=top, right=right, bottom=bottom) + (x0, y0), (x1, y1) = line.polygon[0], line.polygon[1] + # Counterclockwise from horizontal, as hOCR's textangle; image y runs down. + angle = -math.degrees(math.atan2(y1 - y0, x1 - x0)) + # Word geometry is absent, so the line is one word; it is never split evenly. + word = OcrElement(ocr_class=OcrClass.WORD, bbox=box, text=line.text, + confidence=line.confidence) + page.children.append(OcrElement( + ocr_class=OcrClass.LINE, bbox=box, poly=list(line.polygon), + textangle=angle if abs(angle) >= 0.5 else None, children=[word], + )) + return page, _sidecar(result) + + +@hookimpl +def get_ocr_engine(options): + if options is not None and options.ocr_engine not in ("auto", ENGINE_NAME): + return None # "none" selects OCRmyPDF's null engine + return PyjpdfiumOcrEngine() diff --git a/src/pyjpdfium/profiles.py b/src/pyjpdfium/profiles.py new file mode 100644 index 0000000..7ea2eb6 --- /dev/null +++ b/src/pyjpdfium/profiles.py @@ -0,0 +1,76 @@ +"""OCR model profiles on disk: find one, and check its files against the digests it pins. + +Pure Python, so it runs before any native code loads. Opening an OcrEngine checks the same +digests again natively. +""" + +from __future__ import annotations + +import hashlib +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Union + +from .errors import InvalidArgumentError + +StrPath = Union[str, "os.PathLike[str]"] +PROFILE_ENV = "PYJPDFIUM_OCR_PROFILE" +MANIFEST = "profile.txt" + + +@dataclass(frozen=True) +class ProfileFile: + role: str + name: str + sha256: str + + +@dataclass(frozen=True) +class Profile: + id: str + directory: Path + files: tuple[ProfileFile, ...] + + +def default_profile_dir() -> Path | None: + configured = os.environ.get(PROFILE_ENV) + return Path(configured) if configured else None + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def read_profile(directory: StrPath) -> Profile: + """Parses a fetched profile directory's manifest without checking digests.""" + directory = Path(directory) + manifest = directory / MANIFEST + if not manifest.is_file(): + raise InvalidArgumentError(f"{directory} is not a fetched OCR profile: no {MANIFEST}") + profile_id, files = "", [] + for line in manifest.read_text(encoding="utf-8").splitlines(): + fields = line.split() + if fields[:1] == ["id"] and len(fields) == 2: + profile_id = fields[1] + elif fields[:1] in (["file"], ["derived"]) and len(fields) >= 4: + files.append(ProfileFile(fields[1], fields[2], fields[3])) + if not profile_id or not files: + raise InvalidArgumentError(f"{manifest} names no profile id or files") + return Profile(profile_id, directory, tuple(files)) + + +def verify_profile(directory: StrPath) -> Profile: + """Checks every pinned file; raises InvalidArgumentError naming the first that fails.""" + profile = read_profile(directory) + for item in profile.files: + path = profile.directory / item.name + if not path.is_file(): + raise InvalidArgumentError(f"OCR profile file {path} is missing") + if sha256_file(path) != item.sha256: + raise InvalidArgumentError(f"OCR profile file {path} does not match its pinned digest") + return profile diff --git a/tests/native/test_cli.py b/tests/native/test_cli.py new file mode 100644 index 0000000..cfccf11 --- /dev/null +++ b/tests/native/test_cli.py @@ -0,0 +1,101 @@ +import json +import os +import shutil +import subprocess +import sys + +import pytest + +import pyjpdfium + +pytestmark = pytest.mark.native +PROFILE = os.environ.get("PYJPDFIUM_OCR_PROFILE") +TEXT_PAGE = ( + b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n" + b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n" + b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 " + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> /Contents 4 0 R >>\nendobj\n" + b"4 0 obj\n<< /Length 44 >>\nstream\nBT /F1 28 Tf 72 700 Td (Command line) Tj ET\nendstream\n" + b"endobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n" +) + + +def cli(*args, env=None): + # A child process, as a user would run it; the package's own entry point. + return subprocess.run([sys.executable, "-m", "pyjpdfium", *args], capture_output=True, + text=True, env={**os.environ, **(env or {})}, check=False, timeout=300) + + +@pytest.fixture +def pdf(tmp_path): + path = tmp_path / "page.pdf" + path.write_bytes(TEXT_PAGE) + return path + + +def test_info_reports_identity_and_capabilities(): + result = cli("info") + assert result.returncode == 0, result.stderr + assert f"pyjpdfium {pyjpdfium.__version__}" in result.stdout + assert "text_layer.bidi" in result.stdout and "implemented" in result.stdout + + +def test_info_on_a_pdf_lists_its_pages(pdf): + result = cli("info", str(pdf)) + assert result.returncode == 0, result.stderr + assert "1 pages" in result.stdout and "612.00 x 792.00 pt" in result.stdout + + +def test_render_writes_a_png(tmp_path, pdf): + out = tmp_path / "page.png" + result = cli("render", str(pdf), str(out), "--dpi", "100", "--device", "pnggray") + assert result.returncode == 0, result.stderr + with pyjpdfium.load_image(out) as bitmap: + assert (bitmap.width, bitmap.height) == (850, 1100) + + +def test_failures_exit_1_with_a_message(tmp_path): + result = cli("info", str(tmp_path / "missing.pdf")) + assert result.returncode == 1 + assert result.stderr.startswith("pyjpdfium info:") + + +def test_fonts_lists_the_glyphless_font(): + result = cli("fonts") + assert result.returncode == 0 and result.stdout.startswith("glyphless") + + +def test_models_refuses_a_tampered_profile(tmp_path): + if not PROFILE: + pytest.skip("PYJPDFIUM_OCR_PROFILE is not set") + assert cli("models", PROFILE).returncode == 0 + copy = tmp_path / "profile" + shutil.copytree(PROFILE, copy) + (copy / "det.yml").write_text("tampered\n", encoding="utf-8") + result = cli("models", str(copy)) + assert result.returncode == 1 and "does not match its pinned digest" in result.stdout + + +def test_doctor_passes_without_recognizing_anything(): + result = cli("doctor") + assert result.returncode == 0, result.stdout + result.stderr + assert "0 problem(s)" in result.stdout + assert "capability pdf.render" in result.stdout + + +def test_doctor_flags_a_broken_profile(tmp_path): + (tmp_path / "profile.txt").write_text("format 1\n", encoding="utf-8") + result = cli("doctor", "--profile", str(tmp_path)) + assert result.returncode == 1 and "PROBLEM ocr profile" in result.stdout + + +def test_ocr_reads_a_page(tmp_path, pdf): + if not PROFILE: + pytest.skip("PYJPDFIUM_OCR_PROFILE is not set") + layer = tmp_path / "layer.pdf" + result = cli("ocr", str(pdf), "--layer", str(layer), "--json") + assert result.returncode == 0, result.stderr + found = json.loads(result.stdout) + assert [line["text"] for line in found["lines"]] == ["Command line"] + with pyjpdfium.open_document(layer) as document, document.page(0) as page: + assert " ".join(page.text().split()) == "Command line" diff --git a/tests/native/test_coexistence.py b/tests/native/test_coexistence.py new file mode 100644 index 0000000..fbd28dc --- /dev/null +++ b/tests/native/test_coexistence.py @@ -0,0 +1,52 @@ +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.native + +# pypdfium2 first, as OCRmyPDF imports it, then pyjpdfium in the same process. +PROBE = r""" +import sys +import pypdfium2 +import pyjpdfium +from pyjpdfium.errors import NativeLoadError + +pdf = pypdfium2.PdfDocument(sys.argv[1]) +theirs = pdf[0].render(scale=1).to_pil().size +try: + with pyjpdfium.open_document(sys.argv[1]) as document, document.page(0) as page: + ours = page.size +except NativeLoadError as exc: + print("refused:", exc) + raise SystemExit(0) +maps = open("/proc/self/maps").read() +pdfiums = sorted({line.split()[-1] for line in maps.splitlines() + if "pdfium" in line.split()[-1] and line.split()[-1].startswith("/")}) +print("shared:", theirs, round(ours.width), round(ours.height), pdfiums) +""" + + +def test_pyjpdfium_and_pypdfium2_share_a_process(tmp_path): + if importlib.util.find_spec("pypdfium2") is None: # never imported here: it loads PDFium + pytest.skip("pypdfium2 is not installed") + if not sys.platform.startswith("linux"): + pytest.skip("soname staging is Linux-only") + directory = Path(os.environ.get("PYJPDFIUM_NATIVE_DIR", "")) + pdf = tmp_path / "page.pdf" + pdf.write_bytes(b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n" + b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n" + b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 100] >>\n" + b"endobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n") + result = subprocess.run([sys.executable, "-c", PROBE, str(pdf)], capture_output=True, + text=True, check=False, timeout=120) + assert result.returncode == 0, result.stderr + if (directory / "libpdfium.so").exists(): + # Unstaged natives would bind to pypdfium2's PDFium; the loader says so instead. + assert "refused:" in result.stdout and "stage_natives.py" in result.stdout + else: + assert "shared: (200, 100) 200 100" in result.stdout + assert "libjpdfium_pdfium.so" in result.stdout and "pypdfium2" in result.stdout diff --git a/tests/ocrmypdf/mask_plugin.py b/tests/ocrmypdf/mask_plugin.py new file mode 100644 index 0000000..729922f --- /dev/null +++ b/tests/ocrmypdf/mask_plugin.py @@ -0,0 +1,12 @@ +"""Test plugin: blanks the top of every OCR image, where the test page's first line sits.""" + +from ocrmypdf import hookimpl +from PIL import ImageDraw + + +@hookimpl +def filter_ocr_image(page, image): + masked = image.copy() + # The first line ends 97 pt below the top of the 792 pt page; the second starts at 135 pt. + ImageDraw.Draw(masked).rectangle((0, 0, masked.width, masked.height * 116 // 792), fill="white") + return masked diff --git a/tests/ocrmypdf/requirements.txt b/tests/ocrmypdf/requirements.txt new file mode 100644 index 0000000..5cf1647 --- /dev/null +++ b/tests/ocrmypdf/requirements.txt @@ -0,0 +1,65 @@ +# OCRmyPDF 17.13.0 and its dependencies for the end-to-end plugin tests (Linux x86_64, +# CPython 3.12). Install into its own venv with +# pip install --require-hashes --only-binary=:all: -r tests/ocrmypdf/requirements.txt +annotated-types==0.8.0 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +cffi==2.1.1 \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf +charset-normalizer==3.5.2 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 +cryptography==50.0.2 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 +defusedxml==0.7.1 \ + --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 +fonttools==4.66.1 \ + --hash=sha256:7b8ff9e0edbcee2fbf7dff0c41b9041c1901c26acf64e23adb67495012df11de +fpdf2==2.8.9 \ + --hash=sha256:6e1d94af6d6311950a23dec7fb5fc84b000203eb59aee8e76c1e701b12a14976 +img2pdf==0.6.3 \ + --hash=sha256:44d12d235752edd17c43c04ff39952cdc5dd4c6aba90569c4902bd445085266b +jsonschema==4.26.0 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +lxml==6.1.3 \ + --hash=sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c +markdown-it-py==4.2.0 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +ocrmypdf==17.13.0 \ + --hash=sha256:f4d226f361c4b1498637f9c674daebfaf84223b84d9eba6bfba60a3ba260f9de +packaging==26.3 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pdfminer-six==20260107 \ + --hash=sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9 +pikepdf==10.16.0 \ + --hash=sha256:cd42de7d4f5d8dfda4da5d807a8407378dbd5a8f0e87b03c3e30e66d2228aadb +pillow==12.3.0 \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 +pluggy==1.6.0 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +pycparser==3.11 \ + --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 +pydantic==2.14.0 \ + --hash=sha256:15fab1bea6f1dc5003b54fc2ecab230c1fd1dbade2acd4addc52d81e32416d4b +pydantic-core==2.50.0 \ + --hash=sha256:f187030fc3d62c668feb0f09e92852e0eb414d7fcefc4748f2e67d245aade37e +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +pypdfium2==5.14.0 \ + --hash=sha256:11f281613fa22313d9c7ab89947665e84eccf8ebe40e1198a84a88352305648d +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb +rpds-py==2026.9.1 \ + --hash=sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.4 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 +uharfbuzz==0.56.3 \ + --hash=sha256:450c32c04dfdfe9dc69b68250605538b493c3444823383a2ede100f0e6686d8e diff --git a/tests/ocrmypdf/test_ocrmypdf_plugin.py b/tests/ocrmypdf/test_ocrmypdf_plugin.py new file mode 100644 index 0000000..3037849 --- /dev/null +++ b/tests/ocrmypdf/test_ocrmypdf_plugin.py @@ -0,0 +1,259 @@ +"""End-to-end OCRmyPDF runs with the pyjpdfium.ocrmypdf plugin. + +Opt-in: set PYJPDFIUM_OCRMYPDF_PYTHON to a Python with OCRmyPDF 17.13 and pyjpdfium installed +(tests/ocrmypdf/requirements.txt, hash-pinned), and PYJPDFIUM_OCR_PROFILE to a fetched profile. +OCRmyPDF runs as its real command line in a child process; results are read back here. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import zlib +from pathlib import Path + +import pytest + +import pyjpdfium + +PYTHON = os.environ.get("PYJPDFIUM_OCRMYPDF_PYTHON") +PROFILE = os.environ.get("PYJPDFIUM_OCR_PROFILE") +HERE = Path(__file__).parent +pytestmark = [ + pytest.mark.native, + pytest.mark.skipif(not (PYTHON and PROFILE), + reason="PYJPDFIUM_OCRMYPDF_PYTHON and PYJPDFIUM_OCR_PROFILE are not set"), +] + +LINES = ["The quick brown fox jumps", "over the lazy dog", "Invoice number 4821"] +FONT = "/Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>" + + +def pdf_bytes(pages: list[tuple[str, str, dict[str, bytes]]]) -> bytes: + """Pages of (page dictionary extras, content, image XObjects by name).""" + objects: list[bytes] = [b"<< /Type /Catalog /Pages 2 0 R >>", b""] + kids = [] + for extra, content, images in pages: + page_number = len(objects) + 1 + objects.append(b"") # the page, filled in below + content_number = len(objects) + 1 + data = content.encode("latin-1") + objects.append(b"<< /Length %d >>\nstream\n" % len(data) + data + b"\nendstream") + xobjects = [] + for name, image in images.items(): + xobjects.append(f"/{name} {len(objects) + 1} 0 R") + objects.append(image) + resources = f"/Resources << /XObject << {' '.join(xobjects)} >> >>" if images else "" + objects[page_number - 1] = (f"<< /Type /Page /Parent 2 0 R {extra} {resources} " + f"/Contents {content_number} 0 R >>").encode("latin-1") + kids.append(f"{page_number} 0 R") + objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>".encode() + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += b"%d 0 obj\n" % number + body + b"\nendobj\n" + xref = len(out) + out += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + out += b"".join(b"%010d 00000 n \n" % o for o in offsets) + out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, + xref) + return bytes(out) + + +def text_page() -> tuple[str, str, dict[str, bytes]]: + content = "".join(f"BT /F1 22 Tf 72 {700 - 60 * i} Td ({line}) Tj ET " + for i, line in enumerate(LINES)) + return f"/MediaBox [0 0 612 792] {FONT}", content, {} + + +def scan(tmp_path: Path, name: str = "scan") -> tuple[bytes, int, int]: + """The text page rasterized at 300 dpi: an 8-bit gray image XObject and its size.""" + source = tmp_path / f"{name}-source.pdf" + source.write_bytes(pdf_bytes([text_page()])) + png = tmp_path / f"{name}.png" + pyjpdfium.rasterize_page(source, 0, png, device="pnggray", raster_dpi=300) + with pyjpdfium.load_image(png) as bitmap: + gray = bitmap.tobytes()[0::3] if bitmap.pixel_format.name == "RGB8" else bitmap.tobytes() + width, height = bitmap.width, bitmap.height + data = zlib.compress(gray) + image = (b"<< /Type /XObject /Subtype /Image /Width %d /Height %d /ColorSpace /DeviceGray " + b"/BitsPerComponent 8 /Filter /FlateDecode /Length %d >>\nstream\n" + % (width, height, len(data)) + data + b"\nendstream") + return image, width, height + + +def scanned_page(tmp_path: Path, rotate: bool = False) -> tuple[str, str, dict[str, bytes]]: + image, _, _ = scan(tmp_path, "rotated" if rotate else "scan") + if not rotate: + return "/MediaBox [0 0 612 792]", "q 612 0 0 792 0 0 cm /Im0 Do Q", {"Im0": image} + # Stored a quarter turn anticlockwise, so /Rotate 90 displays it upright. + return "/MediaBox [0 0 792 612] /Rotate 90", "q 0 612 -792 0 792 0 cm /Im0 Do Q", {"Im0": image} + + +def ocrmypdf(*args: str, plugins=("pyjpdfium.ocrmypdf",)) -> subprocess.CompletedProcess: + command = [PYTHON, "-m", "ocrmypdf"] + for plugin in plugins: + command += ["--plugin", plugin] + env = {**os.environ, "PYJPDFIUM_OCR_PROFILE": PROFILE} + return subprocess.run([*command, *args], capture_output=True, text=True, env=env, + check=False, timeout=600) + + +def page_texts(path: Path) -> list[str]: + with pyjpdfium.open_document(path) as document: + texts = [] + for index in range(document.page_count): + with document.page(index) as page: + texts.append(" ".join(page.text().split())) + return texts + + +def render(path: Path, index: int = 0) -> bytes: + with pyjpdfium.open_document(path) as document, document.page(index) as page: + with page.render(dpi=50, pixel_format="gray") as bitmap: + return bitmap.tobytes() + + +def ok(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr[-3000:] + + +EXPECTED = " ".join(LINES) + + +def test_scanned_page_gains_text_and_looks_the_same(tmp_path): + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + out, sidecar = tmp_path / "out.pdf", tmp_path / "out.txt" + ok(ocrmypdf("--output-type", "pdf", "--sidecar", str(sidecar), str(source), str(out))) + assert page_texts(source) == [""] + assert page_texts(out) == [EXPECTED] + assert render(out) == render(source), "OCR must not change what the page shows" + assert sidecar.read_text(encoding="utf-8").split("\n")[:3] == LINES + + +def test_engine_value_is_registered_on_the_command_line(tmp_path): + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--ocr-engine", "pyjpdfium", "--output-type", "pdf", str(source), str(out))) + assert page_texts(out) == [EXPECTED] + unloaded = ocrmypdf("--ocr-engine", "pyjpdfium", str(source), str(out), plugins=()) + assert unloaded.returncode == 2, "without the plugin the value must stay unknown" + + +def test_rotated_page_reads_upright(tmp_path): + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path, rotate=True)])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--output-type", "pdf", str(source), str(out))) + assert page_texts(out) == [EXPECTED] + assert render(out) == render(source) + + +def test_modes_with_existing_text(tmp_path): + source = tmp_path / "mixed.pdf" + source.write_bytes(pdf_bytes([text_page(), scanned_page(tmp_path)])) + out = tmp_path / "out.pdf" + refused = ocrmypdf("--output-type", "pdf", str(source), str(out)) + assert refused.returncode == 6, "default mode refuses pages that already have text" + + ok(ocrmypdf("--output-type", "pdf", "--skip-text", str(source), str(out))) + assert page_texts(out) == [EXPECTED, EXPECTED] + assert render(out, 0) == render(source, 0), "--skip-text leaves the text page alone" + + forced = tmp_path / "forced.pdf" + ok(ocrmypdf("--output-type", "pdf", "--force-ocr", str(source), str(forced))) + assert page_texts(forced) == [EXPECTED, EXPECTED] + + +def test_redo_replaces_earlier_ocr_without_doubling(tmp_path): + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + first, second = tmp_path / "first.pdf", tmp_path / "second.pdf" + ok(ocrmypdf("--output-type", "pdf", str(source), str(first))) + ok(ocrmypdf("--output-type", "pdf", "--redo-ocr", str(first), str(second))) + assert page_texts(second) == [EXPECTED] + assert render(second) == render(source) + + +def test_engine_reads_the_image_ocrmypdf_supplies(tmp_path): + # mask_plugin.py blanks the first line in each OCR image only, so it must not be found. + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--output-type", "pdf", str(source), str(out), + plugins=("pyjpdfium.ocrmypdf", str(HERE / "mask_plugin.py")))) + assert page_texts(out) == [" ".join(LINES[1:])] + assert render(out) == render(source), "the visible page keeps every line" + + +@pytest.mark.parametrize("workers", [(), ("--no-use-threads",)], ids=["threads", "processes"]) +def test_pages_in_parallel_workers(tmp_path, workers): + # Forked workers each open their own engine; threads share one. + source = tmp_path / "in.pdf" + page = scanned_page(tmp_path) + source.write_bytes(pdf_bytes([page, page, page])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--output-type", "pdf", "--jobs", "2", *workers, str(source), str(out))) + assert page_texts(out) == [EXPECTED] * 3 + + +def test_fpdf2_reference_renderer_reads_the_same_words(tmp_path): + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--output-type", "pdf", "--pdf-renderer", "fpdf2", str(source), str(out))) + assert page_texts(out)[0].split() == EXPECTED.split() + + +@pytest.mark.parametrize( + ("args", "code", "message"), + [ + ((), 1, "PDF/A"), # the default output type is best-effort PDF/A + (("--output-type", "pdfa"), 1, "PDF/A"), + (("--output-type", "pdf", "--deskew"), 1, "--deskew"), + (("--output-type", "pdf", "--rotate-pages"), 1, "--rotate-pages"), + # Without unpaper installed, OCRmyPDF refuses --clean itself, before this plugin. + (("--output-type", "pdf", "--clean"), (1, 3), "--clean"), + (("--output-type", "pdf", "--ocr-engine", "tesseract"), 1, "tesseract"), + (("--output-type", "pdf", "-l", "deu"), 3, "deu"), + (("--output-type", "pdf", "--tesseract-timeout", "5"), 2, "--tesseract-timeout"), + (("--output-type", "pdf", "--pyjpdfium-threads", "-1"), 1, "threads"), + ], +) +def test_unsupported_modes_are_refused(tmp_path, args, code, message): + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([text_page()])) + out = tmp_path / "out.pdf" + result = ocrmypdf(*args, str(source), str(out)) + codes = code if isinstance(code, tuple) else (code,) + assert result.returncode in codes, result.stderr[-2000:] + assert message in result.stderr + assert not out.exists() + + +def test_a_tampered_profile_is_refused_before_any_page(tmp_path): + profile = tmp_path / "profile" + shutil.copytree(PROFILE, profile) + with (profile / "dictionary.txt").open("ab") as handle: + handle.write(b"x\n") + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + out = tmp_path / "out.pdf" + result = ocrmypdf("--output-type", "pdf", "--pyjpdfium-profile", str(profile), str(source), + str(out)) + assert result.returncode == 1 and "pinned digest" in result.stderr + assert not out.exists() + + +def test_runs_without_tesseract_or_ghostscript(tmp_path): + present = [tool for tool in ("tesseract", "gs") if shutil.which(tool)] + if present: + pytest.skip(f"installed here: {present}") + source = tmp_path / "in.pdf" + source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--output-type", "pdf", "--jobs", "2", str(source), str(out))) + assert page_texts(out) == [EXPECTED] diff --git a/tests/unit/test_lazy_import.py b/tests/unit/test_lazy_import.py index 5742218..5d9ee52 100644 --- a/tests/unit/test_lazy_import.py +++ b/tests/unit/test_lazy_import.py @@ -5,7 +5,7 @@ import sys import pyjpdfium import pyjpdfium.errors, pyjpdfium.models, pyjpdfium.pdf, pyjpdfium.runtime -import pyjpdfium.ocr, pyjpdfium.raster, pyjpdfium.textlayer +import pyjpdfium.ocr, pyjpdfium.raster, pyjpdfium.textlayer, pyjpdfium.cli, pyjpdfium.profiles assert pyjpdfium.PixelFormat.RGB == 2 loaded = [m for m in sys.modules if m.startswith(("pyjpdfium._native", "pyjpdfium._ocr_native"))] assert not loaded, loaded diff --git a/tests/unit/test_profiles_and_cli.py b/tests/unit/test_profiles_and_cli.py new file mode 100644 index 0000000..33ef66f --- /dev/null +++ b/tests/unit/test_profiles_and_cli.py @@ -0,0 +1,50 @@ +import hashlib + +import pytest + +from pyjpdfium import cli, errors +from pyjpdfium.profiles import read_profile, verify_profile + + +def fake_profile(directory, content=b"model bytes"): + directory.mkdir() + (directory / "det.onnx").write_bytes(content) + digest = hashlib.sha256(b"model bytes").hexdigest() + (directory / "profile.txt").write_text( + f"format 1\nid tiny\nfile detector det.onnx {digest} https://example.invalid/det.onnx\n", + encoding="utf-8", + ) + return directory + + +def test_a_matching_profile_verifies(tmp_path): + profile = verify_profile(fake_profile(tmp_path / "p")) + assert profile.id == "tiny" and [f.name for f in profile.files] == ["det.onnx"] + + +def test_a_changed_file_is_named(tmp_path): + directory = fake_profile(tmp_path / "p", content=b"other bytes") + assert read_profile(directory).id == "tiny" # parsing alone does not hash + with pytest.raises(errors.InvalidArgumentError, match="det.onnx does not match"): + verify_profile(directory) + + +def test_a_directory_without_a_manifest_is_refused(tmp_path): + with pytest.raises(errors.InvalidArgumentError, match="profile.txt"): + verify_profile(tmp_path) + + +def test_fonts_needs_no_native_code(capsys): + assert cli.main(["fonts"]) == 0 + assert capsys.readouterr().out.startswith("glyphless") + + +def test_models_reports_a_bad_profile(tmp_path, capsys): + assert cli.main(["models", str(tmp_path)]) == 1 + assert "FAIL" in capsys.readouterr().out + + +def test_usage_errors_exit_2(): + with pytest.raises(SystemExit) as raised: + cli.main(["render"]) + assert raised.value.code == 2 diff --git a/tools/stage_natives.py b/tools/stage_natives.py new file mode 100644 index 0000000..1da9193 --- /dev/null +++ b/tools/stage_natives.py @@ -0,0 +1,103 @@ +"""Stages JPDFium's natives for Python, so they can share a process with other Python packages. + + python tools/stage_natives.py JPDFIUM_DIST DEST + +OCRmyPDF imports pypdfium2, whose libpdfium.so carries the soname of JPDFium's PDFium, and +onnxruntime wheels ship libonnxruntime.so.1. The dynamic linker binds a dependency to a loaded +library with the same soname, so JPDFium's core would call into another PDFium. Staging gives +our copies their own sonames, points RUNPATH at the staging directory only, and writes +natives-manifest.json with every file's digest before and after. Linux only; needs patchelf. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + +RENAMES = { + "libpdfium.so": "libjpdfium_pdfium.so", + "libonnxruntime.so.1": "libjpdfium_onnxruntime.so.1", +} +MANIFEST = "natives-manifest.json" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def patchelf(*args: str) -> str: + done = subprocess.run(["patchelf", *args], capture_output=True, text=True, check=True) + return done.stdout.strip() + + +def is_elf(path: Path) -> bool: + with path.open("rb") as handle: + return handle.read(4) == b"\x7fELF" + + +def stage(source: Path, dest: Path) -> dict: + if not sys.platform.startswith("linux"): + raise SystemExit("stage_natives.py stages Linux natives only") + if shutil.which("patchelf") is None: + raise SystemExit("patchelf is required") + if dest.exists(): + raise SystemExit(f"{dest} exists; stage into a new directory") + dest.mkdir(parents=True) + staged: dict[str, Path] = {} + origins: dict[str, Path] = {} + # Real files only, each under its soname, so no symlink is needed after staging. + for path in sorted(source.iterdir()): + if path.is_symlink() or not path.is_file(): + continue + name = path.name + if is_elf(path): + try: + name = patchelf("--print-soname", str(path)) or name + except subprocess.CalledProcessError: + pass # no DT_SONAME: keep the file name + name = RENAMES.get(name, name) + if name in staged: + raise SystemExit(f"two files stage as {name}") + target = dest / name + shutil.copy2(path, target) + staged[name], origins[name] = target, path + for name, target in staged.items(): + if not is_elf(target): + continue + if target.name in RENAMES.values(): + patchelf("--set-soname", target.name, str(target)) + for needed in patchelf("--print-needed", str(target)).split(): + if needed in RENAMES: + patchelf("--replace-needed", needed, RENAMES[needed], str(target)) + patchelf("--set-rpath", "$ORIGIN", str(target)) + entries = [] + for name, target in sorted(staged.items()): + if is_elf(target): + needed = patchelf("--print-needed", str(target)).split() + stale = sorted(set(needed) & set(RENAMES)) + if stale: + raise SystemExit(f"{name} still needs {stale}") + entries.append({ + "name": name, + "sha256": sha256(target), + "source": origins[name].name, + "source_sha256": sha256(origins[name]), + }) + manifest = {"source": str(source.resolve()), "renames": RENAMES, "files": entries} + (dest / MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit(__doc__) + result = stage(Path(sys.argv[1]), Path(sys.argv[2])) + print(f"staged {len(result['files'])} files into {sys.argv[2]}") From 355b7f38bcf91d4c77e893beb39b12a5f07fb986 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:25:28 +0100 Subject: [PATCH 09/10] Build self-contained wheels and check them on a clean offline machine --- .github/workflows/ci.yml | 39 ++++-- CMakeLists.txt | 3 +- README.md | 3 + docs/PACKAGING.md | 69 +++++++++++ src/pyjpdfium/cli.py | 29 +++++ tests/clean_machine/check.py | 151 +++++++++++++++++++++++ tests/clean_machine/prune.sh | 34 ++++++ tools/build_wheel.py | 224 +++++++++++++++++++++++++++++++++++ tools/stage_natives.py | 13 +- 9 files changed, 554 insertions(+), 11 deletions(-) create mode 100644 docs/PACKAGING.md create mode 100644 tests/clean_machine/check.py create mode 100644 tests/clean_machine/prune.sh create mode 100644 tools/build_wheel.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a457217..51e56c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake ninja-build pkg-config \ libpcre2-dev libfreetype6-dev libharfbuzz-dev libicu-dev \ - libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf libgl1 poppler-utils + libqpdf-dev libpugixml-dev libunibreak-dev lld patchelf libgl1 poppler-utils icu-devtools - name: Fetch pinned PDFium and verify its published digest env: @@ -91,9 +91,13 @@ jobs: JPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile run: ctest --test-dir jpdfium/native/build-real --output-on-failure + # One layout for tests and wheels: own sonames, with the OCR module beside the core. + - name: Stage the natives for Python + run: python tools/stage_natives.py jpdfium/native/dist/linux-x64 "$RUNNER_TEMP/natives" + - name: Build pyjpdfium env: - JPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + JPDFIUM_NATIVE_DIR: ${{ runner.temp }}/natives JPDFIUM_INCLUDE_DIR: ${{ github.workspace }}/jpdfium/native/bridge/include run: | python -m pip install "scikit-build-core>=0.11" "nanobind>=2.9" "pytest>=8" @@ -102,7 +106,7 @@ jobs: - name: Python tests (unit + native) env: - PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + PYJPDFIUM_NATIVE_DIR: ${{ runner.temp }}/natives PYJPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile run: | # Native tests skip when the extension is missing; fail loudly here instead. @@ -111,7 +115,7 @@ jobs: - name: Parity with OCRmyPDF and independent text extractors env: - PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + PYJPDFIUM_NATIVE_DIR: ${{ runner.temp }}/natives run: | python -m pip install --require-hashes --only-binary=:all: -r tests/parity/requirements.txt export PYJPDFIUM_PARITY_SRC="$(python tests/parity/fetch_reference.py "$RUNNER_TEMP/parity")" @@ -119,7 +123,7 @@ jobs: - name: OCR parity with the official PaddleOCR pipeline env: - PYJPDFIUM_NATIVE_DIR: ${{ github.workspace }}/jpdfium/native/dist/linux-x64 + PYJPDFIUM_NATIVE_DIR: ${{ runner.temp }}/natives PYJPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile PYJPDFIUM_OCR_REFERENCE: ${{ runner.temp }}/ocr-reference/bin/python run: | @@ -130,14 +134,33 @@ jobs: - name: OCRmyPDF end to end with the pyjpdfium plugin env: + PYJPDFIUM_NATIVE_DIR: ${{ runner.temp }}/natives PYJPDFIUM_OCR_PROFILE: ${{ runner.temp }}/ocr-profile PYJPDFIUM_OCRMYPDF_PYTHON: ${{ runner.temp }}/ocrmypdf/bin/python run: | - # OCRmyPDF imports pypdfium2, so our natives need their own sonames in that process. - python tools/stage_natives.py jpdfium/native/dist/linux-x64 "$RUNNER_TEMP/natives" - export PYJPDFIUM_NATIVE_DIR="$RUNNER_TEMP/natives" python -m venv "$RUNNER_TEMP/ocrmypdf" "$RUNNER_TEMP/ocrmypdf/bin/pip" install --require-hashes --only-binary=:all: \ -r tests/ocrmypdf/requirements.txt "$RUNNER_TEMP/ocrmypdf/bin/pip" install --no-deps "$RUNNER_TEMP"/wheel/pyjpdfium-*.whl python -m pytest tests/ocrmypdf tests/native/test_cli.py -q -rs + + # The release closure: JPDFium's bundler and checks, then a fresh Ubuntu with no network. + - name: Wheel on a clean offline machine + run: | + bash jpdfium/native/build-minimal-icu.sh + python tools/build_wheel.py jpdfium "$RUNNER_TEMP/wheelhouse" + python -m pip download --require-hashes --only-binary=:all: \ + -r tests/ocrmypdf/requirements.txt -d "$RUNNER_TEMP/ocrmypdf-wheels" + printf 'FROM ubuntu:24.04\nRUN apt-get update && apt-get install -y --no-install-recommends python3 python3-venv && rm -rf /var/lib/apt/lists/*\n' \ + | docker build -t pyjpdfium-clean - + docker run --rm --network=none -v "$RUNNER_TEMP:/ci:ro" -v "$PWD:/src:ro" \ + -e PYJPDFIUM_OCR_PROFILE=/ci/ocr-profile pyjpdfium-clean bash -c ' + set -e + python3 -m venv /tmp/venv + /tmp/venv/bin/pip install -q --no-index /ci/wheelhouse/pyjpdfium-*.whl + /tmp/venv/bin/pip install -q --no-index --require-hashes --only-binary=:all: \ + --find-links /ci/ocrmypdf-wheels -r /src/tests/ocrmypdf/requirements.txt + cd /tmp && /tmp/venv/bin/python -I /src/tests/clean_machine/check.py + bash /src/tests/clean_machine/prune.sh /ci/wheelhouse/pyjpdfium-*.whl \ + /ci/ocrmypdf-wheels /src/tests/ocrmypdf/requirements.txt \ + /src/tests/clean_machine/check.py' diff --git a/CMakeLists.txt b/CMakeLists.txt index 2998809..846f7d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,7 +72,8 @@ else() endif() if(PYJPDFIUM_BUNDLE_NATIVES) + # The staged closure with its manifests and license texts (tools/build_wheel.py). install(DIRECTORY "${JPDFIUM_NATIVE_DIR}/" DESTINATION pyjpdfium/_libs FILES_MATCHING PATTERN "*.so*" PATTERN "*.dylib" PATTERN "*.dll" - PATTERN "native-libs.*") + PATTERN "*.json" PATTERN "licenses/*") endif() diff --git a/README.md b/README.md index c89d2bc..3470f74 100644 --- a/README.md +++ b/README.md @@ -92,4 +92,7 @@ Parity with OCRmyPDF's own rasterizer and sandwich graft is opt-in: install `PYJPDFIUM_OCRMYPDF_PYTHON`, a Python with `tests/ocrmypdf/requirements.txt` and pyjpdfium installed, and natives staged by `tools/stage_natives.py`. +Self-contained wheels are built by `tools/build_wheel.py` +([docs/PACKAGING.md](docs/PACKAGING.md)). + See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design. diff --git a/docs/PACKAGING.md b/docs/PACKAGING.md new file mode 100644 index 0000000..0a5485f --- /dev/null +++ b/docs/PACKAGING.md @@ -0,0 +1,69 @@ +# Packaging + +`tools/build_wheel.py` builds a self-contained Linux wheel from a JPDFium checkout: + +```bash +python tools/build_wheel.py /path/to/JPDFium dist/ +``` + +1. JPDFium's own `native/bundle-runtime-deps.sh` collects the core's runtime closure and runs the + same checks as its Java natives jar: no executable stacks, hermetic dependencies, exported + symbols, no orphans, lean layout, and the size budget. The optional OCR module joins from + `native/dist/ocr-` afterwards, as it is not part of the core bundle. +2. `tools/stage_natives.py` gives JPDFium's PDFium and ONNX Runtime their own sonames and sets + every RUNPATH to `$ORIGIN`, so the wheel can share a process with pypdfium2 or onnxruntime + wheels ([OCRMYPDF.md](OCRMYPDF.md)). +3. License texts go to `pyjpdfium/_libs/licenses`: JPDFium's notice and licenses, ONNX + Runtime's license and third-party notices, and the Debian copyright file of every system + library the bundler copied in. +4. `pyjpdfium/_libs/identity.json` records: + - PyJPDFium's and JPDFium's revisions, and the safe and OCR ABI versions; + - the PDFium pin and ONNX Runtime version, and every capability's level; + - the model profiles the module supports, with their digests; + - the bundled system libraries, the license inventory, and every file's digest before and + after staging. +5. The wheel is tagged `manylinux_2_` for the newest glibc symbol any bundled file needs. + +`pyjpdfium doctor` checks an installation against `natives-manifest.json`, file by file, and +reports any PDFium, ONNX Runtime or JPDFium library loaded from outside the wheel. + +## Size + +The wheel built on Ubuntu 24.04 is 26.1 MiB (`manylinux_2_38_x86_64`). The largest parts are +ONNX Runtime (29 MB unpacked), JPDFium's core (7 MB) and PDFium (6 MB). Model profiles are not +in the wheel; they are fetched and pinned separately (JPDFium `docs/design/OCR.md`). + +JPDFium's release builds rebuild qpdf with native crypto and HarfBuzz without GLib. Those +scripts fetch upstream's newest release without a pinned version or digest, so this build does +not run them. It bundles Ubuntu's qpdf and HarfBuzz instead, which bring GnuTLS, Nettle, GMP, +GLib and their dependencies. These are LGPL libraries, shipped as replaceable shared libraries +with their copyright files. A build without Ghostscript still has some LGPL components. + +## Clean machine + +`tests/clean_machine/check.py` runs on a fresh `ubuntu:24.04` container with only `python3` +and `python3-venv` added, no network, and no `PYJPDFIUM_NATIVE_DIR`. It installs the wheel +offline and checks the following: + +- `pyjpdfium doctor` reports no problems, and all 38 bundled files match their digests; +- a page renders, its text reads back, OCR recognizes it, and a text layer is written; +- OCRmyPDF, installed offline from `tests/ocrmypdf/requirements.txt`, OCRs a scanned page with + the plugin. + +Every native library the process loads comes from the wheel, except the C runtime and the five +that Python's own standard library loads: zlib, bzip2, libffi, liblzma and expat. The bundled +zlib then resolves to the already-loaded system copy, which has the same ABI. + +## OCRmyPDF dependencies + +`tests/clean_machine/prune.sh` installs OCRmyPDF's pinned dependencies with some left out: + +| Dependency | With the plugin | +|---|---| +| Tesseract, Ghostscript | Not installed; never needed | +| pypdfium2 | Imported by OCRmyPDF when installed, but never called: its rasterizer logs nothing. Works when not installed | +| fpdf2, uharfbuzz | Needed only for `--pdf-renderer fpdf2`. Work when not installed | +| img2pdf | Required: OCRmyPDF imports it at startup | + +OCRmyPDF's package metadata still requires pypdfium2, fpdf2 and uharfbuzz. Leaving them out +needs an install with `--no-deps` from a pinned list, as the script does. diff --git a/src/pyjpdfium/cli.py b/src/pyjpdfium/cli.py index bb20166..7fed9e5 100644 --- a/src/pyjpdfium/cli.py +++ b/src/pyjpdfium/cli.py @@ -147,6 +147,33 @@ def _loaded_libraries() -> list[str]: return [p for p in _mapped_libraries() if Path(p).name.startswith(names)] +def _check_bundle(report: _Report, directory: Path) -> None: + """Staged natives: every file matches its manifest, and nothing native loads from elsewhere.""" + from .profiles import sha256_file + + manifest = directory / "natives-manifest.json" + if not manifest.is_file(): + report.note("bundle", "no natives-manifest.json: not staged by tools/stage_natives.py") + return + entries = json.loads(manifest.read_text(encoding="utf-8"))["files"] + changed = [e["name"] for e in entries + if not (directory / e["name"]).is_file() + or sha256_file(directory / e["name"]) != e["sha256"]] + if changed: + report.problem("bundle", f"{len(changed)} file(s) differ from the manifest: {changed[:3]}") + else: + report.ok("bundle", f"{len(entries)} files match natives-manifest.json") + root = directory.resolve() + outside = [p for p in _loaded_libraries() if not Path(p).is_relative_to(root)] + if outside: + report.problem("loaded", f"native libraries from outside {root}: {outside}") + identity = directory / "identity.json" + if identity.is_file(): + built = json.loads(identity.read_text(encoding="utf-8")) + report.ok("identity", f"pyjpdfium {built['pyjpdfium']['revision'][:12]}, " + f"jpdfium {built['jpdfium']['revision'][:12]}, {built['pdfium']}") + + def cmd_doctor(args: argparse.Namespace) -> int: """Reports identity, capabilities, profiles and dependencies; recognizes no document.""" from . import _loader @@ -182,6 +209,8 @@ def cmd_doctor(args: argparse.Namespace) -> int: # Listed once both modules are in, so the identities cover everything that loaded. for path in _loaded_libraries(): report.ok("loaded", f"{path} sha256 {sha256_file(Path(path))[:16]}") + if directory is not None: + _check_bundle(report, directory) profile_dir = Path(args.profile) if args.profile else default_profile_dir() if profile_dir is None: report.note("ocr profile", "none given; pass --profile or set PYJPDFIUM_OCR_PROFILE") diff --git a/tests/clean_machine/check.py b/tests/clean_machine/check.py new file mode 100644 index 0000000..44ad3ac --- /dev/null +++ b/tests/clean_machine/check.py @@ -0,0 +1,151 @@ +"""Checks an installed wheel on a clean machine: no compiler, no PDF, image or ICU libraries +from the system, no network, no PYJPDFIUM_NATIVE_DIR. Exits non-zero at the first failure. + + python check.py # PYJPDFIUM_OCR_PROFILE names a fetched profile directory + +Runs the pyjpdfium command, renders, recognizes, writes a text layer, runs OCRmyPDF with the +plugin when OCRmyPDF is installed, and audits every native library the process loaded. +""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +import sys +import tempfile +import zlib +from pathlib import Path + +LINES = ["Clean machine check", "invoice 4821 paid"] + + +def step(name: str) -> None: + print(f"--- {name}", flush=True) + + +def run(*command: str) -> str: + done = subprocess.run(command, capture_output=True, text=True, timeout=600, check=False) + if done.returncode != 0: + raise SystemExit(f"FAILED {' '.join(command)}\n{done.stdout}\n{done.stderr}") + return done.stdout + + +def pdf(pages: list[tuple[str, str, bytes | None]]) -> bytes: + """Pages of (page dictionary extras, content, optional /Im0 image object).""" + objects = [b"<< /Type /Catalog /Pages 2 0 R >>", b""] + kids = [] + for extra, content, image in pages: + number = len(objects) + 1 + kids.append(f"{number} 0 R") + resources = f"/Resources << /XObject << /Im0 {number + 2} 0 R >> >>" if image else "" + objects.append(f"<< /Type /Page /Parent 2 0 R {extra} {resources} " + f"/Contents {number + 1} 0 R >>".encode()) + data = content.encode() + objects.append(b"<< /Length %d >>\nstream\n" % len(data) + data + b"\nendstream") + if image: + objects.append(image) + objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>".encode() + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for index, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += b"%d 0 obj\n" % index + body + b"\nendobj\n" + xref = len(out) + out += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + out += b"".join(b"%010d 00000 n \n" % o for o in offsets) + out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, + xref) + return bytes(out) + + +def text_page() -> tuple[str, str, None]: + font = "/Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>" + content = "".join(f"BT /F1 26 Tf 72 {700 - 60 * i} Td ({line}) Tj ET " + for i, line in enumerate(LINES)) + return f"/MediaBox [0 0 612 792] {font}", content, None + + +def main() -> None: + import pyjpdfium + + if os.environ.get("PYJPDFIUM_NATIVE_DIR"): + raise SystemExit("unset PYJPDFIUM_NATIVE_DIR: the check is of the bundled natives") + work = Path(tempfile.mkdtemp(prefix="clean-")) + scripts = Path(sys.executable).parent + libs = (Path(pyjpdfium.__file__).parent / "_libs").resolve() + expected = " ".join(LINES) + + step("pyjpdfium doctor") + report = run(str(scripts / "pyjpdfium"), "doctor") + print(report) + if "0 problem(s)" not in report or "match natives-manifest.json" not in report: + raise SystemExit("doctor found problems") + + step("render, text and OCR in this process") + source = work / "text.pdf" + source.write_bytes(pdf([text_page()])) + with pyjpdfium.open_document(source) as document, document.page(0) as page: + assert " ".join(page.text().split()) == expected, page.text() + png = work / "page.png" + pyjpdfium.rasterize_page(source, 0, png, device="pnggray", raster_dpi=300) + with pyjpdfium.OcrEngine(os.environ["PYJPDFIUM_OCR_PROFILE"]) as engine: + result = engine.recognize(png) + assert [line.text for line in result.lines] == LINES, result.text + layer = work / "layer.pdf" + pyjpdfium.write_text_layer(layer, result.text_lines(), image_size=result.image_size, + dpi=result.dpi) + with pyjpdfium.open_document(layer) as document, document.page(0) as page: + assert " ".join(page.text().split()) == expected + + step("every native library this process loaded") + system = ("libc.so", "libm.so", "libstdc++.so", "libgcc_s.so", "ld-linux", "libpthread.so", + "libdl.so", "librt.so") + stray = [] + for line in Path("/proc/self/maps").read_text().splitlines(): + parts = line.split(maxsplit=5) + if len(parts) < 6 or ".so" not in parts[5]: + continue + path = Path(os.path.realpath(parts[5])) + if path.is_relative_to(libs) or path.name.startswith(system): + continue + stdlib = Path(sys.base_prefix, "lib", f"python{sys.version_info[0]}.{sys.version_info[1]}") + if any(path.is_relative_to(p.resolve()) for p in (Path(sys.prefix), stdlib)): + continue # the venv's packages and the standard library's extension modules + if "python3" in path.name: + continue # the interpreter + stray.append(str(path)) + print("\n".join(sorted(set(stray))) or "(none outside the wheel and the C runtime)") + allowed = ("libz.so", "libexpat.so", "libffi.so", "libbz2.so", "liblzma.so", "libcrypto.so", + "libssl.so", "libuuid.so", "libmpdec.so", "libsqlite3.so", "libtinfo.so", + "libreadline.so", "libncursesw.so") # loaded by Python's standard library + unexpected = [p for p in set(stray) if not Path(p).name.startswith(allowed)] + if unexpected: + raise SystemExit(f"native libraries loaded from the system: {sorted(unexpected)}") + + if importlib.util.find_spec("ocrmypdf") is None: + step("OCRmyPDF is not installed; plugin check skipped") + return + step("OCRmyPDF with the pyjpdfium plugin") + scan_png = work / "scan.png" + pyjpdfium.rasterize_page(source, 0, scan_png, device="pnggray", raster_dpi=300) + with pyjpdfium.load_image(scan_png) as bitmap: + gray = bitmap.tobytes()[0::3] if bitmap.pixel_format.name == "RGB8" else bitmap.tobytes() + width, height = bitmap.width, bitmap.height + data = zlib.compress(gray) + image = (b"<< /Type /XObject /Subtype /Image /Width %d /Height %d /ColorSpace /DeviceGray " + b"/BitsPerComponent 8 /Filter /FlateDecode /Length %d >>\nstream\n" + % (width, height, len(data)) + data + b"\nendstream") + scanned = work / "scanned.pdf" + scanned.write_bytes(pdf([("/MediaBox [0 0 612 792]", "q 612 0 0 792 0 0 cm /Im0 Do Q", + image)])) + out = work / "ocr.pdf" + run(sys.executable, "-m", "ocrmypdf", "--plugin", "pyjpdfium.ocrmypdf", "--output-type", + "pdf", str(scanned), str(out)) + with pyjpdfium.open_document(out) as document, document.page(0) as page: + assert " ".join(page.text().split()) == expected, page.text() + print("OCRmyPDF output reads:", expected) + + +if __name__ == "__main__": + main() + print("clean machine check passed") diff --git a/tests/clean_machine/prune.sh b/tests/clean_machine/prune.sh new file mode 100644 index 0000000..c69e619 --- /dev/null +++ b/tests/clean_machine/prune.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Which of OCRmyPDF's dependencies the plugin can do without, on a clean machine, offline. +# Usage: prune.sh WHEEL OCRMYPDF_WHEELS_DIR REQUIREMENTS CHECK_PY +set -euo pipefail +WHEEL="$1" WHEELS="$2" REQUIREMENTS="$3" CHECK="$4" +mismatches=0 + +# variant EXPECT NAME [DROP...]: EXPECT is works or fails, as docs/PACKAGING.md records. +variant() { + local expect="$1" name="$2"; shift 2 + local venv="/tmp/prune-$name" + python3 -m venv "$venv" + "$venv/bin/pip" install -q --no-index "$WHEEL" + # Pinned versions only, minus the packages this variant leaves out. + grep -E '^[a-z0-9-]+==' "$REQUIREMENTS" | awk '{print $1}' | while read -r pin; do + skip=0 + for drop in "$@"; do [ "${pin%%==*}" = "$drop" ] && skip=1; done + [ "$skip" = 1 ] || echo "$pin" + done > "/tmp/prune-$name.txt" + "$venv/bin/pip" install -q --no-index --no-deps --find-links "$WHEELS" -r "/tmp/prune-$name.txt" + local got=fails + if (cd /tmp && "$venv/bin/python" -I "$CHECK" > "/tmp/prune-$name.log" 2>&1); then got=works; fi + echo "$name: $got without [$*]" + if [ "$got" != "$expect" ]; then + echo " expected it to $expect; last lines:" && tail -n 5 "/tmp/prune-$name.log" + mismatches=$((mismatches + 1)) + fi +} + +variant works all +variant works no-pypdfium2 pypdfium2 +variant works no-fpdf2 pypdfium2 fpdf2 uharfbuzz +variant fails no-img2pdf pypdfium2 img2pdf +exit "$mismatches" diff --git a/tools/build_wheel.py b/tools/build_wheel.py new file mode 100644 index 0000000..8e2d265 --- /dev/null +++ b/tools/build_wheel.py @@ -0,0 +1,224 @@ +"""Builds a self-contained Linux wheel from a JPDFium checkout's staged natives. + + python tools/build_wheel.py JPDFIUM_CHECKOUT OUT_DIR + +1. Copies native/dist/linux- and runs JPDFium's own native/bundle-runtime-deps.sh on the + copy, the same closure step and checks its Java natives jar uses. The optional OCR module in + native/dist/ocr-linux- joins after, as it is not part of the core bundle. +2. Stages the result with tools/stage_natives.py, so it can share a process with pypdfium2. +3. Collects license texts: JPDFium's, ONNX Runtime's, and the Debian copyright file of every + system library the bundler copied in. +4. Writes identity.json: revisions, ABI versions, capabilities, pins and every file's digest. +5. Builds the wheel with PYJPDFIUM_BUNDLE_NATIVES=ON and tags it manylinux for the newest glibc + symbol any bundled file needs. +""" + +from __future__ import annotations + +import base64 +import ctypes +import hashlib +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from stage_natives import sha256, stage_dirs # noqa: E402 + +ROOT = Path(__file__).resolve().parent.parent +CAPABILITIES = ("runtime.lifecycle", "pdf.open", "pdf.render", "pdf.text", "pdf.raster", + "image.load", "image.png_write", "image.rectify", "output.transaction", + "legacy.guards", "text_layer.write", "text_layer.bidi", "text_layer.quad") +OCR_CAPABILITIES = ("ocr.recognize", "ocr.word_geometry") +JPDFIUM_LICENSES = ("PDFIUM-BSD-3.txt", "FREETYPE.txt", "HARFBUZZ.txt", "ICU.txt", + "QPDF-APACHE-2.0.txt", "ZLIB.txt", "CLIPPER-BSL-1.0.txt") + + +def run(*args: str, cwd: Path | None = None) -> str: + return subprocess.run(args, cwd=cwd, capture_output=True, text=True, check=True).stdout + + +def pin(path: Path) -> str: + return next(line for line in path.read_text().splitlines() + if line.strip() and not line.startswith("#")) + + +def bundle(jpdfium: Path, arch_dir: str, work: Path) -> Path: + dist = work / "native" / "dist" / arch_dir + shutil.copytree(jpdfium / "native" / "dist" / arch_dir, dist, symlinks=True) + before = {p.name for p in dist.iterdir()} + run("bash", str(jpdfium / "native" / "bundle-runtime-deps.sh"), arch_dir, cwd=work) + added = sorted({p.name for p in dist.iterdir()} - before) + (work / "bundled-system-libs.txt").write_text("\n".join(added) + "\n", encoding="utf-8") + return dist + + +def collect_licenses(jpdfium: Path, staged: Path, system_libs: list[str]) -> list[str]: + licenses = staged / "licenses" + licenses.mkdir() + shutil.copyfile(jpdfium / "NOTICE", licenses / "JPDFIUM-NOTICE.txt") + shutil.copyfile(jpdfium / "LICENSE", licenses / "JPDFIUM-LICENSE.txt") + for name in JPDFIUM_LICENSES: + shutil.copyfile(jpdfium / "native" / "licenses" / name, licenses / name) + if (staged / "libjpdfium_ocr.so").exists(): + runtime = jpdfium / "native" / "ocr" / "onnxruntime" + shutil.copyfile(runtime / "LICENSE", licenses / "ONNXRUNTIME-LICENSE.txt") + shutil.copyfile(runtime / "ThirdPartyNotices.txt", licenses / "ONNXRUNTIME-NOTICES.txt") + packages = set() + for lib in system_libs: + owner = subprocess.run(["dpkg", "-S", f"*/{lib}"], capture_output=True, text=True) + if owner.returncode != 0: + raise SystemExit(f"no package owns the bundled library {lib}") + packages.add(owner.stdout.split(":")[0]) + for package in sorted(packages): + copyright_file = Path("/usr/share/doc") / package / "copyright" + if not copyright_file.exists(): + raise SystemExit(f"{package} ships no copyright file") + shutil.copyfile(copyright_file, licenses / f"{package}.copyright.txt") + return sorted(p.name for p in licenses.iterdir()) + + +def capabilities(library: Path, function: str, names: tuple[str, ...]) -> dict[str, int]: + lib = ctypes.CDLL(str(library), mode=ctypes.RTLD_LOCAL) + query = getattr(lib, function) + query.argtypes = (ctypes.c_char_p, ctypes.POINTER(ctypes.c_int32)) + levels = {} + for name in names: + level = ctypes.c_int32(0) + query(name.encode(), ctypes.byref(level)) + levels[name] = level.value + return levels + + +def api_version(library: Path, function: str) -> str: + query = getattr(ctypes.CDLL(str(library), mode=ctypes.RTLD_LOCAL), function) + query.restype = ctypes.c_uint32 + value = query() + return f"{value >> 16}.{value & 0xFFFF}" + + +def identity(jpdfium: Path, staged: Path, licenses: list[str], system_libs: list[str]) -> dict: + manifest = json.loads((staged / "natives-manifest.json").read_text(encoding="utf-8")) + profiles = {} + for profile in sorted((jpdfium / "native" / "ocr" / "profiles").glob("*.profile")): + files = [line.split() for line in profile.read_text().splitlines() + if line.startswith(("file ", "derived "))] + profiles[profile.stem] = {fields[2]: fields[3] for fields in files} + ocr = staged / "libjpdfium_ocr.so" + return { + "pyjpdfium": {"version": _version(), "revision": _git(ROOT, "PYJPDFIUM_REVISION")}, + "jpdfium": {"revision": _git(jpdfium, "JPDFIUM_REVISION"), + "api": api_version(staged / "libjpdfium.so", "jpdf_api_version")}, + "platform": {"system": "linux", "machine": platform.machine(), + "glibc": platform.libc_ver()[1]}, + "pdfium": pin(jpdfium / "native" / "pdfium.version"), + "capabilities": capabilities(staged / "libjpdfium.so", "jpdf_capability", CAPABILITIES), + "ocr": None if not ocr.exists() else { + "api": api_version(ocr, "jpdf_ocr_api_version"), + "onnxruntime": pin(jpdfium / "native" / "ocr" / "onnxruntime.version").split()[1], + "capabilities": capabilities(ocr, "jpdf_ocr_capability", OCR_CAPABILITIES), + "profiles": profiles, + }, + "fonts": {"glyphless": "built into libjpdfium.so"}, + "system_libraries": system_libs, + "licenses": licenses, + "files": manifest["files"], + "renames": manifest["renames"], + } + + +def _git(path: Path, env: str) -> str: + """The checkout's commit, or the named variable for exported sources without .git.""" + try: + return run("git", "-C", str(path), "rev-parse", "HEAD").strip() + except (OSError, subprocess.CalledProcessError): + return os.environ.get(env, "unknown") + + +def _version() -> str: + text = (ROOT / "src" / "pyjpdfium" / "__init__.py").read_text(encoding="utf-8") + return re.search(r'__version__ = "([^"]+)"', text).group(1) + + +def glibc_floor(paths: list[Path]) -> tuple[int, int]: + newest = (2, 17) + for path in paths: + symbols = run("objdump", "-T", str(path)) + for major, minor in re.findall(r"GLIBC_(\d+)\.(\d+)", symbols): + newest = max(newest, (int(major), int(minor))) + return newest + + +def retag(wheel: Path, tag: str) -> Path: + """Rewrites the platform tag in the file name and WHEEL, and WHEEL's RECORD line.""" + old_tag = wheel.stem.rsplit("-", 1)[1] + target = wheel.with_name(wheel.name.replace(f"-{old_tag}.whl", f"-{tag}.whl")) + with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(target, "w", + zipfile.ZIP_DEFLATED) as out: + names = source.namelist() + wheel_name = next(n for n in names if n.endswith(".dist-info/WHEEL")) + record_name = next(n for n in names if n.endswith(".dist-info/RECORD")) + wheel_text = source.read(wheel_name).decode().replace(f"-{old_tag}\n", f"-{tag}\n") + digest = base64.urlsafe_b64encode(hashlib.sha256(wheel_text.encode()).digest()) + record = [] + for line in source.read(record_name).decode().splitlines(): + if line.startswith(f"{wheel_name},"): + line = f"{wheel_name},sha256={digest.rstrip(b'=').decode()},{len(wheel_text)}" + record.append(line) + for item in source.infolist(): + if item.filename == wheel_name: + out.writestr(item, wheel_text) + elif item.filename == record_name: + out.writestr(item, "\n".join(record) + "\n") + else: + out.writestr(item, source.read(item.filename)) + wheel.unlink() + return target + + +def main(jpdfium: Path, out: Path) -> Path: + arch_dir = {"x86_64": "linux-x64", "aarch64": "linux-arm64"}[platform.machine()] + out.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pyjpdfium-wheel-") as tmp: + work = Path(tmp) + dist = bundle(jpdfium, arch_dir, work) + system_libs = (work / "bundled-system-libs.txt").read_text().split() + staged = work / "staged" + ocr = jpdfium / "native" / "dist" / f"ocr-{arch_dir}" + stage_dirs([dist, *([ocr] if ocr.is_dir() else [])], staged) + licenses = collect_licenses(jpdfium, staged, system_libs) + (staged / "identity.json").write_text( + json.dumps(identity(jpdfium, staged, licenses, system_libs), indent=2) + "\n", + encoding="utf-8") + env = {**os.environ, "JPDFIUM_NATIVE_DIR": str(staged), "PYJPDFIUM_BUNDLE_NATIVES": "ON", + "JPDFIUM_INCLUDE_DIR": str(jpdfium / "native" / "bridge" / "include"), + "JPDFIUM_OCR_INCLUDE_DIR": str(jpdfium / "native" / "ocr" / "include")} + built = Path(tmp) / "wheel" + # A fresh CMake directory, so no cached library path from a dev build leaks in. + subprocess.run([sys.executable, "-m", "pip", "wheel", "--no-build-isolation", "--no-deps", + "-C", f"build-dir={work / 'cmake'}", str(ROOT), "-w", str(built)], + env=env, check=True) + wheel = next(built.glob("pyjpdfium-*.whl")) + with zipfile.ZipFile(wheel) as archive: + unpacked = Path(tmp) / "unpacked" + archive.extractall(unpacked) + elves = [p for p in unpacked.rglob("*.so*") if p.is_file()] + major, minor = glibc_floor(elves) + tagged = retag(wheel, f"manylinux_{major}_{minor}_{platform.machine()}") + final = out / tagged.name + shutil.move(tagged, final) + print(f"{final} ({final.stat().st_size / 2**20:.1f} MiB), sha256 {sha256(final)}") + return final + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit(__doc__) + main(Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()) diff --git a/tools/stage_natives.py b/tools/stage_natives.py index 1da9193..805b151 100644 --- a/tools/stage_natives.py +++ b/tools/stage_natives.py @@ -7,6 +7,9 @@ library with the same soname, so JPDFium's core would call into another PDFium. Staging gives our copies their own sonames, points RUNPATH at the staging directory only, and writes natives-manifest.json with every file's digest before and after. Linux only; needs patchelf. + +JPDFium stages its optional OCR module apart from the core, in dist/ocr-; when that +sibling of JPDFIUM_DIST exists, its libraries are staged too. """ from __future__ import annotations @@ -44,6 +47,11 @@ def is_elf(path: Path) -> bool: def stage(source: Path, dest: Path) -> dict: + ocr = source.parent / f"ocr-{source.name}" + return stage_dirs([source, *([ocr] if ocr.is_dir() else [])], dest) + + +def stage_dirs(sources: list[Path], dest: Path) -> dict: if not sys.platform.startswith("linux"): raise SystemExit("stage_natives.py stages Linux natives only") if shutil.which("patchelf") is None: @@ -54,7 +62,7 @@ def stage(source: Path, dest: Path) -> dict: staged: dict[str, Path] = {} origins: dict[str, Path] = {} # Real files only, each under its soname, so no symlink is needed after staging. - for path in sorted(source.iterdir()): + for path in sorted(p for source in sources for p in source.iterdir()): if path.is_symlink() or not path.is_file(): continue name = path.name @@ -91,7 +99,8 @@ def stage(source: Path, dest: Path) -> dict: "source": origins[name].name, "source_sha256": sha256(origins[name]), }) - manifest = {"source": str(source.resolve()), "renames": RENAMES, "files": entries} + manifest = {"sources": [str(s.resolve()) for s in sources], "renames": RENAMES, + "files": entries} (dest / MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") return manifest From ef078e5948b2e1c2710ae307cc1837fe39c7cf93 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:52:29 +0100 Subject: [PATCH 10/10] Bound OCR concurrency and qualify the balanced profile with measured benchmarks --- .github/workflows/ci.yml | 7 + README.md | 5 +- docs/COMPATIBILITY.md | 2 + docs/OCRMYPDF.md | 9 +- docs/PERFORMANCE.md | 54 ++++++++ src/pyjpdfium/cli.py | 5 +- src/pyjpdfium/models.py | 8 ++ src/pyjpdfium/ocr.py | 17 ++- src/pyjpdfium/ocrmypdf.py | 18 ++- tests/native/test_ocr.py | 5 +- tests/ocrmypdf/test_ocrmypdf_plugin.py | 11 ++ tests/parity/ocr_reference_runner.py | 17 +++ tests/unit/test_ocr_concurrency.py | 57 ++++++++ tools/bench_ocrmypdf.py | 184 +++++++++++++++++++++++++ 14 files changed, 379 insertions(+), 20 deletions(-) create mode 100644 docs/PERFORMANCE.md create mode 100644 tests/unit/test_ocr_concurrency.py create mode 100644 tools/bench_ocrmypdf.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51e56c2..549b96e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,11 @@ jobs: bash jpdfium/native/ocr/fetch-onnxruntime.sh linux-x64 python jpdfium/native/ocr/tools/fetch_profile.py \ jpdfium/native/ocr/profiles/pp-ocrv6-small.profile "$RUNNER_TEMP/ocr-profile" + # The balanced profile pins the same files; the fetcher reuses them after its digest check. + cp -r "$RUNNER_TEMP/ocr-profile" "$RUNNER_TEMP/ocr-profile-balanced" + python jpdfium/native/ocr/tools/fetch_profile.py \ + jpdfium/native/ocr/profiles/pp-ocrv6-small-balanced.profile \ + "$RUNNER_TEMP/ocr-profile-balanced" - name: Build the JPDFium core run: bash jpdfium/native/build-real.sh @@ -131,6 +136,8 @@ jobs: "$RUNNER_TEMP/ocr-reference/bin/pip" install --require-hashes --only-binary=:all: \ -r tests/parity/ocr-requirements.txt python -m pytest tests/parity/test_ocr_parity.py -q -rs + PYJPDFIUM_OCR_PROFILE="$RUNNER_TEMP/ocr-profile-balanced" \ + python -m pytest tests/parity/test_ocr_parity.py -q -rs - name: OCRmyPDF end to end with the pyjpdfium plugin env: diff --git a/README.md b/README.md index 3470f74..a92506c 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,9 @@ line = TextLine((300, 300, 1500, 400), (TextWord("Hello", (300, 300, 800, 400)), pyjpdfium.write_text_layer("text.pdf", [line], image_size=(2550, 3300), dpi=300) # OCR, when the natives include the OCR module (see JPDFium docs/design/OCR.md). -with pyjpdfium.OcrEngine("pp-ocrv6-small/") as engine: +with pyjpdfium.OcrEngine("pp-ocrv6-small-balanced/") as engine: result = engine.recognize("page.png") - pyjpdfium.write_text_layer("text.pdf", result.text_lines(), image_size=result.image_size, - dpi=300) + result.write_text_layer("text.pdf") # sized to the image, at the image's DPI ``` ## Command line and OCRmyPDF diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 2938345..c4ac401 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -64,6 +64,8 @@ models of a profile directory fetched by JPDFium's `native/ocr/tools/fetch_profi parity fixtures it reproduces the official `paddleocr` 3.7.0 pipeline exactly: the same polygons, the same text, and confidences within 1.2e-7 (JPDFium's `docs/design/OCR.md`). +- Two profiles: `pp-ocrv6-small` and `pp-ocrv6-small-balanced`, each matching the reference + pipeline at its own detection size ([PERFORMANCE.md](PERFORMANCE.md)). - PNG input is read as OpenCV's `imread(IMREAD_COLOR)` reads it: alpha dropped, 16-bit samples cut to their high byte. - Lines only: `OcrResult.word_geometry` is `"absent"`, and `text_lines()` keeps each line whole. diff --git a/docs/OCRMYPDF.md b/docs/OCRMYPDF.md index 6e2cb0c..3017e7c 100644 --- a/docs/OCRMYPDF.md +++ b/docs/OCRMYPDF.md @@ -5,7 +5,7 @@ and its invisible text layer writer. Tesseract and Ghostscript are not needed. ```bash pip install "pyjpdfium[ocrmypdf]" -export PYJPDFIUM_OCR_PROFILE=/path/to/pp-ocrv6-small # fetched with fetch_profile.py +export PYJPDFIUM_OCR_PROFILE=/path/to/pp-ocrv6-small-balanced # see fetch_profile.py ocrmypdf --plugin pyjpdfium.ocrmypdf --output-type pdf input.pdf output.pdf ``` @@ -27,7 +27,12 @@ without it. found while the visible page keeps it. - Engines load once per worker process and are shared by its threads. ONNX Runtime uses one thread per page by default (`--pyjpdfium-threads`), since OCRmyPDF already runs pages in - parallel. + parallel. `--pyjpdfium-max-concurrent N` caps the pages recognized at once in a process, to + bound memory. + +Speed, memory and accuracy against Tesseract, per profile, are in +[PERFORMANCE.md](PERFORMANCE.md). The balanced profile is the one to use for ordinary +documents. The plugin blocks OCRmyPDF's built-in Tesseract plugin, so Tesseract's checks and options do not apply. OCRmyPDF's own validation still reads `options.tesseract`, so the plugin registers that diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 0000000..d298fef --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,54 @@ +# Performance + +Measured with `tools/bench_ocrmypdf.py`. Each run is OCRmyPDF 17.13's real command line, offline, +on a 16-core x86-64 Linux container. The corpus is 12 scanned pages at 300 dpi, made from seeded +English text in Helvetica, Times and Courier at 11 to 13 pt, so the ground truth is exact. + +- Accuracy is the character error rate (CER) of the text PDFium extracts from the output. +- Memory is the peak resident size of the largest process. +- Tesseract is 5.3.4 from Ubuntu 24.04 with OCRmyPDF's defaults. + +## OCRmyPDF, one page at a time (`--jobs 1`) + +| Engine | Seconds per page | CPU seconds | Peak memory | CER | +|---|---|---|---|---| +| Tesseract | 1.20 | 14.2 | 127 MiB | 0.0001 | +| pyjpdfium, `pp-ocrv6-small` | 5.44 | 65.2 | 3427 MiB | 0.0002 | +| pyjpdfium, `pp-ocrv6-small-balanced` | 3.39 | 40.6 | 1342 MiB | 0.0001 | + +## OCRmyPDF, eight pages at once (`--jobs 8`) + +| Engine | Seconds per page | CPU seconds | Peak memory | CER | +|---|---|---|---|---| +| Tesseract | 0.37 | 15.8 | 176 MiB | 0.0001 | +| pyjpdfium, `pp-ocrv6-small` | 2.29 | 155.9 | 14710 MiB | 0.0002 | +| pyjpdfium, balanced | 1.22 | 79.0 | 3561 MiB | 0.0001 | +| pyjpdfium, balanced, `--pyjpdfium-max-concurrent 4` | 1.33 | 51.9 | 3118 MiB | 0.0001 | +| pyjpdfium, balanced, `--pyjpdfium-max-concurrent 2` | 1.89 | 43.7 | 2301 MiB | 0.0001 | + +## What the numbers say + +- On clean 300 dpi scans of English, Tesseract is faster and smaller at the same accuracy. The + plugin's case is OCR without Tesseract or Ghostscript, and the PP-OCR models. +- Detection size dominates memory. `pp-ocrv6-small` detects at the page's full resolution + (about 2560 x 3328 pixels here), which is PaddleOCR's default. `pp-ocrv6-small-balanced` caps + the longer side at 1600 pixels. It is 1.6 times faster alone and 1.9 times faster at eight + jobs, uses 2.6 and 4.1 times less memory, and loses no accuracy on this corpus. Both profiles + match the official PaddleOCR pipeline exactly at their own settings + (`tests/parity/test_ocr_parity.py`). +- With the balanced profile, recognition now takes most of each page: about 2.2 of 3.4 + seconds for 34 long lines. Its inputs are fixed by the reference pipeline. +- Each page being recognized holds its own model activations. `--pyjpdfium-max-concurrent` + (or `OcrEngine(max_concurrent=...)`) caps how many run at once in one process, trading wall + time for memory. It also cuts CPU time, since eight concurrent runs compete for memory + bandwidth. +- Opening an engine, which reads and checks the 31 MB of models, takes 0.2 seconds. The plugin + opens one per worker process and shares it across that process's threads. + +## Recommendations + +- Use `pp-ocrv6-small-balanced` for ordinary documents. Keep `pp-ocrv6-small` for small text, + where full-resolution detection can find what the capped size misses. +- On machines with less than about 0.5 GB per OCRmyPDF job, set `--pyjpdfium-max-concurrent`. +- Outside OCRmyPDF, `OcrEngine(threads=0)` lets ONNX Runtime use every core for one page: 1.08 + seconds per page with the balanced profile and four threads, against 2.67 with one. diff --git a/src/pyjpdfium/cli.py b/src/pyjpdfium/cli.py index 7fed9e5..e00e069 100644 --- a/src/pyjpdfium/cli.py +++ b/src/pyjpdfium/cli.py @@ -75,7 +75,7 @@ def _profile(args: argparse.Namespace) -> Path: def cmd_ocr(args: argparse.Namespace) -> int: - from . import OcrEngine, rasterize_page, write_text_layer + from . import OcrEngine, rasterize_page source = Path(args.input) with tempfile.TemporaryDirectory(prefix="pyjpdfium-ocr-") as work: @@ -86,8 +86,7 @@ def cmd_ocr(args: argparse.Namespace) -> int: with OcrEngine(_profile(args), threads=args.threads) as engine: result = engine.recognize(image) if args.layer: - write_text_layer(args.layer, result.text_lines(), image_size=result.image_size, - dpi=result.dpi) + result.write_text_layer(args.layer) if args.json: lines = [{"text": line.text, "polygon": line.polygon, "confidence": line.confidence} for line in result.lines] diff --git a/src/pyjpdfium/models.py b/src/pyjpdfium/models.py index 331b3b0..4ccb9f8 100644 --- a/src/pyjpdfium/models.py +++ b/src/pyjpdfium/models.py @@ -190,6 +190,14 @@ def text_lines(self) -> tuple[TextLine, ...]: out.append(TextLine(line.box, (TextWord(line.text, line.box, quad),), quad)) return tuple(out) + def write_text_layer(self, output, *, durable: bool = False, + max_bytes: int | None = None) -> None: + """The invisible text layer for this image: write_text_layer with its size and DPI.""" + from .textlayer import write_text_layer + + write_text_layer(output, self.text_lines(), image_size=self.image_size, dpi=self.dpi, + durable=durable, max_bytes=max_bytes) + @dataclass(frozen=True) class RectifyResult: diff --git a/src/pyjpdfium/ocr.py b/src/pyjpdfium/ocr.py index e863972..98104ad 100644 --- a/src/pyjpdfium/ocr.py +++ b/src/pyjpdfium/ocr.py @@ -7,7 +7,9 @@ from __future__ import annotations +import contextlib import os +import threading from typing import Union from ._loader import ocr_native @@ -21,9 +23,15 @@ class OcrEngine(_Handle): """A verified profile with its two models loaded. Safe to share across threads.""" - def __init__(self, profile: StrPath, *, threads: int = 0) -> None: + def __init__( + self, profile: StrPath, *, threads: int = 0, max_concurrent: int | None = None + ) -> None: if threads < 0: raise InvalidArgumentError("threads must not be negative") + if max_concurrent is not None and max_concurrent < 1: + raise InvalidArgumentError("max_concurrent must be at least 1") + # Each recognition holds its own model activations, so this bounds peak memory. + self._slots = threading.BoundedSemaphore(max_concurrent) if max_concurrent else None lib = ocr_native() token = lib.engine_open(os.fsdecode(os.fspath(profile)), threads) super().__init__(token, lib.engine_close) @@ -38,9 +46,10 @@ def recognize(self, image: Bitmap | StrPath, *, max_pixels: int | None = None) - return self._recognize(bitmap, max_pixels) def _recognize(self, bitmap: Bitmap, max_pixels: int | None) -> OcrResult: - width, height, dpi_x, dpi_y, lines = ocr_native().recognize( - self._require_open(), bitmap._require_open(), max_pixels or 0 - ) + with self._slots or contextlib.nullcontext(): + width, height, dpi_x, dpi_y, lines = ocr_native().recognize( + self._require_open(), bitmap._require_open(), max_pixels or 0 + ) return OcrResult( image_size=(width, height), dpi=(dpi_x, dpi_y), diff --git a/src/pyjpdfium/ocrmypdf.py b/src/pyjpdfium/ocrmypdf.py index 0182270..f543064 100644 --- a/src/pyjpdfium/ocrmypdf.py +++ b/src/pyjpdfium/ocrmypdf.py @@ -59,6 +59,7 @@ class PyjpdfiumOptions(BaseModel): profile: str | None = None threads: int = 1 # ONNX Runtime threads per page; OCRmyPDF already runs pages in parallel + max_concurrent: int | None = None # pages recognized at once per process; bounds memory def ocrmypdf_version() -> tuple[int, ...]: @@ -108,6 +109,10 @@ def add_options(parser): "--pyjpdfium-threads", dest="pyjpdfium_threads", type=int, default=1, metavar="N", help="ONNX Runtime threads per page (default 1; 0 lets ONNX Runtime choose)", ) + group.add_argument( + "--pyjpdfium-max-concurrent", dest="pyjpdfium_max_concurrent", type=int, metavar="N", + help="Most pages recognized at once in one process, to bound memory (default: --jobs)", + ) for action in parser._actions: # noqa: SLF001 - argparse exposes no public lookup if "--ocr-engine" in action.option_strings and ENGINE_NAME not in action.choices: action.choices = [*action.choices, ENGINE_NAME] @@ -133,6 +138,8 @@ def check_options(options): raise BadArgsError(f"Tesseract options do not apply to this engine: {', '.join(changed)}") if options.pyjpdfium.threads < 0: raise BadArgsError("--pyjpdfium-threads must not be negative") + if options.pyjpdfium.max_concurrent is not None and options.pyjpdfium.max_concurrent < 1: + raise BadArgsError("--pyjpdfium-max-concurrent must be at least 1") if options.ocr_engine == "none": return # OCRmyPDF's null engine; only this plugin's rasterizer is used if options.pdf_renderer == "auto": @@ -179,7 +186,7 @@ def filter_ocr_image(page, image): _engine_lock = threading.Lock() -_engine: tuple[int, tuple[str, int], pyjpdfium.OcrEngine] | None = None +_engine: tuple[int, tuple[str, int, int | None], pyjpdfium.OcrEngine] | None = None def _get_engine(options: OcrOptions) -> pyjpdfium.OcrEngine: @@ -188,10 +195,11 @@ def _get_engine(options: OcrOptions) -> pyjpdfium.OcrEngine: directory = profile_dir(options) if directory is None: raise BadArgsError(f"pass --pyjpdfium-profile DIR or set {PROFILE_ENV}") - key = (os.fspath(directory), options.pyjpdfium.threads) + key = (os.fspath(directory), options.pyjpdfium.threads, options.pyjpdfium.max_concurrent) with _engine_lock: if _engine is None or _engine[0] != os.getpid() or _engine[1] != key: - _engine = (os.getpid(), key, pyjpdfium.OcrEngine(key[0], threads=key[1])) + engine = pyjpdfium.OcrEngine(key[0], threads=key[1], max_concurrent=key[2]) + _engine = (os.getpid(), key, engine) return _engine[2] @@ -239,9 +247,7 @@ def generate_hocr(input_file, output_hocr, output_text, options) -> None: @staticmethod def generate_pdf(input_file, output_pdf, output_text, options) -> None: result = _recognize(Path(input_file), options) - pyjpdfium.write_text_layer( - output_pdf, result.text_lines(), image_size=result.image_size, dpi=result.dpi - ) + result.write_text_layer(output_pdf) Path(output_text).write_text(_sidecar(result), encoding="utf-8") @staticmethod diff --git a/tests/native/test_ocr.py b/tests/native/test_ocr.py index 1503ac6..acc8674 100644 --- a/tests/native/test_ocr.py +++ b/tests/native/test_ocr.py @@ -9,6 +9,7 @@ import pyjpdfium from pyjpdfium import errors +from pyjpdfium.profiles import read_profile PROFILE = os.environ.get("PYJPDFIUM_OCR_PROFILE") pytestmark = [ @@ -50,7 +51,7 @@ def engine(): def test_engine_identity(engine): - assert engine.profile == "pp-ocrv6-small" + assert engine.profile == read_profile(PROFILE).id assert engine.runtime_version == "1.30.0" @@ -78,7 +79,7 @@ def test_ocr_feeds_the_text_layer(tmp_path, engine): pyjpdfium.rasterize_page(page_pdf(tmp_path), 0, png, device="pnggray", raster_dpi=300) result = engine.recognize(png) layer = tmp_path / "layer.pdf" - pyjpdfium.write_text_layer(layer, result.text_lines(), image_size=result.image_size, dpi=300) + result.write_text_layer(layer) with pyjpdfium.open_document(layer) as document, document.page(0) as page: assert " ".join(page.text().split()) == " ".join(LINES) diff --git a/tests/ocrmypdf/test_ocrmypdf_plugin.py b/tests/ocrmypdf/test_ocrmypdf_plugin.py index 3037849..1f346ff 100644 --- a/tests/ocrmypdf/test_ocrmypdf_plugin.py +++ b/tests/ocrmypdf/test_ocrmypdf_plugin.py @@ -200,6 +200,16 @@ def test_pages_in_parallel_workers(tmp_path, workers): assert page_texts(out) == [EXPECTED] * 3 +def test_one_page_at_a_time_bounds_memory_and_reads_the_same(tmp_path): + source = tmp_path / "in.pdf" + page = scanned_page(tmp_path) + source.write_bytes(pdf_bytes([page, page])) + out = tmp_path / "out.pdf" + ok(ocrmypdf("--output-type", "pdf", "--jobs", "2", "--pyjpdfium-max-concurrent", "1", + str(source), str(out))) + assert page_texts(out) == [EXPECTED] * 2 + + def test_fpdf2_reference_renderer_reads_the_same_words(tmp_path): source = tmp_path / "in.pdf" source.write_bytes(pdf_bytes([scanned_page(tmp_path)])) @@ -221,6 +231,7 @@ def test_fpdf2_reference_renderer_reads_the_same_words(tmp_path): (("--output-type", "pdf", "-l", "deu"), 3, "deu"), (("--output-type", "pdf", "--tesseract-timeout", "5"), 2, "--tesseract-timeout"), (("--output-type", "pdf", "--pyjpdfium-threads", "-1"), 1, "threads"), + (("--output-type", "pdf", "--pyjpdfium-max-concurrent", "0"), 1, "max-concurrent"), ], ) def test_unsupported_modes_are_refused(tmp_path, args, code, message): diff --git a/tests/parity/ocr_reference_runner.py b/tests/parity/ocr_reference_runner.py index a34eb02..189e28c 100644 --- a/tests/parity/ocr_reference_runner.py +++ b/tests/parity/ocr_reference_runner.py @@ -22,6 +22,22 @@ def model_dir(profile: Path, work: Path, role: str, stem: str) -> Path: return target +def pipeline_args(profile: Path) -> dict: + """The profile's parameters as PaddleOCR arguments; one PaddleOCR cannot take is refused.""" + params = dict(line.split()[1:3] for line in (profile / "profile.txt").read_text().splitlines() + if line.startswith("param ")) + if params.get("det_max_side_limit", "4000") != "4000": + raise SystemExit("PaddleOCR fixes det_max_side_limit at 4000") + return { + "text_det_limit_side_len": int(params["det_limit_side_len"]), + "text_det_limit_type": params["det_limit_type"], + "text_det_thresh": float(params["det_thresh"]), + "text_det_box_thresh": float(params["det_box_thresh"]), + "text_det_unclip_ratio": float(params["det_unclip_ratio"]), + "text_recognition_batch_size": int(params["rec_batch_size"]), + } + + def main() -> None: profile, images = Path(sys.argv[1]), sys.argv[2:] os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True" @@ -39,6 +55,7 @@ def main() -> None: use_doc_orientation_classify=False, use_doc_unwarping=False, use_textline_orientation=False, + **pipeline_args(profile), ) for image in images: for res in ocr.predict(image): diff --git a/tests/unit/test_ocr_concurrency.py b/tests/unit/test_ocr_concurrency.py new file mode 100644 index 0000000..5e28f9e --- /dev/null +++ b/tests/unit/test_ocr_concurrency.py @@ -0,0 +1,57 @@ +import threading +import time + +import pytest + +from pyjpdfium import errors, ocr + + +class FakeNative: + """Stands in for the OCR extension and records how many recognitions overlap.""" + + def __init__(self): + self.lock = threading.Lock() + self.active = self.peak = 0 + + def engine_open(self, profile, threads): + return 1 + + def engine_info(self, token): + return "fake", "0" + + def engine_close(self, token): + pass + + def recognize(self, engine, bitmap, max_pixels): + with self.lock: + self.active += 1 + self.peak = max(self.peak, self.active) + time.sleep(0.05) + with self.lock: + self.active -= 1 + return 10, 10, 300.0, 300.0, [] + + +class FakeBitmap: + def _require_open(self): + return 7 + + +@pytest.mark.parametrize(("limit", "peak"), [(None, 6), (1, 1), (2, 2)]) +def test_max_concurrent_bounds_overlapping_recognitions(monkeypatch, limit, peak): + fake = FakeNative() + monkeypatch.setattr(ocr, "ocr_native", lambda: fake) + engine = ocr.OcrEngine("profile", max_concurrent=limit) + threads = [threading.Thread(target=engine._recognize, args=(FakeBitmap(), None)) + for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + assert fake.peak == peak + + +def test_max_concurrent_must_be_positive(monkeypatch): + monkeypatch.setattr(ocr, "ocr_native", FakeNative) + with pytest.raises(errors.InvalidArgumentError): + ocr.OcrEngine("profile", max_concurrent=0) diff --git a/tools/bench_ocrmypdf.py b/tools/bench_ocrmypdf.py new file mode 100644 index 0000000..25fc65c --- /dev/null +++ b/tools/bench_ocrmypdf.py @@ -0,0 +1,184 @@ +"""Benchmarks OCRmyPDF with Tesseract against OCRmyPDF with the pyjpdfium plugin. + + python tools/bench_ocrmypdf.py OCRMYPDF_PYTHON WORK_DIR [--pages N] [--jobs 1,8] + [--variant "LABEL|EXTRA OCRMYPDF ARGS" ...] + +Generates scanned pages of seeded English text in three base-14 fonts, so the ground truth is +known. It then runs each engine's real OCRmyPDF command line, and reports for each run: + +- wall and CPU time, and the peak RSS of any one process; +- character and word error rates of the text that PDFium extracts from the output. + +Needs Tesseract with English data for the Tesseract runs, and PYJPDFIUM_OCR_PROFILE. +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import subprocess +import sys +import zlib +from pathlib import Path + +import pyjpdfium + +WORDS = ( + "the of and to in is was for that with as on by at from his her are were this which be or " + "an have had not but they all one their there been has more when who will would other into " + "time only new some could these two may first then do any like my now over such our man me " + "even most made after also did many before must through back years where much your way well " + "down should because each just those people how too little state good very make world still " + "own see men work long get here between both life being under never day same another know " + "while last might us great old year off come since against go came right used take three " + "invoice account payment balance report total amount number date order customer service " + "quarter annual review budget meeting schedule project office address request policy" +).split() +FONTS = ("Helvetica", "Times-Roman", "Courier") + + +def page_lines(rng: random.Random, count: int) -> list[str]: + lines = [] + for _ in range(count): + words = [rng.choice(WORDS) for _ in range(rng.randint(6, 10))] + if rng.random() < 0.3: + words.insert(rng.randrange(len(words)), str(rng.randint(10, 99999))) + words[0] = words[0].capitalize() + lines.append(" ".join(words) + rng.choice([".", ",", "", ":"])) + return lines + + +def pdf(pages: list[tuple[str, str, bytes | None]]) -> bytes: + objects = [b"<< /Type /Catalog /Pages 2 0 R >>", b""] + kids = [] + for extra, content, image in pages: + number = len(objects) + 1 + kids.append(f"{number} 0 R") + resources = f"/Resources << /XObject << /Im0 {number + 2} 0 R >> >>" if image else "" + objects.append(f"<< /Type /Page /Parent 2 0 R {extra} {resources} " + f"/Contents {number + 1} 0 R >>".encode()) + data = content.encode("latin-1") + objects.append(b"<< /Length %d >>\nstream\n" % len(data) + data + b"\nendstream") + if image: + objects.append(image) + objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>".encode() + out, offsets = bytearray(b"%PDF-1.7\n"), [] + for index, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += b"%d 0 obj\n" % index + body + b"\nendobj\n" + xref = len(out) + out += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + out += b"".join(b"%010d 00000 n \n" % o for o in offsets) + out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, + xref) + return bytes(out) + + +def corpus(work: Path, pages: int) -> tuple[Path, list[str]]: + rng = random.Random(20261011) + truth, scanned = [], [] + for index in range(pages): + font, size = FONTS[index % len(FONTS)], 11 + index % 3 + lines = page_lines(rng, 34) + content = "".join(f"BT /F1 {size} Tf 60 {740 - 20 * i} Td ({line}) Tj ET " + for i, line in enumerate(lines)) + resources = (f"/Resources << /Font << /F1 << /Type /Font /Subtype /Type1 " + f"/BaseFont /{font} >> >> >>") + source = work / f"source-{index}.pdf" + source.write_bytes(pdf([(f"/MediaBox [0 0 612 792] {resources}", content, None)])) + png = work / f"scan-{index}.png" + pyjpdfium.rasterize_page(source, 0, png, device="pnggray", raster_dpi=300) + with pyjpdfium.load_image(png) as bitmap: + raw = bitmap.tobytes() + gray = raw[0::3] if bitmap.pixel_format.name == "RGB8" else raw + width, height = bitmap.width, bitmap.height + data = zlib.compress(gray) + image = (b"<< /Type /XObject /Subtype /Image /Width %d /Height %d /ColorSpace " + b"/DeviceGray /BitsPerComponent 8 /Filter /FlateDecode /Length %d >>\n" + b"stream\n" % (width, height, len(data)) + data + b"\nendstream") + scanned.append(("/MediaBox [0 0 612 792]", "q 612 0 0 792 0 0 cm /Im0 Do Q", image)) + truth.append(" ".join(lines)) + path = work / "scanned.pdf" + path.write_bytes(pdf(scanned)) + return path, truth + + +def distance(a: list, b: list) -> int: + previous = list(range(len(b) + 1)) + for i, x in enumerate(a, start=1): + current = [i] + for j, y in enumerate(b, start=1): + current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (x != y))) + previous = current + return previous[-1] + + +def accuracy(output: Path, truth: list[str]) -> tuple[float, float]: + chars = words = char_errors = word_errors = 0 + with pyjpdfium.open_document(output) as document: + for index, expected in enumerate(truth): + with document.page(index) as page: + got = " ".join(page.text().split()) + char_errors += distance(list(got), list(expected)) + word_errors += distance(got.split(), expected.split()) + chars += len(expected) + words += len(expected.split()) + return char_errors / chars, word_errors / words + + +MEASURE = r""" +import json, resource, subprocess, sys, time +start = time.perf_counter() +done = subprocess.run(sys.argv[1:], capture_output=True, text=True) +wall = time.perf_counter() - start +usage = resource.getrusage(resource.RUSAGE_CHILDREN) +print(json.dumps({"rc": done.returncode, "wall": wall, "cpu": usage.ru_utime + usage.ru_stime, + "rss_mib": usage.ru_maxrss / 1024, "stderr": done.stderr[-2000:]})) +""" + + +def run(python: str, args: list[str]) -> dict: + done = subprocess.run([sys.executable, "-c", MEASURE, python, "-m", "ocrmypdf", *args], + capture_output=True, text=True, check=True) + result = json.loads(done.stdout) + if result["rc"] != 0: + raise SystemExit(f"ocrmypdf {' '.join(args)} failed:\n{result['stderr']}") + return result + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("python") + parser.add_argument("work", type=Path) + parser.add_argument("--pages", type=int, default=12) + parser.add_argument("--jobs", default="1,8") + parser.add_argument("--variant", action="append", default=[], + help="LABEL|ARGS; the default compares Tesseract with the plugin") + args = parser.parse_args() + args.work.mkdir(parents=True, exist_ok=True) + source, truth = corpus(args.work, args.pages) + variants = args.variant or ["tesseract|", "pyjpdfium|--plugin pyjpdfium.ocrmypdf"] + engines = {} + for variant in variants: + label, _, extra = variant.partition("|") + engines[label] = ["--output-type", "pdf", *extra.split()] + results = [] + for jobs in (int(j) for j in args.jobs.split(",")): + for name, engine_args in engines.items(): + out = args.work / f"{name}-{jobs}.pdf" + measured = run(args.python, [*engine_args, "--jobs", str(jobs), str(source), str(out)]) + cer, wer = accuracy(out, truth) + row = {"engine": name, "jobs": jobs, "pages": args.pages, + "wall_s": round(measured["wall"], 2), + "s_per_page": round(measured["wall"] / args.pages, 3), + "cpu_s": round(measured["cpu"], 1), "peak_rss_mib": round(measured["rss_mib"]), + "cer": round(cer, 4), "wer": round(wer, 4)} + results.append(row) + print(json.dumps(row), flush=True) + (args.work / "results.json").write_text(json.dumps(results, indent=2) + "\n") + + +if __name__ == "__main__": + main()