Reduce Excel conversion and final-result memory

backendN/app/common/app_factory.py

@@ -24,10 +24,10 @@
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
 
-from .config import Settings
+from .config import Settings, deployment_revision
 from .errors import ConversionRejected, EngineUnavailable
 from .files import read_limited, sanitize_filename, split_name, title_from_stem
-from .profiles import PROFILES, finalize_markdown
+from .profiles import PROFILES, count_words, finalize_markdown
 from .security import require_internal_secret
 from .storage import StorageClient, StorageError
 
@@ -100,7 +100,7 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool
             markdown, warning = converter.convert(data, ext, filename=original_name, title=title)
             if raw:
                 content = markdown.strip() + "\n"
-                words = len(content.split())
+                words = count_words(content)
                 tokens = round(words * 1.33)
             else:
                 content, words, tokens = finalize_markdown(
@@ -171,12 +171,15 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool
 
     @app.get("/api/v1/health")
     def health() -> dict:
-        return {
+        body = {
             "status": "ok",
             "role": settings.role,
             "instance": settings.instance,
             "version": settings.version,
         }
+        if revision := deployment_revision():
+            body["revision"] = revision
+        return body
 
     @app.get("/api/v1/ready")
     def ready() -> JSONResponse:

backendN/app/common/config.py

@@ -3,10 +3,20 @@
 from __future__ import annotations
 
 import os
+import re
 from dataclasses import dataclass
 
 VALID_ROLES = ("normal", "ocr", "archive")
 
+
+def deployment_revision() -> str | None:
+    """Allowlisted public build identity, never arbitrary environment values."""
+    for name in ("RENDER_GIT_COMMIT", "VERCEL_GIT_COMMIT_SHA"):
+        value = os.environ.get(name, "")
+        if re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", value):
+            return value.lower()
+    return None
+
 # Upload byte limits per role; MAX_UPLOAD_BYTES overrides. Keep in sync with
 # MAX_FILE_SIZE / MAX_IMAGE_FILE_SIZE in frontend/lib/formats.js.
 DEFAULT_MAX_UPLOAD_BYTES = {

backendN/app/common/profiles.py

@@ -7,6 +7,19 @@
 PROFILES = ("Standard", "Clean", "Compact", "RAG-ready")
 
 _H1_BLOCK = re.compile(r"^(# .+\n\n)")
+COUNT_CHUNK_CHARS = 64 * 1024
+
+
+def count_words(content: str) -> int:
+    """Exact str.split() count with bounded temporary strings/word arrays."""
+    count = 0
+    for start in range(0, len(content), COUNT_CHUNK_CHARS):
+        chunk = content[start:start + COUNT_CHUNK_CHARS]
+        count += len(chunk.split())
+        if start and not content[start - 1].isspace() and not chunk[0].isspace():
+            # A word spanning a chunk boundary was counted in both chunks.
+            count -= 1
+    return count
 
 
 def ensure_title(markdown: str, title: str) -> str:
@@ -48,5 +61,5 @@ def finalize_markdown(
             r"(^## [^\n]+)", r"<!-- chunk-boundary -->\n\1", content, flags=re.M
         )
 
-    words = len(content.split())
+    words = count_words(content)
     return content, words, round(words * 1.33)

backendN/app/converter.py

@@ -56,8 +56,10 @@ def __init__(self) -> None:
             return
         self._md = MarkItDown(enable_plugins=False)
         from app.csv_converter import register_csv_converter
+        from app.excel_converter import register_excel_converters
 
         register_csv_converter(self._md)
+        register_excel_converters(self._md)
         self._stream_info = StreamInfo
         self._import_error = ""
 

backendN/app/excel_converter.py

@@ -0,0 +1,78 @@
+"""Same pandas/HTML Excel conversion, retaining one DataFrame at a time."""
+from __future__ import annotations
+
+from contextlib import closing
+from importlib.metadata import version
+import io
+
+from markitdown import DocumentConverterResult
+from markitdown.converters import _xlsx_converter as upstream
+
+VERIFIED_VERSIONS = frozenset({"0.1.8", "0.1.9"})
+
+
+class _SheetViewRepair(TypeError):
+    """Only pandas load/parse failures may trigger the upstream XLSX repair."""
+
+
+def _read(call, *args, repair=False, **kwargs):
+    try:
+        return call(*args, **kwargs)
+    except TypeError as error:
+        if repair and "showZeroes" in str(error):
+            raise _SheetViewRepair(str(error)) from error
+        raise
+
+
+def _convert_workbook(file_stream, engine, html_converter, options, *, repair=False):
+    # xlrd otherwise keeps every sheet's raw cells as well as pandas frames.
+    # Older BIFF versions may disable on-demand loading; retain their fallback.
+    engine_options = {"engine_kwargs": {"on_demand": True}} if engine == "xlrd" else {}
+    workbook = _read(upstream.pd.ExcelFile, file_stream, engine=engine, repair=repair, **engine_options)
+    with workbook, io.StringIO() as output:
+        for name in workbook.sheet_names:
+            frame = _read(workbook.parse, sheet_name=name, repair=repair)
+            if engine == "xlrd" and workbook.book.on_demand:
+                workbook.book.unload_sheet(name)
+            html = frame.to_html(index=False)
+            table = html_converter.convert_string(html, **options).markdown.strip()
+            output.write(f"## {name}\n")
+            output.write(table)
+            output.write("\n\n")
+            # Do not retain the previous sheet while parsing/rendering the next.
+            del frame, html, table
+        return DocumentConverterResult(markdown=output.getvalue().strip())
+
+
+class SheetwiseXlsxConverter(upstream.XlsxConverter):
+    def convert(self, file_stream, stream_info, **kwargs):
+        # Preserve upstream missing-dependency errors and custom image hooks.
+        if (upstream._xlsx_dependency_exc_info is not None
+                or type(self)._image_to_html is not upstream.XlsxConverter._image_to_html):
+            return super().convert(file_stream, stream_info, **kwargs)
+        start = file_stream.tell()
+        try:
+            return _convert_workbook(file_stream, "openpyxl", self._html_converter, kwargs, repair=True)
+        except _SheetViewRepair:
+            # Discard the partial text before retrying the entire repaired book.
+            with closing(upstream._repair_sheetview_show_zeroes(file_stream, start)) as repaired:
+                return _convert_workbook(repaired, "openpyxl", self._html_converter, kwargs)
+
+
+class SheetwiseXlsConverter(upstream.XlsConverter):
+    def convert(self, file_stream, stream_info, **kwargs):
+        if upstream._xls_dependency_exc_info is not None:
+            return super().convert(file_stream, stream_info, **kwargs)
+        return _convert_workbook(file_stream, "xlrd", self._html_converter, kwargs)
+
+
+def register_excel_converters(engine) -> bool:
+    """Unknown versions retain their built-in converters, including image hooks."""
+    if version("markitdown") not in VERIFIED_VERSIONS:
+        return False
+    if (not callable(getattr(upstream, "_repair_sheetview_show_zeroes", None))
+            or not hasattr(upstream.XlsxConverter, "_image_to_html")):
+        return False
+    engine.register_converter(SheetwiseXlsxConverter())
+    engine.register_converter(SheetwiseXlsConverter())
+    return True

backendN/requirements-dev.txt

@@ -2,3 +2,5 @@
 pytest>=8
 httpx>=0.27
 uvicorn>=0.30
+# Generates real legacy XLS fixtures; not a runtime dependency.
+xlwt==1.3.0

backendN/tests/excel_fixtures.py

@@ -0,0 +1,85 @@
+"""Generated XLSX/XLS fixtures; no binary workbooks or private data in Git."""
+from datetime import datetime
+import io
+import zipfile
+
+
+def make_dense_book(ext, sheets, rows):
+    """Benchmark fixture generation is separate from measured conversion."""
+    output = io.BytesIO()
+    if ext == "xlsx":
+        from openpyxl import Workbook
+        book = Workbook(write_only=True)
+        for index in range(sheets):
+            sheet = book.create_sheet(f"Sheet-{index}")
+            sheet.append(["id", "name", "amount", "note"])
+            for row in range(rows):
+                sheet.append([row, f"sheet-{index}-row-{row}", row * 3.5, f"generated café 日本語 {index}-{row}"])
+        book.save(output)
+        book.close()
+    else:
+        import xlwt
+        book = xlwt.Workbook()
+        for index in range(sheets):
+            sheet = book.add_sheet(f"Sheet-{index}")
+            for column, value in enumerate(["id", "name", "amount", "note"]):
+                sheet.write(0, column, value)
+            for row in range(rows):
+                for column, value in enumerate([row, f"sheet-{index}-row-{row}", row * 3.5, f"generated café 日本語 {index}-{row}"]):
+                    sheet.write(row + 1, column, value)
+        book.save(output)
+    return output.getvalue()
+
+
+def make_book(ext="xlsx", *, legacy_view=False, image=False):
+    sheets = [
+        ("Mixed 日本語", [["Name", "Name", "Amount", "Date", "Flag"],
+          ["café | pipe", "line\nbreak", 1.23456789, datetime(2024, 2, 3, 4, 5), True],
+          ["<tag>&", "", 1e-9, None, False], ["NaN", "NULL", -1000000000.5],
+          [None] * 5, ["last-first-sheet", 'quote"value', 123.0]]),
+        ("Empty", []),
+        ("Results", [["id", "value"], [1, "before blank"], [], [2, "終わり🙂"]]),
+    ]
+    output = io.BytesIO()
+    if ext == "xls":
+        import xlwt
+        book = xlwt.Workbook()
+        date_style = xlwt.easyxf(num_format_str="YYYY-MM-DD HH:MM:SS")
+        for name, rows in sheets:
+            sheet = book.add_sheet(name)
+            for r, row in enumerate(rows):
+                for c, value in enumerate(row):
+                    if value is not None:
+                        sheet.write(r, c, value, date_style if isinstance(value, datetime) else xlwt.Style.default_style)
+        book.save(output)
+        return output.getvalue()
+    from openpyxl import Workbook
+    book = Workbook()
+    book.remove(book.active)
+    for name, rows in sheets:
+        sheet = book.create_sheet(name)
+        for row in rows:
+            sheet.append(row)
+    book["Results"]["D2"] = "=SUM(A2:A4)"
+    book["Results"].merge_cells("F2:G2")
+    book["Results"]["F2"] = "merged value"
+    book["Empty"].sheet_state = "hidden"
+    if image:
+        from PIL import Image
+        from openpyxl.drawing.image import Image as WorkbookImage
+        pixels = io.BytesIO()
+        Image.new("RGB", (4, 4), "white").save(pixels, format="PNG")
+        pixels.seek(0)
+        book["Results"].add_image(WorkbookImage(pixels), "A8")
+    book.save(output)
+    book.close()
+    if not legacy_view:
+        return output.getvalue()
+    repaired_fixture = io.BytesIO()
+    with zipfile.ZipFile(io.BytesIO(output.getvalue())) as source, zipfile.ZipFile(repaired_fixture, "w", zipfile.ZIP_DEFLATED) as target:
+        for item in source.infolist():
+            data = source.read(item.filename)
+            if item.filename.startswith("xl/worksheets/"):
+                data = data.replace(b'<sheetView ', b'<sheetView showZeroes="1" ')
+            target.writestr(item, data)
+    return repaired_fixture.getvalue()

backendN/tests/test_common.py

@@ -67,6 +67,29 @@ def test_finalize_counts_words_and_tokens():
     assert tokens == round(words * 1.33)
 
 
+def test_health_exposes_only_valid_provider_revision(client, monkeypatch):
+    monkeypatch.delenv("RENDER_GIT_COMMIT", raising=False)
+    monkeypatch.delenv("VERCEL_GIT_COMMIT_SHA", raising=False)
+    assert client.get("/api/v1/health").json() == {"status": "ok", "role": "normal", "instance": "N1", "version": "test"}
+    monkeypatch.setenv("RENDER_GIT_COMMIT", "a" * 40)
+    monkeypatch.setenv("UNRELATED_SECRET", "do-not-expose")
+    assert client.get("/api/v1/health").json() == {"status": "ok", "role": "normal", "instance": "N1", "version": "test", "revision": "a" * 40}
+
+
+def test_deployment_revision_is_limited_to_provider_commit_identifiers(monkeypatch):
+    from app.common.config import deployment_revision
+    monkeypatch.delenv("RENDER_GIT_COMMIT", raising=False)
+    monkeypatch.delenv("VERCEL_GIT_COMMIT_SHA", raising=False)
+    monkeypatch.setenv("UNRELATED_SECRET", "do-not-expose")
+    assert deployment_revision() is None
+    monkeypatch.setenv("RENDER_GIT_COMMIT", "not-a-commit")
+    assert deployment_revision() is None
+    monkeypatch.setenv("VERCEL_GIT_COMMIT_SHA", "A" * 40)
+    assert deployment_revision() == "a" * 40
+    monkeypatch.setenv("RENDER_GIT_COMMIT", "b" * 40)
+    assert deployment_revision() == "b" * 40
+
+
 def test_common_package_in_sync_with_other_backends():
     here = Path(__file__).resolve().parents[1] / "app" / "common"
     root = here.parents[2]