diff --git a/backendN/app/common/app_factory.py b/backendN/app/common/app_factory.py index f0be84f..08f92f8 100644 --- a/backendN/app/common/app_factory.py +++ b/backendN/app/common/app_factory.py @@ -24,10 +24,10 @@ from fastapi.responses import JSONResponse from pydantic import BaseModel, Field -from .config import Settings +from .config import Settings, deployment_revision from .errors import ConversionRejected, EngineUnavailable from .files import read_limited, sanitize_filename, split_name, title_from_stem -from .profiles import PROFILES, finalize_markdown +from .profiles import PROFILES, count_words, finalize_markdown from .security import require_internal_secret from .storage import StorageClient, StorageError @@ -100,7 +100,7 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool markdown, warning = converter.convert(data, ext, filename=original_name, title=title) if raw: content = markdown.strip() + "\n" - words = len(content.split()) + words = count_words(content) tokens = round(words * 1.33) else: content, words, tokens = finalize_markdown( @@ -171,12 +171,15 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool @app.get("/api/v1/health") def health() -> dict: - return { + body = { "status": "ok", "role": settings.role, "instance": settings.instance, "version": settings.version, } + if revision := deployment_revision(): + body["revision"] = revision + return body @app.get("/api/v1/ready") def ready() -> JSONResponse: diff --git a/backendN/app/common/config.py b/backendN/app/common/config.py index e9a6d42..554beb1 100644 --- a/backendN/app/common/config.py +++ b/backendN/app/common/config.py @@ -3,10 +3,20 @@ from __future__ import annotations import os +import re from dataclasses import dataclass VALID_ROLES = ("normal", "ocr", "archive") + +def deployment_revision() -> str | None: + """Allowlisted public build identity, never arbitrary environment values.""" + for name in ("RENDER_GIT_COMMIT", "VERCEL_GIT_COMMIT_SHA"): + value = os.environ.get(name, "") + if re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", value): + return value.lower() + return None + # Upload byte limits per role; MAX_UPLOAD_BYTES overrides. Keep in sync with # MAX_FILE_SIZE / MAX_IMAGE_FILE_SIZE in frontend/lib/formats.js. DEFAULT_MAX_UPLOAD_BYTES = { diff --git a/backendN/app/common/profiles.py b/backendN/app/common/profiles.py index bad62a8..db79753 100644 --- a/backendN/app/common/profiles.py +++ b/backendN/app/common/profiles.py @@ -7,6 +7,19 @@ PROFILES = ("Standard", "Clean", "Compact", "RAG-ready") _H1_BLOCK = re.compile(r"^(# .+\n\n)") +COUNT_CHUNK_CHARS = 64 * 1024 + + +def count_words(content: str) -> int: + """Exact str.split() count with bounded temporary strings/word arrays.""" + count = 0 + for start in range(0, len(content), COUNT_CHUNK_CHARS): + chunk = content[start:start + COUNT_CHUNK_CHARS] + count += len(chunk.split()) + if start and not content[start - 1].isspace() and not chunk[0].isspace(): + # A word spanning a chunk boundary was counted in both chunks. + count -= 1 + return count def ensure_title(markdown: str, title: str) -> str: @@ -48,5 +61,5 @@ def finalize_markdown( r"(^## [^\n]+)", r"\n\1", content, flags=re.M ) - words = len(content.split()) + words = count_words(content) return content, words, round(words * 1.33) diff --git a/backendN/app/converter.py b/backendN/app/converter.py index 00ca23b..ef56173 100644 --- a/backendN/app/converter.py +++ b/backendN/app/converter.py @@ -56,8 +56,10 @@ def __init__(self) -> None: return self._md = MarkItDown(enable_plugins=False) from app.csv_converter import register_csv_converter + from app.excel_converter import register_excel_converters register_csv_converter(self._md) + register_excel_converters(self._md) self._stream_info = StreamInfo self._import_error = "" diff --git a/backendN/app/excel_converter.py b/backendN/app/excel_converter.py new file mode 100644 index 0000000..fdffdd0 --- /dev/null +++ b/backendN/app/excel_converter.py @@ -0,0 +1,78 @@ +"""Same pandas/HTML Excel conversion, retaining one DataFrame at a time.""" +from __future__ import annotations + +from contextlib import closing +from importlib.metadata import version +import io + +from markitdown import DocumentConverterResult +from markitdown.converters import _xlsx_converter as upstream + +VERIFIED_VERSIONS = frozenset({"0.1.8", "0.1.9"}) + + +class _SheetViewRepair(TypeError): + """Only pandas load/parse failures may trigger the upstream XLSX repair.""" + + +def _read(call, *args, repair=False, **kwargs): + try: + return call(*args, **kwargs) + except TypeError as error: + if repair and "showZeroes" in str(error): + raise _SheetViewRepair(str(error)) from error + raise + + +def _convert_workbook(file_stream, engine, html_converter, options, *, repair=False): + # xlrd otherwise keeps every sheet's raw cells as well as pandas frames. + # Older BIFF versions may disable on-demand loading; retain their fallback. + engine_options = {"engine_kwargs": {"on_demand": True}} if engine == "xlrd" else {} + workbook = _read(upstream.pd.ExcelFile, file_stream, engine=engine, repair=repair, **engine_options) + with workbook, io.StringIO() as output: + for name in workbook.sheet_names: + frame = _read(workbook.parse, sheet_name=name, repair=repair) + if engine == "xlrd" and workbook.book.on_demand: + workbook.book.unload_sheet(name) + html = frame.to_html(index=False) + table = html_converter.convert_string(html, **options).markdown.strip() + output.write(f"## {name}\n") + output.write(table) + output.write("\n\n") + # Do not retain the previous sheet while parsing/rendering the next. + del frame, html, table + return DocumentConverterResult(markdown=output.getvalue().strip()) + + +class SheetwiseXlsxConverter(upstream.XlsxConverter): + def convert(self, file_stream, stream_info, **kwargs): + # Preserve upstream missing-dependency errors and custom image hooks. + if (upstream._xlsx_dependency_exc_info is not None + or type(self)._image_to_html is not upstream.XlsxConverter._image_to_html): + return super().convert(file_stream, stream_info, **kwargs) + start = file_stream.tell() + try: + return _convert_workbook(file_stream, "openpyxl", self._html_converter, kwargs, repair=True) + except _SheetViewRepair: + # Discard the partial text before retrying the entire repaired book. + with closing(upstream._repair_sheetview_show_zeroes(file_stream, start)) as repaired: + return _convert_workbook(repaired, "openpyxl", self._html_converter, kwargs) + + +class SheetwiseXlsConverter(upstream.XlsConverter): + def convert(self, file_stream, stream_info, **kwargs): + if upstream._xls_dependency_exc_info is not None: + return super().convert(file_stream, stream_info, **kwargs) + return _convert_workbook(file_stream, "xlrd", self._html_converter, kwargs) + + +def register_excel_converters(engine) -> bool: + """Unknown versions retain their built-in converters, including image hooks.""" + if version("markitdown") not in VERIFIED_VERSIONS: + return False + if (not callable(getattr(upstream, "_repair_sheetview_show_zeroes", None)) + or not hasattr(upstream.XlsxConverter, "_image_to_html")): + return False + engine.register_converter(SheetwiseXlsxConverter()) + engine.register_converter(SheetwiseXlsConverter()) + return True diff --git a/backendN/requirements-dev.txt b/backendN/requirements-dev.txt index 7c54944..1af4fb3 100644 --- a/backendN/requirements-dev.txt +++ b/backendN/requirements-dev.txt @@ -2,3 +2,5 @@ pytest>=8 httpx>=0.27 uvicorn>=0.30 +# Generates real legacy XLS fixtures; not a runtime dependency. +xlwt==1.3.0 diff --git a/backendN/tests/excel_fixtures.py b/backendN/tests/excel_fixtures.py new file mode 100644 index 0000000..7d09bf5 --- /dev/null +++ b/backendN/tests/excel_fixtures.py @@ -0,0 +1,85 @@ +"""Generated XLSX/XLS fixtures; no binary workbooks or private data in Git.""" +from datetime import datetime +import io +import zipfile + + +def make_dense_book(ext, sheets, rows): + """Benchmark fixture generation is separate from measured conversion.""" + output = io.BytesIO() + if ext == "xlsx": + from openpyxl import Workbook + book = Workbook(write_only=True) + for index in range(sheets): + sheet = book.create_sheet(f"Sheet-{index}") + sheet.append(["id", "name", "amount", "note"]) + for row in range(rows): + sheet.append([row, f"sheet-{index}-row-{row}", row * 3.5, f"generated café 日本語 {index}-{row}"]) + book.save(output) + book.close() + else: + import xlwt + book = xlwt.Workbook() + for index in range(sheets): + sheet = book.add_sheet(f"Sheet-{index}") + for column, value in enumerate(["id", "name", "amount", "note"]): + sheet.write(0, column, value) + for row in range(rows): + for column, value in enumerate([row, f"sheet-{index}-row-{row}", row * 3.5, f"generated café 日本語 {index}-{row}"]): + sheet.write(row + 1, column, value) + book.save(output) + return output.getvalue() + + +def make_book(ext="xlsx", *, legacy_view=False, image=False): + sheets = [ + ("Mixed 日本語", [["Name", "Name", "Amount", "Date", "Flag"], + ["café | pipe", "line\nbreak", 1.23456789, datetime(2024, 2, 3, 4, 5), True], + ["&", "", 1e-9, None, False], ["NaN", "NULL", -1000000000.5], + [None] * 5, ["last-first-sheet", 'quote"value', 123.0]]), + ("Empty", []), + ("Results", [["id", "value"], [1, "before blank"], [], [2, "終わり🙂"]]), + ] + output = io.BytesIO() + if ext == "xls": + import xlwt + book = xlwt.Workbook() + date_style = xlwt.easyxf(num_format_str="YYYY-MM-DD HH:MM:SS") + for name, rows in sheets: + sheet = book.add_sheet(name) + for r, row in enumerate(rows): + for c, value in enumerate(row): + if value is not None: + sheet.write(r, c, value, date_style if isinstance(value, datetime) else xlwt.Style.default_style) + book.save(output) + return output.getvalue() + from openpyxl import Workbook + book = Workbook() + book.remove(book.active) + for name, rows in sheets: + sheet = book.create_sheet(name) + for row in rows: + sheet.append(row) + book["Results"]["D2"] = "=SUM(A2:A4)" + book["Results"].merge_cells("F2:G2") + book["Results"]["F2"] = "merged value" + book["Empty"].sheet_state = "hidden" + if image: + from PIL import Image + from openpyxl.drawing.image import Image as WorkbookImage + pixels = io.BytesIO() + Image.new("RGB", (4, 4), "white").save(pixels, format="PNG") + pixels.seek(0) + book["Results"].add_image(WorkbookImage(pixels), "A8") + book.save(output) + book.close() + if not legacy_view: + return output.getvalue() + repaired_fixture = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(output.getvalue())) as source, zipfile.ZipFile(repaired_fixture, "w", zipfile.ZIP_DEFLATED) as target: + for item in source.infolist(): + data = source.read(item.filename) + if item.filename.startswith("xl/worksheets/"): + data = data.replace(b'embedded-pixel-marker

" + class OptimizedHook(module.SheetwiseXlsxConverter): + _image_to_html = OriginalHook._image_to_html + data = make_book(image=True) + info = StreamInfo(extension=".xlsx") + expected = OriginalHook().convert(io.BytesIO(data), info).markdown + actual = OptimizedHook().convert(io.BytesIO(data), info).markdown + assert actual == expected + assert "embedded-pixel-marker" in actual + + +@optimized +def test_sheet_frame_is_released_and_workbook_closed_before_next_parse(monkeypatch): + frames, closed = [], [] + class Frame: + def to_html(self, **kwargs): + return "" + class Workbook: + sheet_names = ["One", "Two", "Three"] + def __enter__(self): return self + def __exit__(self, *args): closed.append(True) + def parse(self, **kwargs): + assert all(reference() is None for reference in frames) + frame = Frame() + frames.append(weakref.ref(frame)) + return frame + monkeypatch.setattr(module.upstream.pd, "ExcelFile", lambda *args, **kwargs: Workbook()) + html = SimpleNamespace(convert_string=lambda *args, **kwargs: SimpleNamespace(markdown="table")) + result = module._convert_workbook(io.BytesIO(), "openpyxl", html, {}) + assert result.markdown == "## One\ntable\n\n## Two\ntable\n\n## Three\ntable" + assert closed == [True] + assert all(reference() is None for reference in frames) + + +@optimized +def test_late_sheet_repair_discards_partial_output_and_closes_repaired_stream(monkeypatch): + data = make_book() + info = StreamInfo(extension=".xlsx") + expected = XlsxConverter().convert(io.BytesIO(data), info).markdown + original_read = module._read + repair = module.upstream._repair_sheetview_show_zeroes + streams, failed = [], [] + def fail_late(call, *args, **kwargs): + if kwargs.get("sheet_name") == "Results" and not failed: + failed.append(True) + raise module._SheetViewRepair("showZeroes") + return original_read(call, *args, **kwargs) + def observe_repair(*args): + stream = repair(*args) + streams.append(stream) + return stream + monkeypatch.setattr(module, "_read", fail_late) + monkeypatch.setattr(module.upstream, "_repair_sheetview_show_zeroes", observe_repair) + actual = module.SheetwiseXlsxConverter().convert(io.BytesIO(data), info).markdown + assert actual == expected + assert len(streams) == 1 and streams[0].closed + + +@optimized +def test_html_failure_is_not_mistaken_for_a_workbook_repair(monkeypatch): + converter = module.SheetwiseXlsxConverter() + def fail(*args, **kwargs): raise TypeError("showZeroes is an unrelated HTML error") + monkeypatch.setattr(converter._html_converter, "convert_string", fail) + monkeypatch.setattr(module.upstream, "_repair_sheetview_show_zeroes", lambda *args: pytest.fail("HTML error triggered workbook repair")) + with pytest.raises(TypeError, match="unrelated HTML"): + converter.convert(io.BytesIO(make_book()), StreamInfo(extension=".xlsx")) + + +def test_unknown_version_keeps_original_excel_converters(monkeypatch): + monkeypatch.setattr(module, "version", lambda name: "0.1.10") + assert not module.register_excel_converters(SimpleNamespace(register_converter=lambda *args: pytest.fail("unknown adapter registered"))) + + +@pytest.mark.parametrize("ext", ["xlsx", "xls"]) +def test_unknown_version_normal_conversion_keeps_builtin_output(monkeypatch, ext): + from app.converter import NormalConverter + from markitdown import MarkItDown + monkeypatch.setattr(module, "version", lambda name: "0.1.10") + def unexpected(*args, **kwargs): pytest.fail("unverified adapter used") + monkeypatch.setattr(module.SheetwiseXlsxConverter, "convert", unexpected) + monkeypatch.setattr(module.SheetwiseXlsConverter, "convert", unexpected) + data = make_book(ext) + expected = MarkItDown(enable_plugins=False).convert_stream(io.BytesIO(data), stream_info=StreamInfo(extension=f".{ext}")).markdown.strip() + actual = NormalConverter().convert(data, ext, filename=f"book.{ext}", title="book")[0] + assert actual == expected + + +@optimized +def test_missing_upstream_repair_helper_retains_builtins(monkeypatch): + monkeypatch.setattr(module.upstream, "_repair_sheetview_show_zeroes", None) + assert not module.register_excel_converters(SimpleNamespace(register_converter=lambda *args: pytest.fail("incompatible adapter registered"))) + + +@optimized +def test_xls_unloads_each_sheet_and_releases_workbook_on_html_failure(monkeypatch): + original = module.upstream.pd.ExcelFile + books = [] + def opened(*args, **kwargs): + workbook = original(*args, **kwargs) + books.append(workbook.book) + assert workbook.book.on_demand + assert not any(workbook.book.sheet_loaded(name) for name in workbook.sheet_names) + return workbook + converter = module.SheetwiseXlsConverter() + def fail(*args, **kwargs): + assert not any(books[0].sheet_loaded(name) for name in books[0].sheet_names()) + raise ValueError("HTML failure") + monkeypatch.setattr(module.upstream.pd, "ExcelFile", opened) + monkeypatch.setattr(converter._html_converter, "convert_string", fail) + with pytest.raises(ValueError, match="HTML failure"): + converter.convert(io.BytesIO(make_book("xls")), StreamInfo(extension=".xls")) + assert books[0]._resources_released + + +def test_document_pools_keep_identical_excel_adapter(): + root = Path(__file__).resolve().parents[2] + assert (root / "backendN/app/excel_converter.py").read_bytes() == (root / "backendZ/app/excel_converter.py").read_bytes() + + +@optimized +@pytest.mark.parametrize("ext", ["xlsx", "xls"]) +def test_normal_route_uses_sheetwise_converter(convert, monkeypatch, ext): + cls = module.SheetwiseXlsxConverter if ext == "xlsx" else module.SheetwiseXlsConverter + calls, original = [], cls.convert + def observed(self, *args, **kwargs): + calls.append(True) + return original(self, *args, **kwargs) + monkeypatch.setattr(cls, "convert", observed) + response = convert(f"generated.{ext}", make_book(ext)) + assert response.status_code == 200, response.text + assert calls and "終わり🙂" in response.json()["content"] diff --git a/backendN/tests/test_result_counts.py b/backendN/tests/test_result_counts.py new file mode 100644 index 0000000..0534bde --- /dev/null +++ b/backendN/tests/test_result_counts.py @@ -0,0 +1,28 @@ +"""Final result statistics must retain Python's whitespace/token semantics.""" +import random + +import pytest +from app.common import profiles + + +@pytest.mark.parametrize("chunk_size", [1, 2, 7, 65536]) +def test_chunk_boundaries_match_native_split(monkeypatch, chunk_size): + monkeypatch.setattr(profiles, "COUNT_CHUNK_CHARS", chunk_size) + rng = random.Random(20261028) + alphabet = "abc 日本語🙂\r\n\t\v\f\x1c\x1d\x1e\x1f\x85\u00a0\u1680\u2000\u2028\u2029\u202f\u205f\u3000\ufeff" + for text in ["", " ", "x" * 100, "a b", "\r\n", *["".join(rng.choice(alphabet) for _ in range(200)) for _ in range(100)]]: + assert profiles.count_words(text) == len(text.split()) + + +def test_large_unicode_result_and_unbroken_line_keep_complete_counts(): + for text in [("Unicode 日本語 🙂 and café\n" * 100000), "word" * 300000, " " * 100000]: + assert profiles.count_words(text) == len(text.split()) + + +@pytest.mark.parametrize("profile", profiles.PROFILES) +def test_profile_content_words_and_tokens_match_original(monkeypatch, profile): + text = "# Report\n\n## Details\n\n" + ("word 日本語 🙂\x1c next\n" * 5000) + actual = profiles.finalize_markdown(text, title="Report", original_name="generated.xlsx", size_bytes=42, engine="MarkItDown", profile=profile) + monkeypatch.setattr(profiles, "count_words", lambda content: len(content.split())) + expected = profiles.finalize_markdown(text, title="Report", original_name="generated.xlsx", size_bytes=42, engine="MarkItDown", profile=profile) + assert actual == expected diff --git a/backendO/app/common/app_factory.py b/backendO/app/common/app_factory.py index f0be84f..08f92f8 100644 --- a/backendO/app/common/app_factory.py +++ b/backendO/app/common/app_factory.py @@ -24,10 +24,10 @@ from fastapi.responses import JSONResponse from pydantic import BaseModel, Field -from .config import Settings +from .config import Settings, deployment_revision from .errors import ConversionRejected, EngineUnavailable from .files import read_limited, sanitize_filename, split_name, title_from_stem -from .profiles import PROFILES, finalize_markdown +from .profiles import PROFILES, count_words, finalize_markdown from .security import require_internal_secret from .storage import StorageClient, StorageError @@ -100,7 +100,7 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool markdown, warning = converter.convert(data, ext, filename=original_name, title=title) if raw: content = markdown.strip() + "\n" - words = len(content.split()) + words = count_words(content) tokens = round(words * 1.33) else: content, words, tokens = finalize_markdown( @@ -171,12 +171,15 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool @app.get("/api/v1/health") def health() -> dict: - return { + body = { "status": "ok", "role": settings.role, "instance": settings.instance, "version": settings.version, } + if revision := deployment_revision(): + body["revision"] = revision + return body @app.get("/api/v1/ready") def ready() -> JSONResponse: diff --git a/backendO/app/common/config.py b/backendO/app/common/config.py index e9a6d42..554beb1 100644 --- a/backendO/app/common/config.py +++ b/backendO/app/common/config.py @@ -3,10 +3,20 @@ from __future__ import annotations import os +import re from dataclasses import dataclass VALID_ROLES = ("normal", "ocr", "archive") + +def deployment_revision() -> str | None: + """Allowlisted public build identity, never arbitrary environment values.""" + for name in ("RENDER_GIT_COMMIT", "VERCEL_GIT_COMMIT_SHA"): + value = os.environ.get(name, "") + if re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", value): + return value.lower() + return None + # Upload byte limits per role; MAX_UPLOAD_BYTES overrides. Keep in sync with # MAX_FILE_SIZE / MAX_IMAGE_FILE_SIZE in frontend/lib/formats.js. DEFAULT_MAX_UPLOAD_BYTES = { diff --git a/backendO/app/common/profiles.py b/backendO/app/common/profiles.py index bad62a8..db79753 100644 --- a/backendO/app/common/profiles.py +++ b/backendO/app/common/profiles.py @@ -7,6 +7,19 @@ PROFILES = ("Standard", "Clean", "Compact", "RAG-ready") _H1_BLOCK = re.compile(r"^(# .+\n\n)") +COUNT_CHUNK_CHARS = 64 * 1024 + + +def count_words(content: str) -> int: + """Exact str.split() count with bounded temporary strings/word arrays.""" + count = 0 + for start in range(0, len(content), COUNT_CHUNK_CHARS): + chunk = content[start:start + COUNT_CHUNK_CHARS] + count += len(chunk.split()) + if start and not content[start - 1].isspace() and not chunk[0].isspace(): + # A word spanning a chunk boundary was counted in both chunks. + count -= 1 + return count def ensure_title(markdown: str, title: str) -> str: @@ -48,5 +61,5 @@ def finalize_markdown( r"(^## [^\n]+)", r"\n\1", content, flags=re.M ) - words = len(content.split()) + words = count_words(content) return content, words, round(words * 1.33) diff --git a/backendZ/app/archive_tasks.py b/backendZ/app/archive_tasks.py index 37da254..73453ba 100644 --- a/backendZ/app/archive_tasks.py +++ b/backendZ/app/archive_tasks.py @@ -130,28 +130,34 @@ def write_outputs(store, *, job_id: str, name: str, profile: str, source_bytes: if len(content.encode("utf-8")) > archive.MAX_OUTPUT_BYTES: raise resource_limit() parts = archive.split_parts(content, title) + char_count = len(content) + del content out = _out_dir(job_id) - files: list[tuple[str, bytes, str, str]] = [] + files: list[tuple[str, str, str, str]] = [] if len(parts) == 1: - files.append((f"{out}combined.md", parts[0].encode("utf-8"), "text/markdown; charset=utf-8", f"{stem}.md")) + files.append((f"{out}combined.md", parts[0], "text/markdown; charset=utf-8", f"{stem}.md")) else: for i, part in enumerate(parts, start=1): - files.append((f"{out}combined-{i:03d}.md", part.encode("utf-8"), "text/markdown; charset=utf-8", f"{stem}-part-{i:03d}.md")) - files.append((f"{out}PROJECT_INDEX.md", archive.project_index(name, entries, ocr_texts).encode("utf-8"), + files.append((f"{out}combined-{i:03d}.md", part, "text/markdown; charset=utf-8", f"{stem}-part-{i:03d}.md")) + files.append((f"{out}PROJECT_INDEX.md", archive.project_index(name, entries, ocr_texts), "text/markdown; charset=utf-8", "PROJECT_INDEX.md")) - files.append((f"{out}manifest.json", archive.manifest(job_id, name, source_bytes, entries, ocr_texts).encode("utf-8"), + files.append((f"{out}manifest.json", archive.manifest(job_id, name, source_bytes, entries, ocr_texts), "application/json; charset=utf-8", "manifest.json")) # Preflight every final object and their total before the first write. This # includes index/manifest bytes and continuation headings, not just source text. - if (any(len(f[1]) > archive.PART_BYTES for f in files[:-2]) - or any(len(f[1]) > METADATA_BYTES for f in files[-2:]) - or sum(len(f[1]) for f in files) > archive.MAX_OUTPUT_BYTES): + # Measure one encoded object at a time; retain text and byte counts only. + sizes = [len(f[1].encode("utf-8")) for f in files] + if (any(size > archive.PART_BYTES for size in sizes[:-2]) + or any(size > METADATA_BYTES for size in sizes[-2:]) + or sum(sizes) > archive.MAX_OUTPUT_BYTES): raise resource_limit() try: with ThreadPoolExecutor(max_workers=WORKERS) as pool: - list(pool.map(lambda f: store.upload(f[0], f[1], f[2]), files)) + # Encoding inside the worker bounds live application byte buffers + # to WORKERS objects instead of retaining every encoded part. + list(pool.map(lambda f: store.upload(f[0], f[1].encode("utf-8"), f[2]), files)) except StorageError as err: raise HTTPException(status_code=503 if err.transient else 502, detail="Could not store the project result") from None @@ -161,15 +167,15 @@ def write_outputs(store, *, job_id: str, name: str, profile: str, source_bytes: return { "filename": f"{stem}.md", "original_name": name, - "char_count": len(content), + "char_count": char_count, "word_count": words, "tokens_est": tokens, "engine": archive.ENGINE, "backend_role": settings.role, "backend_instance": settings.instance, "warnings": warnings, - "output_bytes": sum(len(f[1]) for f in files), - "outputs": [{"path": f[0], "kind": "OUTPUT", "bytes": len(f[1]), "name": f[3]} for f in files], + "output_bytes": sum(sizes), + "outputs": [{"path": f[0], "kind": "OUTPUT", "bytes": size, "name": f[3]} for f, size in zip(files, sizes)], "primary_output": files[0][0], } diff --git a/backendZ/app/common/app_factory.py b/backendZ/app/common/app_factory.py index f0be84f..08f92f8 100644 --- a/backendZ/app/common/app_factory.py +++ b/backendZ/app/common/app_factory.py @@ -24,10 +24,10 @@ from fastapi.responses import JSONResponse from pydantic import BaseModel, Field -from .config import Settings +from .config import Settings, deployment_revision from .errors import ConversionRejected, EngineUnavailable from .files import read_limited, sanitize_filename, split_name, title_from_stem -from .profiles import PROFILES, finalize_markdown +from .profiles import PROFILES, count_words, finalize_markdown from .security import require_internal_secret from .storage import StorageClient, StorageError @@ -100,7 +100,7 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool markdown, warning = converter.convert(data, ext, filename=original_name, title=title) if raw: content = markdown.strip() + "\n" - words = len(content.split()) + words = count_words(content) tokens = round(words * 1.33) else: content, words, tokens = finalize_markdown( @@ -171,12 +171,15 @@ def run(original_name: str, profile: str, job_id: str, load_bytes, *, raw: bool @app.get("/api/v1/health") def health() -> dict: - return { + body = { "status": "ok", "role": settings.role, "instance": settings.instance, "version": settings.version, } + if revision := deployment_revision(): + body["revision"] = revision + return body @app.get("/api/v1/ready") def ready() -> JSONResponse: diff --git a/backendZ/app/common/config.py b/backendZ/app/common/config.py index e9a6d42..554beb1 100644 --- a/backendZ/app/common/config.py +++ b/backendZ/app/common/config.py @@ -3,10 +3,20 @@ from __future__ import annotations import os +import re from dataclasses import dataclass VALID_ROLES = ("normal", "ocr", "archive") + +def deployment_revision() -> str | None: + """Allowlisted public build identity, never arbitrary environment values.""" + for name in ("RENDER_GIT_COMMIT", "VERCEL_GIT_COMMIT_SHA"): + value = os.environ.get(name, "") + if re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", value): + return value.lower() + return None + # Upload byte limits per role; MAX_UPLOAD_BYTES overrides. Keep in sync with # MAX_FILE_SIZE / MAX_IMAGE_FILE_SIZE in frontend/lib/formats.js. DEFAULT_MAX_UPLOAD_BYTES = { diff --git a/backendZ/app/common/profiles.py b/backendZ/app/common/profiles.py index bad62a8..db79753 100644 --- a/backendZ/app/common/profiles.py +++ b/backendZ/app/common/profiles.py @@ -7,6 +7,19 @@ PROFILES = ("Standard", "Clean", "Compact", "RAG-ready") _H1_BLOCK = re.compile(r"^(# .+\n\n)") +COUNT_CHUNK_CHARS = 64 * 1024 + + +def count_words(content: str) -> int: + """Exact str.split() count with bounded temporary strings/word arrays.""" + count = 0 + for start in range(0, len(content), COUNT_CHUNK_CHARS): + chunk = content[start:start + COUNT_CHUNK_CHARS] + count += len(chunk.split()) + if start and not content[start - 1].isspace() and not chunk[0].isspace(): + # A word spanning a chunk boundary was counted in both chunks. + count -= 1 + return count def ensure_title(markdown: str, title: str) -> str: @@ -48,5 +61,5 @@ def finalize_markdown( r"(^## [^\n]+)", r"\n\1", content, flags=re.M ) - words = len(content.split()) + words = count_words(content) return content, words, round(words * 1.33) diff --git a/backendZ/app/converter.py b/backendZ/app/converter.py index a626308..088a7ef 100644 --- a/backendZ/app/converter.py +++ b/backendZ/app/converter.py @@ -39,8 +39,10 @@ def __init__(self) -> None: return self._md = MarkItDown(enable_plugins=False) from app.csv_converter import register_csv_converter + from app.excel_converter import register_excel_converters register_csv_converter(self._md) + register_excel_converters(self._md) self._stream_info = StreamInfo self.error = "" diff --git a/backendZ/app/excel_converter.py b/backendZ/app/excel_converter.py new file mode 100644 index 0000000..fdffdd0 --- /dev/null +++ b/backendZ/app/excel_converter.py @@ -0,0 +1,78 @@ +"""Same pandas/HTML Excel conversion, retaining one DataFrame at a time.""" +from __future__ import annotations + +from contextlib import closing +from importlib.metadata import version +import io + +from markitdown import DocumentConverterResult +from markitdown.converters import _xlsx_converter as upstream + +VERIFIED_VERSIONS = frozenset({"0.1.8", "0.1.9"}) + + +class _SheetViewRepair(TypeError): + """Only pandas load/parse failures may trigger the upstream XLSX repair.""" + + +def _read(call, *args, repair=False, **kwargs): + try: + return call(*args, **kwargs) + except TypeError as error: + if repair and "showZeroes" in str(error): + raise _SheetViewRepair(str(error)) from error + raise + + +def _convert_workbook(file_stream, engine, html_converter, options, *, repair=False): + # xlrd otherwise keeps every sheet's raw cells as well as pandas frames. + # Older BIFF versions may disable on-demand loading; retain their fallback. + engine_options = {"engine_kwargs": {"on_demand": True}} if engine == "xlrd" else {} + workbook = _read(upstream.pd.ExcelFile, file_stream, engine=engine, repair=repair, **engine_options) + with workbook, io.StringIO() as output: + for name in workbook.sheet_names: + frame = _read(workbook.parse, sheet_name=name, repair=repair) + if engine == "xlrd" and workbook.book.on_demand: + workbook.book.unload_sheet(name) + html = frame.to_html(index=False) + table = html_converter.convert_string(html, **options).markdown.strip() + output.write(f"## {name}\n") + output.write(table) + output.write("\n\n") + # Do not retain the previous sheet while parsing/rendering the next. + del frame, html, table + return DocumentConverterResult(markdown=output.getvalue().strip()) + + +class SheetwiseXlsxConverter(upstream.XlsxConverter): + def convert(self, file_stream, stream_info, **kwargs): + # Preserve upstream missing-dependency errors and custom image hooks. + if (upstream._xlsx_dependency_exc_info is not None + or type(self)._image_to_html is not upstream.XlsxConverter._image_to_html): + return super().convert(file_stream, stream_info, **kwargs) + start = file_stream.tell() + try: + return _convert_workbook(file_stream, "openpyxl", self._html_converter, kwargs, repair=True) + except _SheetViewRepair: + # Discard the partial text before retrying the entire repaired book. + with closing(upstream._repair_sheetview_show_zeroes(file_stream, start)) as repaired: + return _convert_workbook(repaired, "openpyxl", self._html_converter, kwargs) + + +class SheetwiseXlsConverter(upstream.XlsConverter): + def convert(self, file_stream, stream_info, **kwargs): + if upstream._xls_dependency_exc_info is not None: + return super().convert(file_stream, stream_info, **kwargs) + return _convert_workbook(file_stream, "xlrd", self._html_converter, kwargs) + + +def register_excel_converters(engine) -> bool: + """Unknown versions retain their built-in converters, including image hooks.""" + if version("markitdown") not in VERIFIED_VERSIONS: + return False + if (not callable(getattr(upstream, "_repair_sheetview_show_zeroes", None)) + or not hasattr(upstream.XlsxConverter, "_image_to_html")): + return False + engine.register_converter(SheetwiseXlsxConverter()) + engine.register_converter(SheetwiseXlsConverter()) + return True diff --git a/backendZ/requirements-dev.txt b/backendZ/requirements-dev.txt index 7c54944..1af4fb3 100644 --- a/backendZ/requirements-dev.txt +++ b/backendZ/requirements-dev.txt @@ -2,3 +2,5 @@ pytest>=8 httpx>=0.27 uvicorn>=0.30 +# Generates real legacy XLS fixtures; not a runtime dependency. +xlwt==1.3.0 diff --git a/backendZ/tests/test_document_containers.py b/backendZ/tests/test_document_containers.py index 83ac7ec..ba20147 100644 --- a/backendZ/tests/test_document_containers.py +++ b/backendZ/tests/test_document_containers.py @@ -111,3 +111,19 @@ def observed(self, *args, **kwargs): assert calls assert entries[0].status == "DONE" assert entries[0].content == upstream + + +@pytest.mark.parametrize("ext", ["xlsx", "xls"]) +def test_archive_excel_keeps_complete_upstream_workbook(converter, ext): + import io + from markitdown import MarkItDown, StreamInfo + fixture_path = Path(__file__).resolve().parents[2] / "backendN/tests/excel_fixtures.py" + spec = importlib.util.spec_from_file_location("excel_fixtures", fixture_path) + fixtures = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fixtures) + data = fixtures.make_book(ext) + expected = MarkItDown(enable_plugins=False).convert_stream(io.BytesIO(data), stream_info=StreamInfo(extension=f".{ext}")).markdown.strip() + entries = archive.process_archive(make_zip({f"book.{ext}": data}), converter.documents, ocr_available=False) + assert entries[0].status == "DONE" + assert entries[0].content == expected + assert "終わり🙂" in entries[0].content diff --git a/backendZ/tests/test_output_buffer_capacity.py b/backendZ/tests/test_output_buffer_capacity.py new file mode 100644 index 0000000..0f3b006 --- /dev/null +++ b/backendZ/tests/test_output_buffer_capacity.py @@ -0,0 +1,45 @@ +"""Bound encoded output buffers while preserving preflight and full bytes.""" +from types import SimpleNamespace +import threading +import time + +from app import archive, archive_tasks + + +def test_final_upload_encoding_is_bounded_by_worker_capacity(monkeypatch): + counts = {"live": 0, "peak": 0} + lock = threading.Lock() + class Payload(bytes): + def __new__(cls, value): + instance = super().__new__(cls, value) + with lock: + counts["live"] += 1 + counts["peak"] = max(counts["peak"], counts["live"]) + return instance + def __del__(self): + with lock: + counts["live"] -= 1 + class Text(str): + def encode(self, *args, **kwargs): + return Payload(super().encode(*args, **kwargs)) + parts = [Text(f"part-{i}-日本語🙂") for i in range(10)] + monkeypatch.setattr(archive, "split_parts", lambda *args: parts) + monkeypatch.setattr(archive, "project_index", lambda *args: Text("INDEX-日本語")) + monkeypatch.setattr(archive, "manifest", lambda *args: Text('{"value":"🙂"}')) + written = {} + class Store: + def upload(self, path, data, content_type): + assert isinstance(data, bytes) + assert counts["live"] <= archive_tasks.WORKERS + time.sleep(.01) + with lock: + written[path] = (data.decode(), len(data)) + result = archive_tasks.write_outputs(Store(), job_id="job", name="project.zip", profile="Standard", source_bytes=42, entries=[], ocr_texts={}, settings=SimpleNamespace(role="archive", instance="test")) + assert len(written) == 12 + assert counts["live"] == 0 + assert counts["peak"] <= archive_tasks.WORKERS + assert result["output_bytes"] == sum(size for text, size in written.values()) + for record in result["outputs"]: + assert record["bytes"] == written[record["path"]][1] + for index, part in enumerate(parts, start=1): + assert written[f"jobs/job/output/combined-{index:03d}.md"][0] == part diff --git a/docs/LLMISER_EXCEL_ASSEMBLY_MEMORY.md b/docs/LLMISER_EXCEL_ASSEMBLY_MEMORY.md new file mode 100644 index 0000000..76e0d96 --- /dev/null +++ b/docs/LLMISER_EXCEL_ASSEMBLY_MEMORY.md @@ -0,0 +1,63 @@ +# Excel and final-result memory + +Updated 29 October 2026. Candidate implementation on `codex/excel-assembly-memory`; production release is not yet claimed. This follows the released [CSV capacity work](LLMISER_PARSER_CAPACITY.md). + +## Changes and fidelity + +Normal and archive workers register identical `app/excel_converter.py` adapters through MarkItDown's converter API. On verified MarkItDown 0.1.8/0.1.9, they open one pandas `ExcelFile`, parse a sheet, render it through the existing pandas HTML and MarkItDown HTML converter, then release that sheet's DataFrame/HTML before parsing the next. Full per-sheet inference and formatting remain: row-level rendering could change float/type formatting. XLS additionally requests xlrd on-demand loading and unloads each parsed raw sheet; old BIFF versions that disable on-demand loading retain xlrd's fallback. + +The adapters preserve sheet order, empty/hidden sheets, duplicate headers, dates, mixed types, formulas' existing cached-value behavior, Unicode and complete trailing content. XLSX's legacy `showZeroes` repair retries the whole workbook after discarding partial output, closes the repaired stream and does not misclassify HTML failures as workbook failures. A custom XLSX image hook retains the upstream path. Unknown MarkItDown versions or an unavailable repair helper retain built-in converters; do not expand the allowlist without equivalence tests. + +References: [MarkItDown 0.1.9 Excel implementation](https://github.com/microsoft/markitdown/blob/v0.1.9/packages/markitdown/src/markitdown/converters/_xlsx_converter.py), [pandas ExcelFile](https://pandas.pydata.org/docs/reference/api/pandas.ExcelFile.html), [xlrd on-demand loading](https://xlrd.readthedocs.io/en/latest/on_demand.html). + +Shared profile finalization and raw-piece counts now count words in 64 Ki-character slices. Boundary correction preserves Python `str.split()` semantics, including Unicode/control whitespace and words spanning slices. Word/token statistics and all four profile strings stay identical. This avoids a whole-result list of temporary word strings; it does not truncate the result. The common package remains byte-identical across N/O/Z, with no MarkItDown dependency added to OCR. + +Archive final writes keep complete text parts and their UTF-8 byte counts. Preflight still measures every part, continuation heading, index and manifest before the first upload. Encoding occurs inside the existing four upload workers, instead of keeping every encoded output simultaneously. The combined string can be released once parts exist. Paths, order, metadata, warnings, byte limits, Storage errors and full exports retain their contract. Individual outputs and all parsed entries are still materialized; this is not streamed end-to-end processing. + +Backend `/api/v1/health` may additionally report `revision` from a valid provider `RENDER_GIT_COMMIT` or `VERCEL_GIT_COMMIT_SHA`. Only a 40/64-character hexadecimal commit identifier is accepted; no arbitrary environment value is exposed. This supports checking Render as well as Vercel rollout of the shared counting code. Existing health fields remain. + +## Reproduction and interpretation + +Install the existing normal/archive development requirements, including `xlwt` solely for generated legacy XLS fixtures. There is no added runtime dependency. Use a verified MarkItDown version and run from the repo root, one process at a time: + +```sh +python tests/load/excel_assembly_bench.py --case excel --mode upstream --ext xlsx --sheets 6 --rows 2500 +python tests/load/excel_assembly_bench.py --case excel --mode current --ext xlsx --sheets 6 --rows 2500 +python tests/load/excel_assembly_bench.py --case excel --mode current --ext xls --sheets 6 --rows 2500 +python tests/load/excel_assembly_bench.py --case assembly --mode current --characters 6000000 +python tests/load/excel_assembly_bench.py --case counts --mode current --characters 12000000 +``` + +Repeat every case in both modes, three fresh processes per mode, alternating order. `upstream` Excel re-registers the built-in converter; assembly/counts load source from baseline `3f0501ab01302abd0f8402a91741de63d2c256d7`. Use `--heap` separately for allocation tracing; its overhead makes traced timing unsuitable for throughput claims. Workbook generation runs outside the measured child. The script clears Storage settings/disables dotenv before imports, uses generated data only, and makes no server/Storage requests. Assembly uses a discard uploader that hashes complete bytes. + +Process peaks are lifetime high-water readings including imports; pre-run readings are recorded too. Python heap tracing starts after fixture/engine setup. Compare complete output byte counts/hashes, word/token statistics and uploaded bytes before accepting memory differences. Local Windows/Python 3.14 results do not establish Linux/Python 3.12 hosting capacity, billing savings, platform ceilings or safe concurrency. + +## Measured results + +Windows/Python 3.14.6, MarkItDown 0.1.9, pandas 3.0.5, openpyxl 3.1.5 and xlrd 2.0.2. Median of three fresh processes per mode, alternating order; 42 measured processes on 28–29 October. The operator paused/resumed the matrix between completed trials. No build/regression workload ran alongside measurements. Every fixture's complete output hash/byte count matched between modes; result-count statistics and all archive object hashes matched too. + +| Fixture | Tracing | Upstream / current seconds | Upstream / current process peak MiB | Upstream / current traced heap MiB | +| --- | --- | ---: | ---: | ---: | +| XLSX, 1 sheet × 100 rows | Off | 0.239 / 0.126 | 143.84 / 143.76 | — | +| XLSX, 6 sheets × 2,500 rows | Off | 32.462 / 22.965 | 201.07 / 197.57 | — | +| XLS, 6 sheets × 2,500 rows | Off | 13.654 / 17.545 | 202.59 / 191.91 | — | +| ZIP assembly, four 6M-character entries | Off | 2.325 / 1.583 | 624.70 / 530.28 | — | +| Profile finalization, 12M characters | Off | 0.245 / 0.145 | 278.23 / 231.70 | — | +| Same ZIP assembly | On | 7.945 / 5.807 | 958.38 / 530.26 | 424.90 / 366.22 | +| Same profile finalization | On | 4.440 / 2.431 | 446.92 / 231.65 | 166.35 / 137.33 | + +Untraced peaks fell about 2% for XLSX, 5% for XLS, 15% for ZIP assembly and 17% for profile finalization. The small workbook's import-dominated peak was essentially unchanged. XLS elapsed median regressed; multi-sheet XLSX ranged 11.282–35.929 seconds upstream and 20.422–28.168 current, while XLS ranged 7.757–28.077 and 6.955–45.787. This variability precludes a general speedup claim. Tracing notably increases upstream process memory, so do not substitute traced process differences for normal runtime savings. + +Both multi-sheet formats produced 1,058,518 identical Markdown bytes, SHA-256 `73da8bc46453187aabb0fd994c148ebd15a9166731cf6b1755570a04e177ffaa`. Assembly wrote 29,457,096 total object bytes through the discard uploader in both modes. The profile fixture retained 3,000,009 words / 3,990,012 estimated tokens. ZIP measurements include both word-count and output-buffer changes, rather than isolating encoding alone. Raw JSON/logs stay ignored in `project-docs/excel-audit/`. + +## Validation and rollout + +Local MarkItDown 0.1.9 suites: 144 normal tests and 101 archive tests passed, with the existing FastAPI/Starlette test-client deprecation warning. Tests cover upstream output equality, normal/ZIP routing, release of prior frames/raw sheets, exception cleanup, late workbook repair, image hooks, actual unknown-version conversion fallback, Unicode word boundaries, unchanged profiles, upload buffer bounds and all existing preflight limits. + +MarkItDown 0.1.8 passed all 24 Excel/count checks. With the original local 0.1.7 installation, 13 fallback/count/integrity checks passed and 11 deliberately disabled adapter cases skipped. Small generated authenticated multipart probes passed before release: three workbooks on each normal worker and a ZIP containing all three on each archive worker (eight requests total). All four reported MarkItDown 0.1.9; complete expected tables and word counts matched. These direct probes created no Storage jobs. + +Local OCR validation initially lacked `pytesseract`; a no-dependency copy of 0.3.13 was installed only in the ignored audit folder. The full Windows suite then recorded 69 passes, eight orientation skips and one readiness failure claiming the configured binary was unavailable. Isolated readiness and a real text-image conversion both passed with the installed Tesseract 5.4.0. The suite failure remains recorded rather than hidden by weakening assertions. Linux/Python 3.12 OCR CI is required before merge; production Render uses Tesseract 5.5.0. + +Before release, the actual public frontend transport also passed generated XLSX and three-workbook ZIP conversions through create/upload/start/advance, Storage downloads and fresh-link retrieval. The single document output and all three ZIP objects downloaded successfully; table bytes, declared object lengths, final Unicode markers and exact recovered Markdown passed. This created two real jobs under normal retention. It validates the preceding production behavior; repeat after deployment to validate the candidate's Storage path. + +CI, deployment and post-release checks are still pending. Do not treat local checks as production release evidence. No blogs/visuals, database schema, retention policy, queue/concurrency settings, supported formats or limits change in this slice. Native parser peaks, retained archive entries, whole-result normalization/serialization, concurrent requests and real Storage/network buffers remain separate boundaries. diff --git a/docs/LLMISER_OPTIMIZATION_PLAN.md b/docs/LLMISER_OPTIMIZATION_PLAN.md index a5cc9bf..8590445 100644 --- a/docs/LLMISER_OPTIMIZATION_PLAN.md +++ b/docs/LLMISER_OPTIMIZATION_PLAN.md @@ -14,6 +14,8 @@ PR #13 released [a source capacity audit and CSV allocation reduction](LLMISER_P ## Verified starting point +Current implementation: [Excel and final-result assembly memory](LLMISER_EXCEL_ASSEMBLY_MEMORY.md), branch `codex/excel-assembly-memory`. Sequential sheet processing, exact bounded word counts and delayed archive output encoding are under measurement/validation. Keep production status separate until the release checks finish. + The copied source already contains atomic job startup/finalization, cleanup recovery, upload/active-job/storage-reservation admission, shared archive container checks, bounded PDF page uploads/OCR preparation, lazy Markdown/highlighter loading and current-edit export snapshots. Do not reimplement these based on old “proposed” checklist rows. Their deployment/capacity evidence is distinct from source tests. Search migration and public redirects are live; see [search migration](LLMISER_SEARCH_MIGRATION.md). Analytics ingestion is verified; controller login/configuration is verified but authenticated operations are not; see [access and analytics](LLMISER_ANALYTICS_CONTROLLER.md). Keep-warm Vault and cron changes were applied and verified in the earlier cron task. Dependency-aware service retirement remains separate operational work. diff --git a/docs/LLMISER_PARSER_CAPACITY.md b/docs/LLMISER_PARSER_CAPACITY.md index c4757dc..5c62767 100644 --- a/docs/LLMISER_PARSER_CAPACITY.md +++ b/docs/LLMISER_PARSER_CAPACITY.md @@ -72,4 +72,4 @@ All 20 public-page/discovery/health and permanent-redirect checks passed. The fi Local logs, package copies, raw benchmark JSON and operator probes remain ignored under `project-docs/parser-audit/`; no credentials are included in public evidence. Later documentation-only commits do not change the verified parser runtime revision. -Next: profile realistic workbook/native parser memory and full response/merge costs on representative hosts, then evaluate admission/deadlines before any concurrency increase. The CSV reduction alone does not justify raising queue or worker limits. +Follow-up: [Excel and final-result memory](LLMISER_EXCEL_ASSEMBLY_MEMORY.md) tracks the subsequent candidate, measurements and release state. This CSV report remains the evidence for PR #13. Representative-host native/concurrent parser peaks remain separate; neither reduction alone justifies raising queue or worker limits. diff --git a/docs/START_HERE.md b/docs/START_HERE.md index 4dcc696..d2ec417 100644 --- a/docs/START_HERE.md +++ b/docs/START_HERE.md @@ -12,6 +12,8 @@ Released [large-preview control](LLMISER_LARGE_PREVIEWS.md) pauses automatic for Released [parser capacity and CSV memory reduction](LLMISER_PARSER_CAPACITY.md) preserves verified upstream CSV output while reducing retained row allocations in both document pools. The audit distinguishes source limits, local generated measurements and live multipart acceptance. No concurrency increase follows from the benchmark. Workbook/native parser and final response/merge memory remain the next backend investigation. +Current candidate: [Excel and final-result memory](LLMISER_EXCEL_ASSEMBLY_MEMORY.md) is being measured on `codex/excel-assembly-memory`. It processes workbook sheets sequentially, bounds temporary word-count allocations and defers ZIP output encoding to the existing upload workers. The runtime revision above remains the last verified release until CI/deployment and live checks are recorded. + ## Legacy frontend replacement is deployed The owner wants the old converter removed from its frontend hosting, permanent public redirects preserved, and a small LLMiser-only banner as fallback. Replacement source is [legacy-site/](../legacy-site/README.md), inside this repository. It has no converter, API or database client. The working LLMiser project continues using `frontend`; only the old frontend project may use `legacy-site`. diff --git a/docs/ai-handover/HANDOVER.md b/docs/ai-handover/HANDOVER.md index d91d594..21e6017 100644 --- a/docs/ai-handover/HANDOVER.md +++ b/docs/ai-handover/HANDOVER.md @@ -54,6 +54,8 @@ Active local tooling now has zero absolute references to either old checkout. LL ## Verification commands +Active optimization: [Excel and final-result memory](../LLMISER_EXCEL_ASSEMBLY_MEMORY.md), branch `codex/excel-assembly-memory`. Read that report before changing Excel adapters, word counting or archive writes. Current candidate validation is distinct from the deployed CSV release. Preserve the verified-version fallback, upstream per-sheet formatting, complete results and preflight-before-write behavior; no concurrency increase is authorized by local memory measurements. + The owner asked to continue the optimization plan after the retirement handover. PR #8 completed that handover; PR #9 released [shared reader statistics](../LLMISER_READER_STATISTICS.md). PR #11 released [large-preview control and reproducible browser profiling](../LLMISER_LARGE_PREVIEWS.md): above 50,000 UTF-16 characters, formatting waits for an explicit request. Editing/document switching resets the request; complete raw text/copy/exports remain available. All six packs contain 631 messages. Local evidence: 261 full tests, final 20 focused tests, lint/consistency checks and production build passed; generated browser fixtures verify full rendering, edit re-pause, identical-text document switching, manual pause and exact current-edit download. CI passed all seven jobs, production deployed revision 0501cd0, the 20-check HTTP audit passed, and normal/OCR/archive were each 2/2 ready. No fresh production conversion/admin/cleanup acceptance is implied by that audit. PR #13 released [a parser-capacity audit and bounded CSV memory reduction](../LLMISER_PARSER_CAPACITY.md) at 165d2b0. N/Z use identical `app/csv_converter.py` modules, outside the three-pool shared `common/` package so OCR gains no MarkItDown dependency. Enable only on verified upstream 0.1.8/0.1.9; unknown versions/missing helper retain the original converter. Never expand the version allowlist without output-equivalence tests. Generated before/after hashes match, including synchronized conversions; queue limits and all other parsers stay unchanged. Local suites passed 118 normal/98 archive tests, 25 tests on 0.1.8, and the expected 0.1.7 fallback checks. CI/deployment and post-release generated multipart probes passed. These direct worker probes created no Storage jobs and do not replace browser/Storage-backed acceptance. diff --git a/docs/ai-handover/PROGRESS.md b/docs/ai-handover/PROGRESS.md index f7ee313..ba47bef 100644 --- a/docs/ai-handover/PROGRESS.md +++ b/docs/ai-handover/PROGRESS.md @@ -49,6 +49,8 @@ See [release record](../LLMISER_RELEASE_2026-10-26.md), [localization](../LLMISE ## Remaining work +In progress: [Excel conversion and final-result assembly memory](../LLMISER_EXCEL_ASSEMBLY_MEMORY.md), explicitly selected by the owner. Candidate code/tests are on `codex/excel-assembly-memory`; repeated measurements, compatibility checks and release evidence are pending. The shipped-state table above still describes the preceding release. + The owner resumed the implementation plan after the handover was merged in PR #8. [Shared reader statistics](../LLMISER_READER_STATISTICS.md) shipped in PR #9. [Large-preview control](../LLMISER_LARGE_PREVIEWS.md) shipped in PR #11 at 0501cd0 after 261 frontend CI tests/all seven jobs passed. All six production deployments succeeded; the 28 October 20-check HTTP audit passed and all three pools were 2/2 ready. Actual-component generated browser measurements and exact raw/download acceptance support the preview boundary. Representative-device/native textarea profiling and deeper parser-capacity work remain separate. [Parser audit/CSV evidence](../LLMISER_PARSER_CAPACITY.md) shipped in PR #13 at 165d2b0. The synchronized large untraced fixture used about 25% less peak process memory, with identical output; the small timing sample regressed, so no universal speedup is claimed. Local normal/archive suites and compatibility checks passed, all seven CI jobs and six production deployments succeeded, and pre/post-release multipart CSV/ZIP probes passed on all four document workers. All 20 public HTTP checks passed. OCR initially returned 0/2; direct readiness and a later uncached public wake confirmed all pools 2/2, without an OCR configuration change. No worker/queue tuning or Storage-backed conversion acceptance is implied. diff --git a/tests/load/excel_assembly_bench.py b/tests/load/excel_assembly_bench.py new file mode 100644 index 0000000..2cbd631 --- /dev/null +++ b/tests/load/excel_assembly_bench.py @@ -0,0 +1,150 @@ +"""Fresh-process Excel/result assembly comparison with generated data only. + +python tests/load/excel_assembly_bench.py --case excel --mode current --sheets 6 --rows 5000 +python tests/load/excel_assembly_bench.py --case assembly --mode current --characters 6000000 + +Repeat with --mode upstream and optionally --heap. No server/Storage calls. +""" +from __future__ import annotations + +import argparse +import gc +import hashlib +import importlib.metadata +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import tracemalloc +import types + +os.environ["PYTHON_DOTENV_DISABLED"] = "1" +os.environ["SUPABASE_URL"] = "" +os.environ["SUPABASE_SERVICE_ROLE_KEY"] = "" +os.environ["INTERNAL_SHARED_SECRET"] = "offline-benchmark-only" +ROOT = Path(__file__).resolve().parents[2] +BASELINE = "3f0501ab01302abd0f8402a91741de63d2c256d7" + + +def peak_rss_mib(): + if sys.platform != "win32": + import resource + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024**2 if sys.platform == "darwin" else 1024) + import ctypes + from ctypes import wintypes + class Counters(ctypes.Structure): + _fields_ = [(name, wintypes.DWORD) for name in ["cb", "PageFaultCount"]] + [(name, ctypes.c_size_t) for name in ["PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage", "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", "QuotaNonPagedPoolUsage", "PagefileUsage", "PeakPagefileUsage"]] + counters = Counters() + counters.cb = ctypes.sizeof(counters) + probe = ctypes.windll.psapi.GetProcessMemoryInfo + probe.argtypes = [wintypes.HANDLE, ctypes.POINTER(Counters), wintypes.DWORD] + if not probe(wintypes.HANDLE(-1), ctypes.byref(counters), counters.cb): + raise OSError("Could not read process memory") + return counters.PeakWorkingSetSize / 1024**2 + + +def old_module(name, relative): + source = subprocess.check_output(["git", "-C", str(ROOT), "show", f"{BASELINE}:{relative}"], text=True, encoding="utf-8") + module = types.ModuleType(name) + sys.modules[name] = module + exec(compile(source, relative, "exec"), module.__dict__) + return module + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", choices=["excel", "assembly", "counts"], required=True) + parser.add_argument("--mode", choices=["upstream", "current"], required=True) + parser.add_argument("--ext", choices=["xlsx", "xls"], default="xlsx") + parser.add_argument("--sheets", type=int, default=6) + parser.add_argument("--rows", type=int, default=5000) + parser.add_argument("--characters", type=int, default=6000000) + parser.add_argument("--heap", action="store_true") + parser.add_argument("--fixture", help=argparse.SUPPRESS) + args = parser.parse_args() + if not 1 <= args.sheets <= 12 or not 1 <= args.rows <= 30000 or not 1 <= args.characters <= 16000000: + parser.error("fixture parameters exceed the bounded offline corpus") + if args.case == "assembly" and args.characters > 6000000: + parser.error("four generated Unicode entries must fit the existing 40 MiB output cap") + if args.case == "excel" and not args.fixture: + # Generate outside the measured child so writer memory does not mask + # parser peaks. Generated workbooks stay in the ignored corpus folder. + sys.path.insert(0, str(ROOT / "backendN/tests")) + from excel_fixtures import make_dense_book + directory = ROOT / "tests/load/.corpus" + directory.mkdir(parents=True, exist_ok=True) + fixture = directory / f"excel-memory-{args.ext}-{args.sheets}-{args.rows}.{args.ext}" + data = make_dense_book(args.ext, args.sheets, args.rows) + if len(data) > 15 * 1024 * 1024: + raise ValueError("workbook exceeds deployed input limit") + fixture.write_bytes(data) + subprocess.run([sys.executable, __file__, *sys.argv[1:], "--fixture", str(fixture)], check=True) + return + + uploaded = {} + if args.case == "excel": + sys.path.insert(0, str(ROOT / "backendN")) + from app.converter import NormalConverter + from app.excel_converter import VERIFIED_VERSIONS + from markitdown.converters import XlsxConverter, XlsConverter + if importlib.metadata.version("markitdown") not in VERIFIED_VERSIONS: + parser.error("use a verified MarkItDown version for comparison") + data = Path(args.fixture).read_bytes() + engine = NormalConverter() + if args.mode == "upstream": + engine._md.register_converter(XlsxConverter() if args.ext == "xlsx" else XlsConverter()) + engine.validate(data, args.ext) + def run(): + output = engine.convert(data, args.ext, filename=f"generated.{args.ext}", title="generated")[0] + assert all(f"sheet-{sheet}-row-{args.rows-1}" in output for sheet in range(args.sheets)) + return output + else: + sys.path.insert(0, str(ROOT / "backendZ")) + from app import archive, archive_tasks + from app.common import profiles + if args.mode == "upstream": + profiles = old_module("bench_old_profiles", "backendZ/app/common/profiles.py") + archive_tasks = old_module("bench_old_archive_tasks", "backendZ/app/archive_tasks.py") + archive_tasks.finalize_markdown = profiles.finalize_markdown + chunk = "| Generated café 日本語 🙂 | 315 | word value |\n" + text = (chunk * (args.characters // len(chunk) + 1))[:args.characters] + if args.case == "counts": + def run(): + return profiles.finalize_markdown(text, title="generated", original_name="generated.xlsx", size_bytes=42, engine="MarkItDown") + else: + from app.common.config import Settings + settings = Settings(role="archive", instance="offline", shared_secret="test-only", max_upload_bytes=15*1024*1024, version="bench") + entries = [archive.Entry(path=f"sheet-{i}.xlsx", node_type="DOCUMENT", classification="DOCUMENT", status="DONE", content=f"# Sheet {i}\n\n{text}\nEND-{i}-日本語🙂", node_id=f"00000000-0000-4000-8000-{i:012d}") for i in range(4)] + lock = threading.Lock() + class Store: + def upload(self, path, data, content_type): + time.sleep(.02) + with lock: + uploaded[path] = (len(data), hashlib.sha256(data).hexdigest()) + def run(): + result = archive_tasks.write_outputs(Store(), job_id="00000000-0000-4000-8000-000000000001", name="generated.zip", profile="Standard", source_bytes=42, entries=entries, ocr_texts={}, settings=settings) + assert result["output_bytes"] == sum(size for size, digest in uploaded.values()) + return json.dumps({"result": result, "uploads": uploaded}, sort_keys=True) + gc.collect() + before = peak_rss_mib() + if args.heap: + tracemalloc.start() + started = time.perf_counter() + output = run() + seconds = time.perf_counter() - started + peak = peak_rss_mib() + heap = tracemalloc.get_traced_memory()[1] / 1024**2 if args.heap else None + if args.heap: + tracemalloc.stop() + words = tokens = None + if isinstance(output, tuple): + output, words, tokens = output + print(json.dumps({"case": args.case, "mode": args.mode, "python": sys.version.split()[0], "markitdown": importlib.metadata.version("markitdown"), "pandas": importlib.metadata.version("pandas"), "ext": args.ext, "sheets": args.sheets, "rows": args.rows, "characters_per_entry": args.characters, "seconds": round(seconds, 4), "tracing": args.heap, "process_peak_before_MiB": round(before, 2), "process_peak_after_MiB": round(peak, 2), "peak_python_heap_MiB": round(heap, 2) if heap is not None else None, "output_bytes": len(output.encode()), "output_sha256": hashlib.sha256(output.encode()).hexdigest(), "word_count": words, "tokens_est": tokens, "uploaded_bytes": sum(size for size, digest in uploaded.values()), "host_capacity_claim": False})) + + +if __name__ == "__main__": + main()