From dc9730aae4af64079e36c31d47957b93b028b1ee Mon Sep 17 00:00:00 2001 From: DevBehindYou <154013364+DevAtomicRelease@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:48:45 +0530 Subject: [PATCH] Reduce CSV parser allocations without changing output --- backendN/app/converter.py | 3 + backendN/app/csv_converter.py | 90 ++++++++++++++++ backendN/tests/test_csv_capacity.py | 116 +++++++++++++++++++++ backendZ/app/converter.py | 3 + backendZ/app/csv_converter.py | 90 ++++++++++++++++ backendZ/tests/test_document_containers.py | 21 ++++ docs/LLMISER_OPTIMIZATION_PLAN.md | 4 +- docs/LLMISER_PARSER_CAPACITY.md | 71 +++++++++++++ docs/START_HERE.md | 2 + docs/ai-handover/HANDOVER.md | 4 +- docs/ai-handover/PROGRESS.md | 2 + tests/load/csv_memory_bench.py | 103 ++++++++++++++++++ 12 files changed, 507 insertions(+), 2 deletions(-) create mode 100644 backendN/app/csv_converter.py create mode 100644 backendN/tests/test_csv_capacity.py create mode 100644 backendZ/app/csv_converter.py create mode 100644 docs/LLMISER_PARSER_CAPACITY.md create mode 100644 tests/load/csv_memory_bench.py diff --git a/backendN/app/converter.py b/backendN/app/converter.py index e5302ca..00ca23b 100644 --- a/backendN/app/converter.py +++ b/backendN/app/converter.py @@ -55,6 +55,9 @@ def __init__(self) -> None: self._import_error = str(err) return self._md = MarkItDown(enable_plugins=False) + from app.csv_converter import register_csv_converter + + register_csv_converter(self._md) self._stream_info = StreamInfo self._import_error = "" diff --git a/backendN/app/csv_converter.py b/backendN/app/csv_converter.py new file mode 100644 index 0000000..07e4eeb --- /dev/null +++ b/backendN/app/csv_converter.py @@ -0,0 +1,90 @@ +"""Row-at-a-time CSV formatting for verified MarkItDown versions. + +Keep decoding, cell escaping and table semantics identical to upstream. +Two CSV passes determine width/blank boundaries without retaining all rows. +The input text and complete output still occupy memory; this is not a +streaming HTTP/Storage contract or a bound on total parser memory. +""" + +from __future__ import annotations + +import csv +import io +from importlib.metadata import version + +from markitdown import DocumentConverter, DocumentConverterResult + +VERIFIED_VERSIONS = frozenset({"0.1.8", "0.1.9"}) + + +class RowStreamingCsvConverter(DocumentConverter): + def __init__(self): + # The private escape helper is deliberately version-gated below. Reuse + # it so pipes, backslash runs and embedded newlines remain identical. + from markitdown.converters import _csv_converter as upstream + + self._upstream = upstream.CsvConverter() + self._escape = upstream._escape_table_cell + self._detect = upstream.from_bytes + + def accepts(self, file_stream, stream_info, **kwargs): + return self._upstream.accepts(file_stream, stream_info, **kwargs) + + def convert(self, file_stream, stream_info, **kwargs): + if stream_info.charset: + content = file_stream.read().decode(stream_info.charset) + else: + data = file_stream.read() + detected = self._detect(data).best() + content = str(detected) if detected is not None else data.decode("utf-8", errors="ignore") + del data, detected + + # Upstream strips BOMs and preserves embedded CR/LF until cell escaping. + source = io.StringIO(content.lstrip("\ufeff"), newline="") + del content + with source, io.StringIO() as output: + width = 0 + header = first_data = last_data = None + for number, row in enumerate(csv.reader(source)): + if not row: + continue + width = max(width, len(row)) + if header is None: + header = number + elif first_data is None: + first_data = number + last_data = number + if header is None: + return DocumentConverterResult(markdown="") + + source.seek(0) + for number, row in enumerate(csv.reader(source)): + if number < header: + continue + if number > last_data: + break + # Leading/trailing empty rows and the empty run immediately + # after the header are trimmed. Interior empty rows survive. + if number != header and (first_data is None or number < first_data): + continue + row.extend([""] * (width - len(row))) + if number != header: + output.write("\n") + output.write("| " + " | ".join(self._escape(cell) for cell in row) + " |") + if number == header: + output.write("\n| " + " | ".join(["---"] * width) + " |") + return DocumentConverterResult(markdown=output.getvalue()) + + +def register_csv_converter(engine) -> bool: + """Unknown/older upstream versions keep their own converter unchanged.""" + if version("markitdown") not in VERIFIED_VERSIONS: + return False + try: + converter = RowStreamingCsvConverter() + except (ImportError, AttributeError): + return False + # MarkItDown's public registration API tries the latest converter first + # at the default specific-format priority. Detection/fallback stay intact. + engine.register_converter(converter) + return True diff --git a/backendN/tests/test_csv_capacity.py b/backendN/tests/test_csv_capacity.py new file mode 100644 index 0000000..1a085d6 --- /dev/null +++ b/backendN/tests/test_csv_capacity.py @@ -0,0 +1,116 @@ +"""CSV output must match the installed, verified upstream converter exactly.""" +import csv +import io +import random +from importlib.metadata import version +from pathlib import Path +from types import SimpleNamespace + +import pytest +from markitdown import StreamInfo +from markitdown.converters import CsvConverter + +from app import csv_converter + +VERIFIED = version("markitdown") in csv_converter.VERIFIED_VERSIONS +optimized = pytest.mark.skipif(not VERIFIED, reason="unverified upstream retains its original converter") + + +def compare(data, charset="utf-8"): + info = StreamInfo(extension=".csv", charset=charset) + actual = csv_converter.RowStreamingCsvConverter().convert(io.BytesIO(data), info).markdown + expected = CsvConverter().convert(io.BytesIO(data), info).markdown + assert actual == expected + return actual + + +@optimized +@pytest.mark.parametrize("text", [ + "", "\n\r\n\n", '""\n', "A,B", "\n\nA,B\n\n\n", + "\n\nA,B\n\n1,2,3\n\n\n4\n\n", "A\n1,2,3\n4,5", + "A,B\r\n1,2\r3,4\n", '\ufeff\ufeffA,B\n1,2', + '"A|B",C\n"line\nbreak","a\\\\|b"\n"carriage\rreturn","quote""quote"', + "名前,Größe,Español,🙂\n日本語,groß,café,終わり", + "A,B\n,,\n\n1,2\n\n,,\n", "A\n" + "\n" * 2000 + "last\n" * 2, +]) +def test_boundary_rows_and_cells_match_upstream(text): + compare(text.encode()) + + +@optimized +def test_random_ragged_quoted_tables_match_upstream(): + rng = random.Random(428) + values = ["", "plain", "漢字🙂", "a|b", "x\\|y", 'quote"value', "first\nsecond", "CR\rLF\r\n", " café "] + for _ in range(100): + source = io.StringIO(newline="") + writer = csv.writer(source) + writer.writerows([[rng.choice(values) for _ in range(rng.randrange(7))] for _ in range(rng.randrange(30))]) + compare(source.getvalue().encode()) + + +@optimized +@pytest.mark.parametrize("charset", ["utf-8", "utf-16", "cp1252", None]) +def test_explicit_and_detected_encodings_match_upstream(charset): + text = 'Prénom,City\nAndré,Zürich\n' + compare(text.encode(charset or "utf-8"), charset) + + +@optimized +def test_undetectable_bytes_keep_upstream_utf8_fallback(monkeypatch): + from markitdown.converters import _csv_converter as upstream + monkeypatch.setattr(upstream, "from_bytes", lambda data: SimpleNamespace(best=lambda: None)) + compare(b'A,B\n\xff,1', None) + + +@optimized +def test_parser_failure_and_borrowed_stream_ownership_match_upstream(): + data = ('A\n' + 'x' * (csv.field_size_limit() + 1)).encode() + for converter in [CsvConverter(), csv_converter.RowStreamingCsvConverter()]: + source = io.BytesIO(data) + with pytest.raises(csv.Error): + converter.convert(source, StreamInfo(extension=".csv", charset="utf-8")) + assert not source.closed + + +def test_unknown_version_keeps_engine_registration_untouched(monkeypatch): + monkeypatch.setattr(csv_converter, "version", lambda name: "0.1.10") + engine = SimpleNamespace(register_converter=lambda *args, **kwargs: pytest.fail("unknown converter registered")) + assert csv_converter.register_csv_converter(engine) is False + + +def test_unverified_version_keeps_actual_upstream_csv_output(monkeypatch): + from app.converter import NormalConverter + monkeypatch.setattr(csv_converter, "version", lambda name: "0.1.10") + data = b'Name\n"Ada|Lovelace",99\nfinal,100' + expected = CsvConverter().convert(io.BytesIO(data), StreamInfo(extension=".csv", charset="utf-8")).markdown.strip() + actual, warning = NormalConverter().convert(data, "csv", filename="old.csv", title="old") + assert actual == expected + assert warning is None + + +@optimized +def test_missing_upstream_helper_keeps_original_converter(monkeypatch): + from markitdown.converters import _csv_converter as upstream + monkeypatch.delattr(upstream, "_escape_table_cell") + engine = SimpleNamespace(register_converter=lambda *args, **kwargs: pytest.fail("incomplete converter registered")) + assert csv_converter.register_csv_converter(engine) is False + + +def test_document_pools_share_identical_csv_implementation(): + root = Path(__file__).resolve().parents[2] + assert (root / "backendN/app/csv_converter.py").read_bytes() == (root / "backendZ/app/csv_converter.py").read_bytes() + + +@optimized +def test_normal_route_uses_optimized_converter_with_unchanged_output(convert, monkeypatch): + calls = [] + original = csv_converter.RowStreamingCsvConverter.convert + def observed(self, *args, **kwargs): + calls.append(True) + return original(self, *args, **kwargs) + monkeypatch.setattr(csv_converter.RowStreamingCsvConverter, "convert", observed) + response = convert("ragged.csv", b'Name\n"Ada|Lovelace",99\n"end-marker",100') + assert response.status_code == 200, response.text + assert calls + assert "end-marker" in response.json()["content"] + assert r"Ada\|Lovelace" in response.json()["content"] diff --git a/backendZ/app/converter.py b/backendZ/app/converter.py index 2fc6d0c..a626308 100644 --- a/backendZ/app/converter.py +++ b/backendZ/app/converter.py @@ -38,6 +38,9 @@ def __init__(self) -> None: self._md, self._stream_info, self.error = None, None, str(err) return self._md = MarkItDown(enable_plugins=False) + from app.csv_converter import register_csv_converter + + register_csv_converter(self._md) self._stream_info = StreamInfo self.error = "" diff --git a/backendZ/app/csv_converter.py b/backendZ/app/csv_converter.py new file mode 100644 index 0000000..07e4eeb --- /dev/null +++ b/backendZ/app/csv_converter.py @@ -0,0 +1,90 @@ +"""Row-at-a-time CSV formatting for verified MarkItDown versions. + +Keep decoding, cell escaping and table semantics identical to upstream. +Two CSV passes determine width/blank boundaries without retaining all rows. +The input text and complete output still occupy memory; this is not a +streaming HTTP/Storage contract or a bound on total parser memory. +""" + +from __future__ import annotations + +import csv +import io +from importlib.metadata import version + +from markitdown import DocumentConverter, DocumentConverterResult + +VERIFIED_VERSIONS = frozenset({"0.1.8", "0.1.9"}) + + +class RowStreamingCsvConverter(DocumentConverter): + def __init__(self): + # The private escape helper is deliberately version-gated below. Reuse + # it so pipes, backslash runs and embedded newlines remain identical. + from markitdown.converters import _csv_converter as upstream + + self._upstream = upstream.CsvConverter() + self._escape = upstream._escape_table_cell + self._detect = upstream.from_bytes + + def accepts(self, file_stream, stream_info, **kwargs): + return self._upstream.accepts(file_stream, stream_info, **kwargs) + + def convert(self, file_stream, stream_info, **kwargs): + if stream_info.charset: + content = file_stream.read().decode(stream_info.charset) + else: + data = file_stream.read() + detected = self._detect(data).best() + content = str(detected) if detected is not None else data.decode("utf-8", errors="ignore") + del data, detected + + # Upstream strips BOMs and preserves embedded CR/LF until cell escaping. + source = io.StringIO(content.lstrip("\ufeff"), newline="") + del content + with source, io.StringIO() as output: + width = 0 + header = first_data = last_data = None + for number, row in enumerate(csv.reader(source)): + if not row: + continue + width = max(width, len(row)) + if header is None: + header = number + elif first_data is None: + first_data = number + last_data = number + if header is None: + return DocumentConverterResult(markdown="") + + source.seek(0) + for number, row in enumerate(csv.reader(source)): + if number < header: + continue + if number > last_data: + break + # Leading/trailing empty rows and the empty run immediately + # after the header are trimmed. Interior empty rows survive. + if number != header and (first_data is None or number < first_data): + continue + row.extend([""] * (width - len(row))) + if number != header: + output.write("\n") + output.write("| " + " | ".join(self._escape(cell) for cell in row) + " |") + if number == header: + output.write("\n| " + " | ".join(["---"] * width) + " |") + return DocumentConverterResult(markdown=output.getvalue()) + + +def register_csv_converter(engine) -> bool: + """Unknown/older upstream versions keep their own converter unchanged.""" + if version("markitdown") not in VERIFIED_VERSIONS: + return False + try: + converter = RowStreamingCsvConverter() + except (ImportError, AttributeError): + return False + # MarkItDown's public registration API tries the latest converter first + # at the default specific-format priority. Detection/fallback stay intact. + engine.register_converter(converter) + return True diff --git a/backendZ/tests/test_document_containers.py b/backendZ/tests/test_document_containers.py index 4071efe..83ac7ec 100644 --- a/backendZ/tests/test_document_containers.py +++ b/backendZ/tests/test_document_containers.py @@ -90,3 +90,24 @@ def test_real_documents_keep_conversion_content_inside_archive(converter, ext): assert entries[0].status == "DONE" assert entries[0].content == direct assert direct + + +def test_archive_csv_preserves_upstream_table_and_uses_row_converter(converter, monkeypatch): + import io + from importlib.metadata import version + from markitdown import MarkItDown, StreamInfo + from app import csv_converter + if version("markitdown") not in csv_converter.VERIFIED_VERSIONS: + pytest.skip("unverified upstream retains its original converter") + calls = [] + original = csv_converter.RowStreamingCsvConverter.convert + def observed(self, *args, **kwargs): + calls.append(True) + return original(self, *args, **kwargs) + monkeypatch.setattr(csv_converter.RowStreamingCsvConverter, "convert", observed) + data = '\nName\n\n"Ada|Lovelace",99\n\n"終わり🙂",100\n\n'.encode() + upstream = MarkItDown(enable_plugins=False).convert_stream(io.BytesIO(data), stream_info=StreamInfo(extension=".csv", mimetype="text/csv", filename="ragged.csv")).markdown.strip() + entries = archive.process_archive(make_zip({"ragged.csv": data}), converter.documents, ocr_available=False) + assert calls + assert entries[0].status == "DONE" + assert entries[0].content == upstream diff --git a/docs/LLMISER_OPTIMIZATION_PLAN.md b/docs/LLMISER_OPTIMIZATION_PLAN.md index 31df36d..a6ea3ff 100644 --- a/docs/LLMISER_OPTIMIZATION_PLAN.md +++ b/docs/LLMISER_OPTIMIZATION_PLAN.md @@ -10,6 +10,8 @@ Shared reader statistics shipped in PR #9 at `84e510db62ea41f31c131771eb7d32b5e9 PR #11 released [large-preview control](LLMISER_LARGE_PREVIEWS.md) at `0501cd0381b4a41e9a9e94abf791ebd4fd8fd188`. Browser-generated dense Markdown exposed multi-second mount/layout work. Automatic formatting now pauses above 50,000 characters; full formatting is explicitly available, and raw content/exports are complete. Local checks passed 261 tests, the final 20-test selection and production build; all seven CI jobs and all six production deployments passed. The 28 October audit passed 20 HTTP checks with all pools 2/2 ready. Next work should measure native textarea input/representative devices or audit backend parser capacity; a worker alone will not eliminate the observed DOM/layout cost. +The current `codex/csv-parser-memory` candidate contains [a source capacity audit and CSV allocation reduction](LLMISER_PARSER_CAPACITY.md) for N/Z. It preserves output on verified MarkItDown 0.1.8/0.1.9 and falls back unchanged on other versions. Three fresh processes per mode show lower large-fixture peak memory; no concurrency/limit changes are justified by that local evidence. See its PR for release state, then measure workbook/native and final response/merge costs before extending this slice. + ## Verified starting point The copied source already contains atomic job startup/finalization, cleanup recovery, upload/active-job/storage-reservation admission, shared archive container checks, bounded PDF page uploads/OCR preparation, lazy Markdown/highlighter loading and current-edit export snapshots. Do not reimplement these based on old “proposed” checklist rows. Their deployment/capacity evidence is distinct from source tests. @@ -40,7 +42,7 @@ Acceptance: expired link, later-part failure, complete Unicode assembly, restart | Work | Next bounded change / dependency | | --- | --- | | Shared statistics and large previews | Counting released in PR #9; browser profiling and large-preview control released in PR #11; measure native input/devices before deeper editor/worker changes | -| Backend parser capacity | Audit all parsers beyond the already-shipped PDF/OCR preparation bounds; preserve fidelity and measure synchronized memory before tuning concurrency | +| Backend parser capacity | Source audit and synchronized CSV memory slice implemented on the current branch; check its PR for release. Next measure workbook/native and response/merge memory; preserve fidelity before tuning concurrency | | Shared deadline | Add an optional, backward-compatible remaining-budget protocol, reserving finalization time; test slow failures/cold starts | | Controller detail pagination | Top-level lists already paginate; bound files/tree/events/work-items with stable cursors and truthful totals; query-plan evidence before new indexes | | Replay-stable object identities | Fence attempt-specific writes before deterministic IDs; SQL attempt guards alone do not prevent a stale Storage overwrite; additive rollout for new roots only | diff --git a/docs/LLMISER_PARSER_CAPACITY.md b/docs/LLMISER_PARSER_CAPACITY.md new file mode 100644 index 0000000..664c987 --- /dev/null +++ b/docs/LLMISER_PARSER_CAPACITY.md @@ -0,0 +1,71 @@ +# Parser capacity audit and CSV memory reduction + +Updated 28 October 2026. Work is confined to LLMiser. This is a source audit and one measured CSV change; it does not certify every format's maximum safe concurrency on the hosting platforms. + +## Audited starting point + +The four live normal/archive workers reported MarkItDown **0.1.9** before this change. The existing local Python environment had 0.1.7. Small, dependency-free package copies of 0.1.8/0.1.9 were downloaded into the ignored operator folder for compatibility tests; the installed environment and project requirements were not changed. + +| Workload | Existing source limits / behavior | Remaining capacity boundary | +| --- | --- | --- | +| Normal text, JSON, XML and HTML | 15 MiB input; decoding and complete Markdown returned by MarkItDown; HTML can create a full parsed tree | Input size alone does not bound parser/output allocations | +| CSV | Same input cap; upstream retains all parsed rows and formatted row strings before constructing the full output | This slice removes those retained row collections; full text/output still occupy memory | +| XLSX/XLS | MarkItDown reads workbook sheets into pandas objects and renders HTML before Markdown | Measure representative sheets, types and formatting before replacing the path; no read-only/streaming workbook change here | +| DOCX/PPTX/EPUB | Existing shared container validation: at most 10,000 ZIP entries and 250 MiB advertised expansion | Expanded XML/object graphs still need measurement; limits are not a peak-memory estimate | +| PDF | Existing serialized native PDFium operations; one rendered scan uploaded before the next; 12 MP render target and up to 100 scanned pages for OCR | Native text parsing and whole-result assembly remain; optional sampled PDF fast path stays off by default because of its documented late-table fidelity tradeoff | +| Project ZIP | 15 MiB compressed input, 2,000 entries, 100 MiB aggregate expansion, 15 MiB document entries, 10 MiB images, 40 MiB output; document entries parsed sequentially inside each archive | Concurrent requests and retained converted entries/merge buffers remain separate memory costs | +| OCR images | Existing preparation/recognition process lock, one Uvicorn worker, OMP thread limit 1; 25 MP general / 12 MP WebP / 10,000 px side guards and 12 MP preparation target | Existing bounds remain; this slice changes neither OCR concurrency nor image fidelity | + +Normal/archive endpoints are synchronous worker-thread handlers. The source has no new global parser gate in this change. Queue defaults admit four normal items and one item per configured OCR/archive instance across the shared pools, with environment overrides. This is a source observation, not an inspection of current platform memory settings or proof that all in-flight work shares one process. No queue limits, native locks, service sizes or worker counts are changed. + +## CSV implementation and fidelity + +`backendN/app/csv_converter.py` and its identical archive-pool copy register through MarkItDown's public converter API. They live outside the three-pool shared `common/` package because OCR has no document-parser dependency. The first CSV pass keeps only width and nonempty-row boundaries. The second writes formatted rows into a text buffer instead of retaining the entire parsed table and a list of formatted rows. + +The optimization is enabled only for verified MarkItDown **0.1.8 and 0.1.9**. It delegates format acceptance and reuses that version's charset detector and cell-escaping helper. It preserves BOM handling, widest-row padding, blank-row trimming, quoted CR/LF, pipes/backslashes, Unicode and borrowed input ownership. Older/unknown versions or a missing helper retain their original converter. This matters because 0.1.7's ragged-row and escaping behavior differs from the deployed version. + +References: [upstream CSV implementation](https://github.com/microsoft/markitdown/blob/v0.1.9/packages/markitdown/src/markitdown/converters/_csv_converter.py) and [converter registration](https://github.com/microsoft/markitdown/blob/v0.1.9/packages/markitdown/src/markitdown/_markitdown.py). The helper is private, so the version gate and fallback must remain until additional versions are verified. Do not simply expand the allowlist during a dependency upgrade. + +The returned Markdown, API/profile behavior, file limits and complete exports retain their existing contract. This is row-at-a-time formatting, not constant-memory input or streamed HTTP/Storage output. Full input decoding, MarkItDown's final normalization and the entire output remain materialized; very wide/ragged data can still produce a large result. No new dependency, database migration, blog change or user-facing translation is needed. + +## Reproducing measurements + +Use an existing Python environment with `backendN/requirements-dev.txt` and a verified MarkItDown version. From the repository root, run each command in a fresh process: + +```sh +python tests/load/csv_memory_bench.py --mode upstream --rows 50000 --concurrency 2 +python tests/load/csv_memory_bench.py --mode current --rows 50000 --concurrency 2 +``` + +Repeat with `--heap` for Python allocation tracing. The benchmark disables dotenv discovery and clears Storage configuration before parser imports. It uses only generated multilingual CSV; no environment credentials, server or storage calls. Its upstream mode re-registers the original CSV converter; both modes retain identical MarkItDown detection/normalization and normal-converter wrappers. + +The fixture has four columns, quoted text, pipes, café/Japanese/emoji and unique first/last records. At 50,000 data rows its input is 3,996,053 bytes and output is 4,296,087 bytes per conversion. Concurrent threads start at a barrier and retain their complete outputs. Comparisons require matching output hashes and byte counts. This measures the parser/normalizer, not API serialization, profiles, Storage latency or the frontend. + +Local evidence uses Python 3.14.6 / MarkItDown 0.1.9 on Windows; CI uses Python 3.12. Three fresh processes per mode/fixture, alternating mode order, with no concurrent regression/build workload. Process peak is the lifetime high-water memory counter, including imports; the recorded pre-run high-water mark provides context. Python heap tracing starts after fixture/engine setup and adds timing overhead. Compare untraced runs for timing; traced runs for allocation evidence. These are generated-fixture observations, not a hosting capacity, throughput or billing guarantee. + +Median of three fresh processes per mode (24 measured processes total): + +| Rows / synchronized calls | Tracing | Upstream / current seconds | Upstream / current process peak MiB | Upstream / current traced heap MiB | +| --- | --- | ---: | ---: | ---: | +| 1,000 / 1 | Off | 0.042 / 0.082 | 143.46 / 143.19 | — | +| 50,000 / 1 | Off | 1.923 / 1.079 | 225.11 / 190.72 | — | +| 50,000 / 2 | Off | 3.275 / 3.160 | 297.19 / 221.92 | — | +| 50,000 / 2 | On | 16.253 / 12.324 | 368.10 / 238.62 | 146.24 / 96.12 | + +The synchronized untraced sample used about **25% less peak process memory**; its elapsed time was similar. Traced peak Python allocations fell about **34%**. The small sample's timing regressed by 40 ms in this run, so this is not advertised as a universal speed improvement. Pre-run process high-water marks for the large fixtures were about 155–157 MiB. All corresponding before/after output byte counts and SHA-256 hashes matched, including every first/last record. The large output hash was `0481293612cc8e09bca7d1386a9d5ad6afa595f62843e4177eec8716225dbc74`. + +## Verification and release boundary + +Output-equivalence tests compare the installed upstream converter with boundary fixtures and 100 deterministic ragged/quoted tables. They cover explicit/detected encodings, detector failure, oversized fields, Unicode, blank runs, compatibility fallback and standalone/archive wiring. The two document-pool copies must remain identical. + +Local final checks: + +- MarkItDown 0.1.9: **118 normal-backend tests** and **98 archive-backend tests** passed. +- MarkItDown 0.1.8: **25 CSV tests** passed against that package's actual upstream converter. +- Original local 0.1.7: **3 fallback/integrity tests** passed; 22 optimization-only cases skipped because the adapter is deliberately disabled. The fallback test compares an actual normal-converter result with that version's original CSV output. +- The shared three-pool `common/` integrity checks stay intact; OCR has no new parser dependency. No runtime requirements changed. +- Before release, tiny generated authenticated multipart CSV/ZIP probes returned 200 on **N1, N2, Z1 and Z2**, each preserving the expected widest-row table, pipe escape, interior blank row and final Unicode marker. They created no Storage jobs. + +The candidate branch is `codex/csv-parser-memory`. CI/merge/deployment and the matching post-release probe must be checked on its PR before calling it released. Local logs, package copies, raw benchmark JSON and operator probes remain ignored under `project-docs/parser-audit/`; no credentials are included in public evidence. + +Next: profile realistic workbook/native parser memory and full response/merge costs on representative hosts, then evaluate admission/deadlines before any concurrency increase. The CSV reduction alone does not justify raising queue or worker limits. diff --git a/docs/START_HERE.md b/docs/START_HERE.md index 88d79f9..6bec452 100644 --- a/docs/START_HERE.md +++ b/docs/START_HERE.md @@ -10,6 +10,8 @@ The full app supports English (US), German, Japanese, Spanish, Russian and Simpl Released [large-preview control](LLMISER_LARGE_PREVIEWS.md) pauses automatic formatting above 50,000 characters while retaining complete raw text and exports. Full formatting is available on request. Local/generated browser checks cover editing, document switching and exact full downloads; representative-device/native textarea profiling and the backend parser-capacity audit remain next boundaries. The six interface dictionaries now contain 631 messages each. +The next bounded backend slice is [parser capacity and CSV memory reduction](LLMISER_PARSER_CAPACITY.md), on `codex/csv-parser-memory`. It preserves verified upstream CSV output while reducing retained row allocations in both document pools. The audit distinguishes source limits, local generated measurements and live-worker acceptance; check that branch's PR for merge/deployment state before treating it as released. No concurrency increase follows from the benchmark. + ## Legacy frontend replacement is deployed The owner wants the old converter removed from its frontend hosting, permanent public redirects preserved, and a small LLMiser-only banner as fallback. Replacement source is [legacy-site/](../legacy-site/README.md), inside this repository. It has no converter, API or database client. The working LLMiser project continues using `frontend`; only the old frontend project may use `legacy-site`. diff --git a/docs/ai-handover/HANDOVER.md b/docs/ai-handover/HANDOVER.md index beb98f4..0385432 100644 --- a/docs/ai-handover/HANDOVER.md +++ b/docs/ai-handover/HANDOVER.md @@ -56,7 +56,9 @@ Active local tooling now has zero absolute references to either old checkout. LL The owner asked to continue the optimization plan after the retirement handover. PR #8 completed that handover; PR #9 released [shared reader statistics](../LLMISER_READER_STATISTICS.md). PR #11 released [large-preview control and reproducible browser profiling](../LLMISER_LARGE_PREVIEWS.md): above 50,000 UTF-16 characters, formatting waits for an explicit request. Editing/document switching resets the request; complete raw text/copy/exports remain available. All six packs contain 631 messages. Local evidence: 261 full tests, final 20 focused tests, lint/consistency checks and production build passed; generated browser fixtures verify full rendering, edit re-pause, identical-text document switching, manual pause and exact current-edit download. CI passed all seven jobs, production deployed revision 0501cd0, the 20-check HTTP audit passed, and normal/OCR/archive were each 2/2 ready. No fresh production conversion/admin/cleanup acceptance is implied by that audit. -Next investigate native textarea/input behavior and representative devices before workers/editor virtualization, or take the separately bounded backend parser-capacity audit from [the optimization queue](../LLMISER_OPTIMIZATION_PLAN.md). Do not repeat the initial profiling, silently truncate source/exports, or mistake a local component benchmark for production conversion/storage acceptance. The optimization plan's 22 October unmerged-branch statements are historical. +The current `codex/csv-parser-memory` branch adds [a parser-capacity audit and bounded CSV memory reduction](../LLMISER_PARSER_CAPACITY.md). N/Z use identical `app/csv_converter.py` modules, outside the three-pool shared `common/` package so OCR gains no MarkItDown dependency. Enable only on verified upstream 0.1.8/0.1.9; unknown versions/missing helper retain the original converter. Never expand the version allowlist without output-equivalence tests. Generated before/after hashes match, including synchronized conversions; queue limits and all other parsers stay unchanged. Inspect its PR for final CI/merge/deployment state. + +Next investigate native textarea/input behavior and representative devices before workers/editor virtualization, or measure XLSX/native parser and response/merge memory from [the optimization queue](../LLMISER_OPTIMIZATION_PLAN.md). Do not repeat the shipped preview work, silently truncate source/exports, or mistake local parser measurements/multipart probes for Storage-backed acceptance. The optimization plan's 22 October unmerged-branch statements are historical. Use Node.js 22. From `frontend`: diff --git a/docs/ai-handover/PROGRESS.md b/docs/ai-handover/PROGRESS.md index 2824ade..9f14f2f 100644 --- a/docs/ai-handover/PROGRESS.md +++ b/docs/ai-handover/PROGRESS.md @@ -50,6 +50,8 @@ See [release record](../LLMISER_RELEASE_2026-10-26.md), [localization](../LLMISE The owner resumed the implementation plan after the handover was merged in PR #8. [Shared reader statistics](../LLMISER_READER_STATISTICS.md) shipped in PR #9. [Large-preview control](../LLMISER_LARGE_PREVIEWS.md) shipped in PR #11 at 0501cd0 after 261 frontend CI tests/all seven jobs passed. All six production deployments succeeded; the 28 October 20-check HTTP audit passed and all three pools were 2/2 ready. Actual-component generated browser measurements and exact raw/download acceptance support the preview boundary. Representative-device/native textarea profiling and deeper parser-capacity work remain separate. +Current implementation branch: `codex/csv-parser-memory`. [Parser audit/CSV evidence](../LLMISER_PARSER_CAPACITY.md) records existing bounds, verified-version output preservation and three-run memory measurements. The synchronized large untraced fixture used about 25% less peak process memory, with identical output; the small timing sample regressed, so no universal speedup is claimed. Live pre-release multipart CSV/ZIP probes passed on all four document workers. Final regression/CI/deployment evidence must be checked before treating the candidate as released. No worker/queue tuning or Storage-backed conversion acceptance is implied. + - Retain the domains and immediate permanent redirects after the completed static frontend cutover. - Review any additional GitHub-only material the owner needs beyond the completed record exports. Actions artifacts/logs/secrets, provider settings and linked attachments were not exported. - Keep a verified private backup outside the directories that will be deleted, with an owner-controlled second copy if local disk loss matters. The current snapshots are in LLMiser, not an independent/off-machine backup. diff --git a/tests/load/csv_memory_bench.py b/tests/load/csv_memory_bench.py new file mode 100644 index 0000000..da05aff --- /dev/null +++ b/tests/load/csv_memory_bench.py @@ -0,0 +1,103 @@ +"""Offline CSV parser comparison; one fresh process per mode/fixture. + +Use --mode upstream/current --rows 50000 --concurrency 2 [--heap]. +Set PYTHONPATH to a verified MarkItDown package when testing another version. +No servers, environment files, user documents or Storage calls are used. +""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import csv +import ctypes +import gc +import hashlib +from importlib.metadata import version +import io +import json +import os +from pathlib import Path +import sys +import threading +import time +import tracemalloc + +# Guard before importing any parser dependency that may call load_dotenv. +os.environ["PYTHON_DOTENV_DISABLED"] = "1" +os.environ["SUPABASE_URL"] = "" +os.environ["SUPABASE_SERVICE_ROLE_KEY"] = "" +os.environ["INTERNAL_SHARED_SECRET"] = "offline-benchmark-only" +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "backendN")) + +from app.converter import NormalConverter # noqa: E402 +from app.csv_converter import VERIFIED_VERSIONS # noqa: E402 +from markitdown.converters import CsvConverter # noqa: E402 + + +def peak_rss_mib(): + if sys.platform != "win32": + import resource + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return value / (1024**2 if sys.platform == "darwin" else 1024) + from ctypes import wintypes + class Counters(ctypes.Structure): + _fields_ = [(name, wintypes.DWORD) for name in ["cb", "PageFaultCount"]] + [(name, ctypes.c_size_t) for name in ["PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage", "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", "QuotaNonPagedPoolUsage", "PagefileUsage", "PeakPagefileUsage"]] + counters = Counters() + counters.cb = ctypes.sizeof(counters) + probe = ctypes.windll.psapi.GetProcessMemoryInfo + probe.argtypes = [wintypes.HANDLE, ctypes.POINTER(Counters), wintypes.DWORD] + if not probe(wintypes.HANDLE(-1), ctypes.byref(counters), counters.cb): + raise OSError("Could not read process memory") + return counters.PeakWorkingSetSize / 1024**2 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["upstream", "current"], required=True) + parser.add_argument("--rows", type=int, default=50000) + parser.add_argument("--concurrency", type=int, choices=[1, 2, 4], default=2) + parser.add_argument("--heap", action="store_true", help="also trace Python allocations; timing then includes tracing overhead") + args = parser.parse_args() + if not 1 <= args.rows <= 100000: + parser.error("rows must be between 1 and 100000") + if version("markitdown") not in VERIFIED_VERSIONS: + parser.error("use a verified MarkItDown version for a valid before/after comparison") + source = io.StringIO(newline="") + writer = csv.writer(source) + writer.writerow(["id", "name", "amount", "note"]) + for index in range(args.rows): + writer.writerow([index, f"record-{index}", index * 3.5, 'generated café 日本語 🙂 | quoted "value"']) + data = source.getvalue().encode() + source.close() + if len(data) > 15 * 1024 * 1024: + raise ValueError("fixture exceeds the deployed standalone upload cap") + engine = NormalConverter() + if args.mode == "upstream": + # Latest registration at equal priority runs first; all detection, + # normalization and wrapper work remains the same in both modes. + engine._md.register_converter(CsvConverter()) + engine.validate(data, "csv") + gc.collect() + before_peak = peak_rss_mib() + if args.heap: + tracemalloc.start() + start = threading.Barrier(args.concurrency) + def convert(_): + start.wait(timeout=30) + return engine.convert(data, "csv", filename="generated.csv", title="generated")[0] + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + outputs = list(pool.map(convert, range(args.concurrency))) + elapsed = time.perf_counter() - started + peak = peak_rss_mib() + heap = tracemalloc.get_traced_memory()[1] / 1024**2 if args.heap else None + if args.heap: + tracemalloc.stop() + assert all(output == outputs[0] for output in outputs) + assert "record-0" in outputs[0] and f"record-{args.rows-1}" in outputs[0] + print(json.dumps({"mode": args.mode, "python": sys.version.split()[0], "markitdown": version("markitdown"), "rows": args.rows, "input_bytes": len(data), "concurrency": args.concurrency, "seconds": round(elapsed, 4), "tracemalloc_enabled": args.heap, "peak_python_heap_MiB": round(heap, 2) if heap is not None else None, "process_peak_before_MiB": round(before_peak, 2), "process_peak_after_MiB": round(peak, 2), "output_bytes_each": len(outputs[0].encode()), "output_sha256": hashlib.sha256(outputs[0].encode()).hexdigest(), "first_and_last_record": True, "host_capacity_claim": False})) + + +if __name__ == "__main__": + main()