diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..14e8321 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,29 @@ +{ + "name": "research-figure-studio", + "version": "0.2.0", + "description": "Generate paper-grounded scientific figures as editable PowerPoint compositions.", + "author": { + "name": "ResearchFigureStudio contributors", + "url": "https://github.com/yiweiqin/ResearchFigureStudio" + }, + "homepage": "https://github.com/yiweiqin/ResearchFigureStudio", + "repository": "https://github.com/yiweiqin/ResearchFigureStudio", + "license": "MIT", + "keywords": ["research", "scientific-figures", "powerpoint", "pptx", "codex"], + "skills": "./skills/", + "interface": { + "displayName": "Research Figure Studio", + "shortDescription": "Paper to editable PowerPoint figures", + "longDescription": "Build scientifically grounded framework figures whose exact labels and relations remain editable in PowerPoint.", + "developerName": "ResearchFigureStudio contributors", + "category": "Productivity", + "capabilities": ["Read", "Write"], + "websiteURL": "https://github.com/yiweiqin/ResearchFigureStudio", + "defaultPrompt": [ + "Turn this paper into an editable PPT framework figure.", + "Rebuild this scientific figure as editable PowerPoint.", + "Check whether this figure matches the paper semantics." + ], + "brandColor": "#2F6F8F" + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8c5ccd..9e78731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,16 +20,16 @@ jobs: - name: Compile package and bundled scripts run: | python -m compileall -q rfs - python -m py_compile codex-skills/research-figure-making/scripts/validate_framework_outputs.py - python -m py_compile codex-skills/research-figure-making/scripts/estimate_asset_fill.py + python -m py_compile skills/research-figure-studio/scripts/validate_framework_outputs.py + python -m py_compile skills/research-figure-studio/scripts/estimate_asset_fill.py - name: Run unit tests run: python -m unittest discover -s tests -q - name: Check bundled skill metadata shell: python run: | from pathlib import Path - skill = Path('codex-skills/research-figure-making/SKILL.md') + skill = Path('skills/research-figure-studio/SKILL.md') text = skill.read_text(encoding='utf-8') assert text.startswith('---'), 'SKILL.md must start with YAML frontmatter' - assert 'name: research-figure-making' in text + assert 'name: research-figure-studio' in text assert 'description:' in text diff --git a/.gitignore b/.gitignore index 3250ca3..4a7edc3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,11 @@ research_figure_studio.egg-info/ venv/ env/ tmp/translation_env/ +tmp/codex-presentations/ +tmp/pdfs/*/ +tmp/pdfs/ +tmp/paper-to-editable-smoke*/ +benchmarks/**/inputs/ *.key *.pem @@ -45,6 +50,17 @@ assets_raw/ *.tiff *.svg +# Local reference-learning figures; keep only repository-authored guides/placeholders +reference-figures/**/*.png +reference-figures/**/*.jpg +reference-figures/**/*.jpeg +reference-figures/**/*.webp +reference-figures/**/*.tif +reference-figures/**/*.tiff +reference-figures/**/*.pdf +!reference-figures/README.md +!reference-figures/**/.gitkeep + # Keep repository-authored docs/examples if explicitly tracked by extension overrides !docs/**/*.md !docs/assets/*.png diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d647158 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,326 @@ +# AGENTS.md + +## 必读：进入项目后的第一件事 + +任何代理进入本仓库后，先阅读项目核心思路文档 `docs/paperimage.docx`（「论文 → 文生框架图 → 可编辑框架图」整体思路与 Judge 模型训练目标的最新权威描述）。它是 Word 二进制文件，读取时请先抽取文本（解压 `word/document.xml` 并提取 `` 内容，或先转成纯文本）。该文档的优先级高于本文件，冲突时以用户最新意图为准。 + +## 协作身份 + +- 用户叫 Sen-illion。 +- 助手叫 Kira1。 +- 本机 git 可执行文件：`D:\Git\cmd\git.exe`（PATH 中可能没有 `git`，请始终用完整路径调用，例如 `& "D:\Git\cmd\git.exe" -C "D:\文件夹\Paperimage" status`）。 +- 本文件用于帮助后续进入 `C:\Users\User\Desktop\Paperimage` 的代理快速理解项目、遵守约定并安全修改代码。 + +## 项目一句话 + +Paperimage 是从 ResearchFigureStudio 分支 `codex/integrate-teammate-and-restructure` 拆出的公开仓库。它是一个 **PPTX-first 的论文到科研框架图生成与可编辑重建引擎**，同时按 Codex plugin 形态整理：论文语义合同负责科学内容，生成或参考图负责视觉布局，本地 deterministic Python 代码负责编译可编辑 PowerPoint。 + +项目重点不是生成一张扁平化大图，而是生成： + +- 可编辑的 `editable_composition.pptx` +- 槽级别 raster 图像资产或按策略跳过的可编辑对象 +- PPT 可编辑的标签、箭头、连接线、面板、公式和分组 +- 可验证的中间 JSON/Markdown 合同文件 +- 可由 Codex skill/插件继续复现的科研制图工作流 + +## 当前能力边界 + +- 当前是早期工程原型，强项是把多个小图像块稳定放入 PPTX 精确位置，并保留关键文本、箭头、面板和关系为可编辑对象。 +- 生成的图像块仍然是 raster asset，不是完全可编辑的科学矢量对象。 +- 不应把它宣传成端到端顶会级成图器；它更像一个方便人工继续修订的可编辑科研图生成管线。 +- 项目目前最重要的开放难题仍然是箭头和连接器定位，包括 source-target 绑定、多段路由、避让、虚线环和参考图逻辑保持。 +- Paperimage 额外保留了 `rebuild-editable-pro` 的 controlled professional DSL 路线，用于从参考图生成受控 DSL，再由本地代码编译为 PPT 合同；VLM 不应直接写任意 PowerPoint 代码。 +- 默认产品策略是 `smart-api`：reference crop 默认只作为 API 参考上下文，不作为最终 PPT 图像资产；只有显式启用 generation-policy / policy 实验模式时，才允许按槽保留 `reference_crop`。 + +## 技术栈 + +- Python 包名：`research-figure-studio` +- Python 要求：`>=3.10` +- CLI 入口：`rfs = rfs.cli:main` +- 主要依赖： + - `pillow` + - `python-pptx` + - `PyMuPDF` + - `requests` + - `opencv-python-headless` + - Windows 上可选/条件依赖 `pywin32` +- 可选 OCR 依赖：`paddleocr`、`easyocr`，通过 `python -m pip install -e ".[ocr]"` 安装。 +- Windows + 本机 PowerPoint 是最佳导出环境，因为 PDF/PNG 导出依赖 PowerPoint COM 和 PyMuPDF。 + +## 目录结构 + +- `.codex-plugin/plugin.json`：Codex plugin manifest。 +- `skills/research-figure-studio/`：随仓库提供的 Codex skill，描述科研制图和验证工作流。 +- `skills/research-figure-studio/scripts/validate_framework_outputs.py`：独立输出验证脚本。 +- `rfs/`：主 Python 包，仍保持兼容导入。 +- `rfs/cli.py`：命令行入口，提供 `doctor`、`make-framework`、`paper-to-image`、`paper-to-editable`、`rebuild-editable`、`rebuild-editable-eval`、`rebuild-editable-pro`、`rebuild-judge`、`rebuild-evolve`、`coevolve-image`、`coevolution-report`、`presentations-qa`、`validate`。 +- `rfs/workflows/`：产品级 workflow 入口；`paper_to_editable.py` 是 paper-to-image + editable rebuild 的组合路径。 +- `rfs/contracts/`：稳定合同与 semantic binding；`semantic.py` 将 paper review / figure specification 重新绑定到 PPT 文本和关系。 +- `rfs/composition/`：PowerPoint 编译与 preview renderer；`rfs/ppt_compiler.py` 是兼容 wrapper。 +- `rfs/evaluation/`：deterministic rebuild 质量检查。 +- `rfs/paper_to_image/`：论文解析、paper review、模板、blueprint、候选整图生成和候选审查。 +- `rfs/editable_rebuild.py`：reference-only 可编辑 PPT rebuild 流水线。 +- `rfs/rebuild_design_planner.py`：rebuild-editable 的全局参考图理解阶段，生成 narrative、layer plan、generation policy 和 flow graph。 +- `rfs/layout_planner.py`、`rfs/control_localizer.py`、`rfs/layout_semantic_planner.py`：rebuild-editable 使用的布局、控件和语义规划模块。 +- `rfs/rebuild_vlm_adapters.py`、`rfs/rebuild_vlm_validation.py`、`rfs/rebuild_eval.py`、`rfs/vlm_client.py`：rebuild 工作流的 VLM 调用、输出验证和 heuristic-vs-VLM 评估支持。 +- `rfs/professional_*.py`：`rebuild-editable-pro` controlled professional DSL 工作流。 +- `rfs/coevolution/`：whole-image Creator/Judge co-evolution 实验基线和指标汇总。 +- `rfs/run_log.py`：append-only 运行事件日志（`run_log.jsonl`），状态 = fold(事件)；支持续写 seq 与 `last_event`。 +- `rfs/run_projection.py`：把运行日志投影成「给人看的 `run_report.md`」和「给 Judge 的过程轨迹」。 +- `rfs/rebuild_process_judge.py`：过程监督 Judge（prompt-as-judge），按阶段归因输出 `stage_feedback`。 +- `rfs/rebuild_judge.py`：把 Judge 的 fault 阶段映射成粗修复计划（reanalyze/regen_assets/recompile）。 +- `rfs/rebuild_evolution.py`：自动分块进化循环（重建 → 判 → 修 → 再判）。 +- `tests/unit/`、`tests/integration/`、`tests/e2e/`：按层级组织的测试。 +- `benchmarks/`：产品验收用例和指标说明。 +- `experiments/`：研究原型、legacy rebuild 脚本和文献综述材料。 +- `webapp/`：本地 Web Studio 前端（论文/参考图 → 可编辑 PPTX 的可视化运行页），`webapp/run_server.py` 启动并注入 `.env` API，运行节点按中间产物文件实时点亮，产物输出到 `output/webapp_runs/`（被 gitignore）。 +- `docs/architecture.md`、`docs/workflow.md`、`docs/rebuild-editable.md`、`docs/paper-to-image.md`、`docs/gpt-reproduction-workflow.md`：架构、流程、重建入口和复现说明。 +- `docs/run-log-events.md`、`docs/rebuild-process-judge.md`：运行日志事件 schema 与过程监督 Judge 用法。 + +## 核心工作流 + +Paperimage 的主要路线： + +```text +paper-to-image: +paper -> paper_review.json / figure_specification.json -> +template profile / layout_blueprint.png -> +image candidates -> selected_image.png 或 engineering_preview.png + +rebuild-editable: +reference image -> global design plan / layer plan / generation policy / flow graph -> +reference_geometry.json / reference_controls.json -> +semantic binding / editable text layer / arrow routing -> +asset_generation_specs.json -> slot assets -> +figure_program.json -> editable_composition.pptx -> preview/QA/report + +paper-to-editable: +paper-to-image -> load paper semantic contract -> +rebuild-editable under editable/ -> paper_to_editable_result.json + +rebuild-editable-pro: +reference image -> baseline contracts -> controlled professional DSL -> +local compiler -> editable PPTX + professional reports / repairs +``` + +运行日志 + 过程监督（分块进化）： + +```text +rebuild-editable -> run_log.jsonl / run_report.md -> +rebuild_process_judge -> stage_feedback -> +rebuild-judge / rebuild-evolve 粗修复（reanalyze/regen_assets/recompile）-> 再判 +``` + +重要原则： + +- 论文合同负责科学术语、模块含义、实体标签、关系和证据链接。 +- 参考图或生成图负责布局、视觉节奏、颜色和局部视觉风格。 +- 箭头、连接线、虚线环、面板框、标签、公式和关键文本必须是 PPT 可编辑对象，不应作为图像资产生成。 +- 禁止生成一张完整架构大图后裁剪或截图作为交付。 +- 禁止 semantic crop、cover-crop、crop-to-ratio；图像插入应是 contain/no-crop 语义。 +- `smart-api` 是默认产品策略；generation-policy / policy 是显式实验能力，必须在报告里记录每个槽的策略来源和执行结果。 +- OCR/VLM 失败时可以 heuristic fallback，但必须在报告中明示，不能伪装成高置信 OCR/VLM 结果。 + +## CLI 常用命令 + +安装开发包： + +```powershell +python -m pip install --upgrade pip +python -m pip install -e . +``` + +健康检查： + +```powershell +rfs doctor --json +``` + +离线 paper-to-image 烟测： + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_placeholder" ` + --planner-mode heuristic ` + --asset-mode placeholder ` + --candidates 2 ` + --review-mode heuristic ` + --ocr-engine off ` + --json +``` + +reference-only 可编辑重建： + +```powershell +rfs rebuild-editable ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_check" ` + --design-plan-mode vlm ` + --asset-mode api ` + --asset-policy smart-api ` + --layout-mode hybrid ` + --control-mode hybrid ` + --text-mode ocr ` + --export-preview ` + --json +``` + +显式 generation-policy 实验模式： + +```powershell +rfs rebuild-editable ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_policy" ` + --asset-mode policy ` + --asset-policy generation-policy ` + --layout-mode hybrid ` + --control-mode hybrid ` + --json +``` + +professional DSL 重建： + +```powershell +rfs rebuild-editable-pro ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_pro" ` + --asset-mode api ` + --asset-policy smart-api ` + --repair-rounds 2 ` + --export-preview ` + --json +``` + +过程监督判定（需 API）： + +```powershell +rfs rebuild-judge ` + --out "output\rebuild_check" ` + --repair-plan ` + --json +``` + +自动分块进化闭环（需 API）： + +```powershell +rfs rebuild-evolve ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_evolve" ` + --max-rounds 3 ` + --asset-mode api ` + --asset-policy smart-api ` + --layout-mode hybrid ` + --control-mode hybrid ` + --text-mode ocr ` + --export-preview ` + --json +``` + +## 环境变量与密钥 + +不要把 API key 写入源码、文档示例的真实值、测试夹具或输出目录。只使用环境变量： + +- `API_BASE` +- `API_KEY` +- `GEMINI_API_KEY` +- `GEMINI_GEN_IMG_URL` +- `MODEL_VLM` +- `RFS_VLM_TIMEOUT` +- `RFS_VLM_RETRIES` +- `RFS_REBUILD_LAYOUT_MODEL` +- `RFS_REBUILD_DESIGN_MODEL` +- `RFS_REBUILD_CONTROL_MODEL` +- `RFS_REBUILD_SEMANTIC_MODEL` +- `RFS_TEXT_GROUPING_MODEL` +- `RFS_CRITIC_MODEL` +- `RFS_ONLINE_JUDGE_MODEL` +- `RFS_FROZEN_JUDGE_MODEL` +- `RFS_IMAGE_MODEL` +- `RFS_IMAGE_EDIT_URL` +- `IMAGE_MODEL` + +## 开发与测试 + +本地开发检查： + +```powershell +python -m compileall -q rfs +python -m py_compile skills\research-figure-studio\scripts\validate_framework_outputs.py +python -m py_compile skills\research-figure-studio\scripts\estimate_asset_fill.py +python -m unittest discover -s tests -q +``` + +CI 在 Windows 上跑 Python 3.10 和 3.11，步骤包括： + +- editable install +- compile `rfs` +- py_compile skill scripts +- unittest +- 检查 bundled skill frontmatter + +运行项目时优先使用虚拟环境；仓库根目录 `.env` 里的 API 变量需要手动注入当前 shell，不要依赖代码自动读取密钥文件。 + +## 仓库卫生 + +不要提交： + +- `output/`、`outputs/` +- 论文、手稿、私有数据、用户参考图 +- 生成的 PPTX/PDF/PNG/JPG/SVG/TIFF/WebP 等图像或成图资产 +- `.env`、API key、私钥 +- `__pycache__/`、`.pytest_cache/`、`*.egg-info/` +- 本地虚拟环境，如 `.venv/` + +当前 `.gitignore` 已经覆盖多数生成物。注意 `AGENTS.md` 是需要保留的仓库说明文件，修改时不要误删用户内容。 + +## 修改代码时的建议 + +- 优先保持 Paperimage 的插件化和稳定包边界，不要把 `rfs/workflows`、`rfs/contracts`、`rfs/composition` 回退成旧的扁平结构。 +- 对 JSON/Markdown 输出继续使用 `rfs.utils.write_json` 和 `write_text`，保持 UTF-8 和 `ensure_ascii=False`。 +- 新增 artifact 时同步考虑 validator、docs、README 和 tests。 +- 修改输出合同相关代码后，优先补充 `tests/integration/test_validator.py` 或相应 workflow 测试。 +- VLM 只能产生结构化 JSON、布局建议、审查结果或 prompt 计划；不要让 VLM 直接写任意 PowerPoint 代码。 +- 真实 API 失败时不要静默切换到 placeholder 或 vector-only；应明确失败或让调用者选择 fallback。 +- 保持 labels、formulas、arrows、controls 在 PPT 层可编辑，图像资产只承载局部视觉块。 +- Presentations QA 是检查层，不是编译权威；不要让它反向改写 RFS 的 PPTX 合同。 + +## 长期结构门禁（防止代码结构退化） + +每次修改前必须先阅读 `docs/code-structure.md`，并按其中的 canonical +归属落代码。新业务逻辑只能放入以下生产包： + +- `rfs/contracts`：稳定语义合同； +- `rfs/analysis`：论文、参考图、OCR、布局和控件分析； +- `rfs/planning`：设计、布局和语义规划； +- `rfs/generation`：候选图、槽资产和资产策略； +- `rfs/composition`：figure program、PPTX、预览与导出； +- `rfs/evaluation`：validator、质量检查和 benchmark； +- `rfs/providers`：VLM/API、模型选择和重试； +- `rfs/workflows`：跨阶段产品编排； +- `rfs/professional`：controlled professional DSL。 + +强制规则： + +1. 根目录历史模块（如 `rfs.vlm_client`、`rfs.validator`、`rfs.editable_rebuild`、`rfs.professional_rebuild`）仅用于兼容导入；**禁止**新增业务逻辑、第三套实现或新的生产 import。 +2. `rfs/cli.py` 只负责总 parser、dispatch 和结果输出；新命令参数注册进入 `rfs/cli_commands/`，业务逻辑进入 workflow/领域包。 +3. 依赖方向必须保持为 `cli -> workflows -> contracts/analysis/planning/generation/composition/evaluation/providers`；禁止合同、编译或评估层反向依赖 CLI/workflow；评估层只能检查和报告，不能偷偷修写生产合同。 +4. 外部 API 调用进入 `rfs/providers`；实验代码仅进入 `rfs/coevolution`、`rfs/judge_training` 或 `experiments/`，不得被默认产品 workflow 隐式导入。 +5. 新文件必须有模块级 docstring、唯一职责和对应测试；同一职责已有 canonical 实现时，必须扩展它，不得复制一套实现。 +6. 单个改动不得混合“文件移动/结构迁移、算法重写、CLI/输出合同变更”。超过约 500 行的模块或约 100 行的函数，应优先拆分或在 commit 中说明必要性。 +7. 修改后至少运行： + +```powershell +python -m compileall -q rfs +python -m unittest discover -s tests -q +rfs doctor --json +``` + +涉及 CLI、兼容层、合同或 workflow 时，必须同时运行相应的 parser/compatibility/integration 测试。不得删除或绕过 `tests/unit/test_package_boundaries.py` 中的结构门禁；如确需调整门禁，必须同步更新 `docs/code-structure.md` 并在 commit 说明原因。 + +## 约定 +1. 每次进行修改后都要提交一次commit，提交简介简单概括本次修改内容，并且给出commit ID，目的是保证修改不满意可退回到上一次修改。 +2. 每次拉取新的代码先更新 `AGENTS.md` 的快速理解项目部分。 +3. 运行项目时在虚拟环境里运行，API在.env文件中，需要手动注入。 +4. 每次拉取或合并代码时，如果出现冲突，不要自行保守合并；先列出冲突文件、双方差异和推荐选项，询问 Sen-illion 后再决定保留哪个实现或如何组合。 +5. 如果有同时修改的内容，一次commit只要提交本次自己修改的内容，不要碰其他的修改。 diff --git a/README.md b/README.md index cad8544..de75792 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,14 @@ ## Summary -ResearchFigureStudio is a PPTX-first research figure generation pipeline for paper-grounded, reference-guided scientific framework figures. It turns a paper plus a user-provided visual reference image into an editable PowerPoint composition assembled from many slot-level image assets, not one flattened full-diagram bitmap. +ResearchFigureStudio is a paper-to-editable-PPT scientific figure engine and Codex plugin. The primary route uses an evidence-grounded paper contract to ask Image2 for one complete scientific figure, including visible text, arrows, modules, and illustrations, then reconstructs that image into an editable PowerPoint composition. The current workflow is optimized for AI/ML/NLP system figures: - paper-grounded concept extraction -- reference-primary geometry, style, color, and flow alignment -- 25-50 non-arrow image slots -- `slot_visual_spec.json` for dense mini-scene/image-block planning +- complete whole-figure Image2 generation with exact visible terminology and directed relations +- VLM review of scientific faithfulness, text, topology, information density, and visual quality +- image-to-PPT reconstruction of panels, text, assets, and connectors - AutoFigure-inspired control candidates and overlays for arrow/source-target binding - reference-preserving arrow styling/routing reports for softer editable PPT connectors - reference-constrained orthogonal fallback routing for missing or explicitly fallback-allowed connectors @@ -17,22 +17,21 @@ The current workflow is optimized for AI/ML/NLP system figures: - multi-candidate image generation through placeholder, Gemini, or Yunwu image2-compatible APIs - deterministic PPTX composition with editable labels, panels, arrows, connectors, and formulas - optional Presentations-plugin QA for importing/rendering/inspecting the PPTX without mutating it -- strict validation for no single full diagram, no semantic crop, no vector-only fallback, low blank space, and non-trivial image-block complexity +- optional paper-native multi-asset experiments for comparison, not the default product route This repository does not include API keys, papers, reference images, generated outputs, or local run artifacts. ## Current Capability Boundary ResearchFigureStudio is an early engineering prototype. Its current practical -ability is limited to placing many small generated image blocks into precise -PowerPoint positions, then keeping surrounding labels, arrows, panels, formulas, -and grouping elements editable in PPTX. +route can generate a complete paper figure through Image2 and then reconstruct +the result into editable PowerPoint layers. The generated reference image remains +the visual source of truth for the reconstruction stage. -It does **not** yet solve the harder goal of generating truly publication-grade, -fully editable scientific figures end to end. The image blocks themselves are -still raster assets, not editable scientific vector objects. The system can -produce a PPTX composition that is easier to manually revise, but it should not -be treated as a finished top-tier-paper figure generator. +It does **not** yet guarantee perfect OCR, connector recovery, or fully editable +scientific vector objects for every figure. The system can produce a PPTX that +is easier to revise, but it should not yet be treated as a universally finished +top-tier-paper figure generator. ## Help Wanted @@ -77,14 +76,86 @@ python -m pip install -e ".[ocr]" Windows with PowerPoint installed gives the best PPTX/PDF/PNG export path. The Python package itself can still generate and validate most intermediate artifacts without external image APIs when using `--asset-mode placeholder`. -## Paper To Image Without PPTX +## Paper To Editable PPTX + +The main product route is complete-image generation followed by precise reconstruction: + +```powershell +rfs paper-to-editable ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_editable" ` + --json +``` + +This first prepares an evidence-grounded semantic contract under `paper_to_image/`. Image2 then generates the complete figure as one raster image, including visible terminology, arrows, modules, and visual explanations. The approved `selected_image.png` is passed to the image-to-PPT reconstruction stage under `editable/`, where OCR, layout parsing, connector localization, and the paper contract recover editable structure. + +`--editable-route rebuild` is the default and primary route. Use `--positive-reference` when supplied figures should influence style or layout. `--editable-route framework` remains an optional multi-asset experiment and is not the target product behavior. + +The first uncached Image2 generation and VLM review may take minutes. Contract, candidate, and reconstruction caches should be reused when their content fingerprints remain valid. + +For offline engineering validation only, add `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview`. This does not create a production-approved result. + +## Fast Paper Contract And Optional Editable PPTX + +For the fast “paper to evidence-grounded framework prompt” path, without image generation: + +```powershell +rfs fast-framework-prompt ` + --paper "C:\path\paper.pdf" ` + --out "output\fast_result" ` + --deadline 180 ` + --json +``` + +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and raw semantic plans are cached separately by paper hash. Older compiled semantic caches are intentionally not reused because normalizing them again can duplicate or mutate relations; each affected paper needs one fresh VLM compilation, after which the versioned raw-plan cache is stable and repeatable. Every accepted cached string is passed through the current Unicode/mojibake repair before new artifacts are written. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. + +Add `--editable-ppt` to compile the same evidence-backed semantic contract directly into `editable_composition.pptx` without image generation. Every visible module is a native PowerPoint shape with editable text, every relation is a native connector, and `figure_program.json`, `semantic_ppt_report.json`, and `composition_quality_report.json` remain available for auditing. This direct route is intended for fast structural drafts; generated icons or richer visual assets can be added later without changing the scientific graph. + +Add `--verify-ppt-roundtrip` when the editable draft must be checked end to end. It implies `--editable-ppt`, exports the deck through PowerPoint, and writes `ppt_roundtrip_report.json` with editable label/node/connector coverage, flattened-picture area, render status, and canvas-ratio drift. The default fast path leaves this off because launching PowerPoint usually adds roughly 10-20 seconds. + +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean headings plus localized Figure/Table captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, OCR render DPI, and spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently at 84 DPI and filters short vertical margin artifacts. The base installation includes `wordninja` for conservative English spacing recovery, while requiring multiple English anchors so non-English Latin words are not split accidentally. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. + +Run a repeatable fast suite with provider, cache, recall, and timing aggregation: + +```powershell +rfs benchmark fast-suite --root benchmarks --out output\benchmarks\fast-suite --planner-mode heuristic --json +``` + +Run the generated multi-layout PDF extraction stress suite independently of paper semantics: + +```powershell +rfs benchmark pdf-suite --out output\benchmarks\pdf-extraction --ocr-engine auto --json +``` + +It generates 10 deterministic parser cases covering columns, typography, rotation, repeated margins, CJK and Spanish text, hyphenation, formulas, and structured tables. A real OCR engine expands the suite to 16 cases with mixed, skewed, degraded, formula/table, and Spanish scans. Each case writes its fixture, rendered preview, document model, extraction report, assertions, and timing so reading-order or OCR regressions are reproducible without committing third-party PDFs. Use `paper-to-image` when the required endpoint is a generated raster framework -figure rather than an editable PowerPoint file. The production route performs a +figure, with an optional native editable PowerPoint structure layer. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts positive references into content-free architecture templates, selects a template, -renders `layout_blueprint.png`, and uses Image2 edit to create and review three -candidates. It never invokes the PPTX compiler. + renders `layout_blueprint.png`, and uses direct Image2 complete-figure generation to create and review three +candidates. For evidence-grounded contracts with 2-16 visible nodes, a generic +semantic compiler now writes ranked node boxes, distinct input/output ports, +global obstacle-aware orthogonal paths, branch lanes, and outer feedback loops + into `layout_blueprint.json.semantic_plan`. The semantic blueprint is embedded in + the generation Prompt, and Image2 is responsible for the final visible text, arrows, modules, +and visual content in `candidate_XX.png`. The critic checks the generated text +and connector topology against the paper contract. Self-Refine feedback and SAM-style dense +multi-frame contracts retain stricter specialized layouts. The command never +adds programmatic text or arrows after generation. `overlay_spec.json` is audit +metadata and a downstream image-to-PPT reconstruction contract only. + +The contract normalizer preserves semantic roles instead of checking labels alone. Explicit provenance clauses such as `CFP and OCT from MEH-MIDAS and public datasets` compile to `data source -> modality -> method`; multi-source phrases collapse into one evidence-backed source group; explicit internal/external evaluation outputs are restored; generic labels such as `foundation model` are replaced by an explicitly named paper model when the evidence provides the alias; and named techniques such as `masked autoencoder` become required innovations. Paper-contract caching is independent of canvas ratio and display-language preferences, so changing 16:9 to 3:2 cannot select a different scientific contract. + +Production Image-2 review rejects a neat but empty text-only flowchart. The critic checks entity-role mismatches, focused provenance topology, mechanism visualization, visual information density, and publication polish. A deterministic blueprint-enrichment gate requires at least `0.10` active-region change beyond the bare blueprint. The detailed-tile gate defaults to `0.80`, with `0.75` for feedback figures whose functional return-loop corridors otherwise receive a false whitespace penalty. Configure both gates with `RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO` and `RFS_MIN_ACTIVE_DETAIL_TILE_RATIO`. Density is saturated once the applicable gate passes: candidate selection and repair no longer reward packing an already adequate figure more tightly, and prompts forbid extra modules, charts, labels, or decorations added only to fill space. Large flat cards still do not count as useful content. + +Density failures now include a 24x16 active-region detail grid plus per-semantic-node 4x4 density measurements. Sparse named modules are copied into `repair_regions`, so localized Image-2 edits can enrich the actual empty cards instead of receiving only a generic whole-canvas instruction. Review cache versioning invalidates older reports whenever this diagnostic contract changes. + +Branch contracts now promote evidence-backed classification, regression, mask, prediction, and similar parallel leaf heads to one consistent output role, remove ancestor shortcuts that bypass the shared branch point, and merge duplicate innovation labels into the existing node instead of drawing a second node. Feedback blueprints anchor nested feedback connectors at the actual inner artifact rather than the outer container. Self-Refine candidates automatically run a focused topology judge in parallel; that judge sees only the generated candidate, avoiding answer leakage from the layout blueprint, and explicitly distinguishes `Self-Feedback -> REFINE` from the common wrong edges `FEEDBACK -> REFINE` and `Self-Feedback -> Refined output`. Visual enrichment and detailed-tile density are measured inside the blueprint's active figure region, so sparse box-only diagrams and flat color-filled panels fail even when their outer geometry is correct. + +Learned class/query tokens and positional embeddings that feed an internal model are normalized as conditioning modules rather than external inputs. A generic leaf such as `Head` is collapsed when it merely duplicates its upstream `MLP Head` and a concrete sibling output exists; the target overview figure's exact visible output label is preferred. Topology is inferred again after late repairs so removing a false duplicate branch can correctly restore a linear contract. Placeholder runs explicitly return `production_ready: false`, `engineering_only: true`, and `render_owner: engineering_placeholder`; they never claim that Image-2 produced the preview. + +Dense multi-frame overview normalization tolerates descriptive paper wording such as `heavyweight image encoder` and `lightweight mask decoder`. Those phrases still map to the canonical Figure 1 nodes `Image Encoder` and `Mask Decoder`, allowing the overview compiler to remove lower-level implementation details instead of misclassifying the whole contract as a generic branch graph. Offline engineering validation: @@ -115,6 +186,8 @@ rfs paper-to-image ` --positive-reference "C:\path\reference2.png" ` --template auto ` --asset-mode image2 ` + --image-init-mode topology ` + --image-timeout 45 ` --candidates 3 ` --aspect-ratio auto ` --review-mode vlm ` @@ -125,18 +198,98 @@ rfs paper-to-image ` The main outputs are `paper_review.json`, `review_coverage_report.json`, `domain_profile.json`, `template_profiles/`, `selected_template.json`, -`layout_blueprint.png`, `figure_specification.json`, `image_prompt.txt`, -`image2_request_manifest.json`, four critic reports, the `candidates/` -directory, and production-only `selected_image.png`. +`layout_blueprint.png`, `layout_blueprint.json`, `overlay_spec.json`, +`figure_specification.json`, `image_prompt.txt`, +`image2_request_manifest.json`, the general and focused-topology critic reports, the `candidates/` +directory, and production-only `selected_image.png`. `overlay_spec.json` is not +rendered onto the primary image; it records what Image2 must already have drawn +and supports later image-to-PPT reconstruction and benchmark auditing. + +To repair a reviewed failed candidate without paying for a new initial candidate +batch, reuse it as the starting image: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_repair" ` + --asset-mode image2 ` + --review-mode vlm ` + --repair-source "output\paper_to_image\candidates\candidate_01.png" ` + --repair-rounds 1 ` + --image-retries 0 ` + --json +``` -Reference-conditioned production generation requires an Image2 edit endpoint. -It defaults to `/images/edits` and may be overridden with -`RFS_IMAGE_EDIT_URL`. No API key value is written to output artifacts or logs. +The reused source is rechecked against the current paper contract and Prompt. +Only a failing source triggers one localized Image2 edit. This is useful after +fixing a contract or Prompt bug because it preserves correct regions and avoids +regenerating the initial image from scratch. + +Production generation defaults to topology-aware routing. Feedback-loop, +linear, branch, multimodal, and dense-multiframe figures use `--image-init-mode edit` internally +so gpt-image-2 fully redraws their benchmarked semantic blueprints. Linear, +branch, multimodal, and dense-multiframe blueprints include paper-grounded mechanism cues while +keeping all final text, arrows, and pixels owned by gpt-image-2. Other topologies continue to use +direct generation until their own multi-seed benchmarks justify blueprint editing. Use +`--image-init-mode generate` or `edit` to force either route, or `auto` to fall +back to direct generation after an edit timeout. Endpoints may be overridden with +`RFS_IMAGE_GENERATION_URL` and `RFS_IMAGE_EDIT_URL`. No API key value is written to output artifacts or logs. If a key was pasted into a chat, issue, or terminal transcript, revoke it and use a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, template contract, production gates, and failure behavior. +Automatic selection includes dedicated `dense-multiframe`, `multimodal`, +`feedback`, and `branch` templates. Dense task/model/data-engine overviews use +`dense-multiframe`; multiple modality inputs converging into one shared +representation use `multimodal`; +Compact generation → feedback → refinement loops use `feedback`; shared-trunk +systems with parallel prediction heads use `branch`; true search-tree systems +use `arbor`; simple sequential systems use `linear`. Feedback, branch, +multimodal, and dense candidates also run a focused connector judge that +verifies visible arrow endpoints and rejects shortcuts that bypass required +modules. The judge understands explicit containment: an evidence-supported +repeatable shared component inside a labeled operation container may implement +that container's outgoing relation without being treated as an invented edge. + +The default production critic uses one integrated VLM call for exact text, +paper alignment, aesthetics, and topology. Relations include structured source +side, target side, and arrowhead-contact fields. If a complex feedback or +cross-panel edge lacks explicit endpoint evidence, one short focused topology +fallback runs; set `RFS_PAPER_TO_IMAGE_FOCUSED_TOPOLOGY=1` to always run the +independent pass. Timeouts are controlled by +`RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT`. OCR returns one record per physical text +occurrence, allowing deterministic duplicate-label checks. Use `--repair-source` +to re-review or locally repair an existing candidate without regenerating correct +regions. Every reviewed run also writes +`paper_image_comparison_report.json`, which consolidates text, paper evidence, +topology, density, aesthetics, blocking issues, and the next repair action. +The CLI uses a 45-second per-candidate Image-2 timeout through `--image-timeout`. +Low-level edit and generation defaults can be overridden with `RFS_IMAGE2_TIMEOUT` +and `RFS_IMAGE2_GENERATION_TIMEOUT`. A network timeout is not blindly retried; retryable HTTP +or provider failures receive at most the configured `--image-retries` attempts, +whose default is one. + +Set `RFS_IMAGE_QUALITY` to `low`, `medium`, or `high` to request an explicit +Image2 quality tier; leaving it unset preserves the provider default. The value +is recorded in generation artifacts, and every tier must pass the same OCR, +paper-evidence, topology, information-density, and aesthetic gates. + +Use `--candidates 3` for a minimum stability audit. Independent candidates are +generated and reviewed concurrently, and `stability_report.json` records the +production pass rate, mean score, worst-case score, standard deviation, and +failure-mode counts. Benchmark scoring consumes this report automatically +instead of treating one selected best image as evidence of stability. + +Every visible scientific label is checked against the normalized contract +whitelist. Paper-supported but out-of-scope details still fail the overview if +they were intentionally omitted from the figure contract. For interrupted +stability runs, rerun the same output directory with `--resume-candidates`; +existing images are re-reviewed and only missing candidates are generated. +Three-seed stability runs may automatically replace one provider-failed seed, +without regenerating successful candidates. + ## Offline Smoke Test Use placeholder assets to validate the local pipeline without calling any API: @@ -168,6 +321,38 @@ rfs validate --out "output\demo_placeholder" --json `output/` is intentionally ignored by Git. +## Optional Paper-Native Multi-Asset Experiment + +When `fast-framework-prompt` or `paper-to-image` has already produced a validated +`figure_specification.json`, pass that run directory directly to +`make-framework`. In paper-native mode no complete reference image is required. +The semantic contract owns every visible module, relation, label, and visual +facet; the generated seed owns only geometry, density, and palette. + +```powershell +rfs make-framework ` + --paper "C:\path\paper.pdf" ` + --paper-contract-dir "output\paper_to_image\run" ` + --out "output\editable\contract_driven" ` + --slot-count 25 ` + --slot-source paper-native ` + --complexity-profile paper-native-dense ` + --asset-mode placeholder ` + --candidates-per-slot 1 ` + --asset-workers 3 ` + --locator-mode heuristic ` + --control-localizer-mode off ` + --prompt-plan-mode heuristic ` + --text-extractor-mode heuristic ` + --ocr-engine off ` + --no-export ` + --json +``` + +This `make-framework --slot-source paper-native` route is retained for research +comparison only. The product default is `paper-to-editable --editable-route +rebuild`, where Image2 generates the complete figure before reconstruction. + ## Real VLM + Image Generation Set API credentials only through environment variables. Do not write keys into source files. @@ -224,6 +409,7 @@ rfs rebuild-editable-pro ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild_pro" ` --asset-mode api ` + --asset-policy smart-api ` --repair-rounds 2 ` --export-preview ``` @@ -235,14 +421,21 @@ write `professional_gap_report.json` against a specialized-script output. Use `--repair-mode vlm` only when you want the VLM to apply constrained DSL patches after preview comparison. -Lower-cost mode, using VLM structure planning but keeping reference crops as -slot assets: +In `smart-api` policy, reference crops are only used as image-generation context. +They are not inserted as final PPT assets. Text-like slots are filtered into +editable text, duplicate complex icons reuse one generated asset, and API +failures fall back to placeholders instead of reference crops. + +Lower-cost structure check. With `smart-api`, this does not keep reference crops +as final assets; it writes placeholders for visual slots while still producing +the DSL, overlays, and reports: ```powershell rfs rebuild-editable ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild_crop" ` --asset-mode crop ` + --asset-policy smart-api ` --layout-mode hybrid ` --control-mode hybrid ``` @@ -322,14 +515,10 @@ Use lower worker counts if your API provider rate-limits requests. ## Workflow ```text -input archive -> paper brief -> reference_geometry.json/reference_control_candidates.json -> -slot_overlay.png/reference_control_overlay.png -> reference_controls.json -> -arrow_style_profile.json/selected_arrow_routes.json/arrow_quality_report.json -> -reference_style_profile.json/style_sheet.md -> layout_plan.json -> figure_program.json -> -slot_visual_spec.json -> reference_slot_prompt_brief.json -> slot_prompt_plan.json -> -multi-candidate slot assets -> asset_quality_report.json -> asset_complexity_report.json -> -asset_visual_review.json/contact sheets -> editable_composition.pptx -> PDF/PNG export -> -visual_critic_iter_0.json -> critic_report.md -> validation +paper -> evidence-grounded semantic contract -> full layout blueprint and Prompt -> +Image2 complete figure (text + arrows + modules + illustrations) -> VLM review/repair -> +approved selected_image.png -> image-to-PPT reconstruction -> editable_composition.pptx -> +PDF/PNG export -> validation ``` Optional Presentations-plugin QA can run after validation: @@ -351,6 +540,9 @@ User content hierarchy, emphasis, aesthetic, reference-image, and A/B preference Key rules: +- Image2 owns the complete visible raster figure in the primary route, including text and arrows. +- The paper semantic contract constrains which labels, modules, relations, numbers, and claims Image2 may render. +- The reconstruction stage must follow the approved complete image rather than silently redesigning it. - The reference image is the source of truth for layout, local visual object choice, color, visual rhythm, and arrow logic when `--slot-source reference-primary` is used. - The paper provides scientific terminology and concept mapping; it should not override the reference image into a generic template. - Arrows, connector lines, dashed loops, panel frames, labels, formulas, and critical text are PPT editable objects, not image assets. @@ -368,7 +560,7 @@ Optional arrow-beautification pass: rfs make-framework ` --paper "C:\path\paper.pdf" ` --reference "C:\path\reference.png" ` - --out D:\ResearchFigureStudio\output\aesthetic_experiment ` + --out .\output\aesthetic_experiment ` --slot-count 40 ` --slot-source reference-primary ` --control-localizer-mode hybrid ` @@ -458,7 +650,7 @@ Run validation: ```powershell rfs validate --out "output\paper_reference_image2" --json -python codex-skills\research-figure-making\scripts\validate_framework_outputs.py "output\paper_reference_image2" +python skills\research-figure-studio\scripts\validate_framework_outputs.py "output\paper_reference_image2" ``` ## Codex Skill @@ -466,15 +658,15 @@ python codex-skills\research-figure-making\scripts\validate_framework_outputs.py This repository includes the Codex skill under: ```text -codex-skills/research-figure-making +skills/research-figure-studio ``` To install it locally into Codex: ```powershell -$dst = Join-Path $env:USERPROFILE ".codex\skills\research-figure-making" +$dst = Join-Path $env:USERPROFILE ".codex\skills\research-figure-studio" if (Test-Path $dst) { Remove-Item -Recurse -Force $dst } -Copy-Item -Recurse "codex-skills\research-figure-making" $dst +Copy-Item -Recurse "skills\research-figure-studio" $dst ``` The skill documents the full research-figure workflow and includes the standalone framework-output validator. @@ -484,7 +676,7 @@ The skill documents the full research-figure workflow and includes the standalon ```powershell python -m compileall -q rfs python -m unittest discover -s tests -q -python -m py_compile codex-skills\research-figure-making\scripts\validate_framework_outputs.py +python -m py_compile skills\research-figure-studio\scripts\validate_framework_outputs.py ``` ## Repository Hygiene diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..9433360 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,70 @@ +# ResearchFigureStudio Benchmarks + +ResearchFigureStudio uses two independent product benchmark suites plus a generated PDF extraction stress suite, so failures can be attributed to paper parsing, reference-image generation, or editable reconstruction. + +## Suites + +### `paper-to-image` + +Measures scientific faithfulness, terminology, relation correctness, information coverage and density, clarity, aesthetics, reference compliance, hallucinations, and stability across repeated generations. + +Scientific errors are hard failures and cannot be offset by aesthetics. + +### `image-to-ppt` + +Measures rendered visual fidelity, object and relation reconstruction, text alignment, editable PowerPoint structure, anti-cheating rules, visual blockers, and eventually edit-mutation behavior. + +A full-slide copy of the reference image is a hard failure even if pixel similarity is perfect. + +### Generated PDF extraction stress suite + +`benchmark pdf-suite` creates deterministic native two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs at runtime. It validates section boundaries, cross-column reading order, displayed coordinates, caption recovery, local OCR scheduling, English OCR spacing recovery, and elapsed time. `--ocr-engine auto` adds a real installed-OCR probe after the deterministic adapter-backed tier. + +## Commands + +```powershell +rfs benchmark list --root benchmarks --json +rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipeline --json +rfs benchmark fetch --case benchmarks/paper-to-image/cases/101_vit_linear --json +rfs benchmark fast --case benchmarks/paper-to-image/cases/101_vit_linear --out output/benchmarks/vit_fast --planner-mode heuristic --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast_suite --planner-mode heuristic --json +rfs benchmark pdf-suite --out output/benchmarks/pdf_extraction --ocr-engine auto --json +rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json +rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json +``` + +## Case policy + +Each case contains `case.json` plus suite-specific human-authored ground truth. Synthetic cases may be committed. Real papers and figures should only be committed when redistribution is permitted; otherwise use local paths or a private benchmark data repository. + +Generated runs and reports belong under `output/benchmarks/` and are not source fixtures. + +Real-paper cases commit `source.json` and human-authored Ground Truth, while `benchmark fetch` downloads the PDF into an ignored local `inputs/` directory. This keeps the public repository reproducible without redistributing publisher files. + +The committed unseen-generalization set spans vision, multimodal learning, and NLP: ViT, Mask R-CNN, Self-Refine, ImageBind, SAM, DETR, CLIP, NeRF, Transformer, BERT, and retrieval-augmented generation. The NLP cases specifically exercise short overview captions, method-text recovery, embedding summation, pre-training/fine-tuning separation, and retrieval-conditioned generation. Local image-only scan regressions remain under ignored `tmp/pdfs/` and `output/pdf/`; do not commit publisher PDFs or generated OCR artifacts. + +Any locally available PDF benchmark can also be tested as an image-only degradation without committing a publisher scan: + +```powershell +rfs benchmark fast-suite ` + --root benchmarks ` + --out output/benchmarks/scanned ` + --case-id 106_detr_set_prediction ` + --case-id 110_bert_pretrain_finetune ` + --rasterize-dpi 144 ` + --ocr-engine rapidocr ` + --planner-mode heuristic ` + --deadline 180 ` + --json +``` + +The generated image-only PDFs stay under the benchmark output directory. Score them with the same human-authored entity, relation, and forbidden-content contracts as the native PDFs. + +`benchmark fast-suite` writes `fast_suite_report.json` with per-case results and aggregate entity/relation recall, forbidden content, cache hit rates, provider success/retry counts, failure categories, and stage timings. + +## Evaluation tiers + +- Offline contract tier: placeholder assets, deterministic validation, CI-safe. +- Fast planning tier: entity recall, relation recall, forbidden content, evidence grounding, deadline, and cache behavior without image generation. +- Production quality tier: real VLM/image models, frozen judge, repeated seeds, and human calibration. +- Human audit tier: blinded pairwise ratings for aesthetics, clarity, and information density. diff --git a/benchmarks/image-to-ppt/annotation_template.json b/benchmarks/image-to-ppt/annotation_template.json new file mode 100644 index 0000000..7edcc82 --- /dev/null +++ b/benchmarks/image-to-ppt/annotation_template.json @@ -0,0 +1,33 @@ +{ + "summary": "Independent human annotation template for image-to-PPT reconstruction.", + "case_id": "", + "canvas": {"width_px": 0, "height_px": 0}, + "objects": [ + { + "id": "object_01", + "type": "panel|card|asset|text|legend|formula", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0}, + "text": "", + "parent_id": null, + "z_index": 0, + "must_be_editable": true + } + ], + "relations": [ + { + "id": "relation_01", + "source": "object_01", + "target": "object_02", + "type": "data_flow", + "path_percent": [[0.0, 0.0], [0.0, 0.0]], + "must_be_editable": true + } + ], + "groups": [], + "mutation_tests": [ + {"operation": "replace_text", "target_id": "", "value": "Edited label"}, + {"operation": "move_object", "target_id": "", "delta_percent": {"x": 0.05, "y": 0.0}}, + {"operation": "change_fill", "target_id": "", "value": "#FFAA00"}, + {"operation": "delete_relation", "target_id": ""} + ] +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json b/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json new file mode 100644 index 0000000..0763927 --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json @@ -0,0 +1,26 @@ +{ + "summary": "Synthetic three-stage image-to-editable-PPT reconstruction case.", + "case_id": "001_three_stage_layout", + "suite": "image-to-ppt", + "tier": "offline-contract", + "reference_image": "reference.ppm", + "expected_objects": "expected_objects.json", + "preview": "rebuild_preview.png", + "pptx": "editable_composition.pptx", + "run_config": { + "asset_mode": "placeholder", + "asset_policy": "smart-api", + "text_mode": "off", + "layout_mode": "heuristic", + "control_mode": "heuristic", + "design_plan_mode": "heuristic", + "ocr_engine": "off" + }, + "thresholds": { + "object_coverage": 0.8, + "relation_coverage": 1.0, + "editability_score": 0.66, + "full_slide_image_count": 0, + "blocking_visual_issue_count": 0 + } +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json b/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json new file mode 100644 index 0000000..91a070f --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json @@ -0,0 +1,14 @@ +{ + "summary": "Human-authored object and relation annotations for a three-stage layout.", + "objects": [ + {"id": "input", "type": "card", "bbox_percent": {"x": 0.05, "y": 0.25, "w": 0.22, "h": 0.45}}, + {"id": "method", "type": "card", "bbox_percent": {"x": 0.39, "y": 0.20, "w": 0.22, "h": 0.55}}, + {"id": "output", "type": "card", "bbox_percent": {"x": 0.73, "y": 0.25, "w": 0.22, "h": 0.45}} + ], + "relations": [ + {"source": "input", "target": "method", "type": "data_flow"}, + {"source": "method", "target": "output", "type": "data_flow"} + ], + "required_editable_types": ["card", "text", "connector"], + "forbid_full_slide_reference_image": true +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm b/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm new file mode 100644 index 0000000..de7242c --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm @@ -0,0 +1,6 @@ +P3 +6 3 +255 +250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 +80 150 210 80 150 210 250 250 250 70 180 160 70 180 160 220 150 80 +80 150 210 80 150 210 250 250 250 70 180 160 70 180 160 220 150 80 diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json new file mode 100644 index 0000000..0e31eb3 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json @@ -0,0 +1,26 @@ +{ + "summary": "Synthetic three-stage pipeline for scientific and visual benchmark protocol validation.", + "case_id": "001_linear_pipeline", + "suite": "paper-to-image", + "tier": "offline-contract", + "paper": "paper.md", + "expected_semantics": "expected_semantics.json", + "preferences": "preferences.json", + "positive_references": [], + "negative_references": [], + "run_config": { + "planner_mode": "heuristic", + "image_asset_mode": "placeholder", + "image_candidates": 2, + "review_mode": "heuristic", + "aspect_ratio": "16:9", + "ocr_engine": "off" + }, + "thresholds": { + "entity_recall": 1.0, + "relation_recall": 1.0, + "exact_label_rate": 1.0, + "hallucination_count": 0, + "forbidden_content_count": 0 + } +} diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json new file mode 100644 index 0000000..05d6dcc --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json @@ -0,0 +1,17 @@ +{ + "summary": "Human-authored semantic ground truth for the ExampleFlow figure.", + "entities": [ + {"id": "raw_document", "label": "Raw Document", "role": "input", "required": true}, + {"id": "document_encoding", "label": "Document Encoding", "role": "module", "required": true}, + {"id": "evidence_aggregation", "label": "Evidence Aggregation", "role": "module", "required": true}, + {"id": "final_prediction", "label": "Final Prediction", "role": "output", "required": true} + ], + "relations": [ + {"source": "raw_document", "target": "document_encoding", "type": "data_flow", "required": true}, + {"source": "document_encoding", "target": "evidence_aggregation", "type": "data_flow", "required": true}, + {"source": "evidence_aggregation", "target": "final_prediction", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Agent", "Memory", "Retriever", "Knowledge Base"], + "required_innovation": "Evidence Aggregation", + "expected_reading_order": ["raw_document", "document_encoding", "evidence_aggregation", "final_prediction"] +} diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md b/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md new file mode 100644 index 0000000..352fc23 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md @@ -0,0 +1,11 @@ +# ExampleFlow: A Three-Stage Evidence Processing Framework + +## Abstract + +ExampleFlow transforms a Raw Document into a Final Prediction through Document Encoding and Evidence Aggregation. + +## Method + +The system takes a Raw Document as input. Document Encoding converts the Raw Document into contextual representations. Evidence Aggregation consumes the contextual representations and produces a Final Prediction. + +The processing order is Raw Document, Document Encoding, Evidence Aggregation, and Final Prediction. diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json new file mode 100644 index 0000000..594d0a5 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json @@ -0,0 +1,9 @@ +{ + "summary": "Visual preferences separated from scientific ground truth.", + "aspect_ratio": "16:9", + "language": "English", + "preferred_flow": "left_to_right", + "style_description": "clean academic framework figure", + "preferred_palette": ["blue", "teal", "warm orange accent"], + "avoid": ["commercial poster", "cyberpunk", "generic dashboard"] +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/case.json b/benchmarks/paper-to-image/cases/101_vit_linear/case.json new file mode 100644 index 0000000..e12472a --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ICLR 2021 Vision Transformer case for a clean linear architecture figure.", + "case_id": "101_vit_linear", + "suite": "paper-to-image", + "tier": "real-paper-offline-contract", + "topology": "linear_pipeline", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json new file mode 100644 index 0000000..3b65938 --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json @@ -0,0 +1,25 @@ +{ + "summary": "Human-authored semantic contract for the Vision Transformer architecture.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["2D Image", "Image"], "role": "input", "required": true}, + {"id": "image_patches", "label": "Image Patches", "aliases": ["Patches"], "role": "module", "required": true}, + {"id": "linear_projection", "label": "Linear Projection of Flattened Patches", "aliases": ["Linear Projection"], "role": "module", "required": true}, + {"id": "position_embedding", "label": "Position Embedding", "role": "module", "required": true}, + {"id": "class_token", "label": "Class Token", "aliases": ["Class Embedding", "[class] Embedding"], "role": "module", "required": true}, + {"id": "transformer_encoder", "label": "Transformer Encoder", "role": "module", "required": true}, + {"id": "mlp_head", "label": "MLP Head", "role": "module", "required": true}, + {"id": "class_prediction", "label": "Class Prediction", "aliases": ["Image Classification Predictions", "Classification Predictions", "Class Label"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "image_patches", "type": "data_flow", "required": true}, + {"source": "image_patches", "target": "linear_projection", "type": "data_flow", "required": true}, + {"source": "linear_projection", "target": "transformer_encoder", "type": "data_flow", "required": true}, + {"source": "position_embedding", "target": "transformer_encoder", "type": "conditioning", "required": true}, + {"source": "class_token", "target": "transformer_encoder", "type": "conditioning", "required": true}, + {"source": "transformer_encoder", "target": "mlp_head", "type": "data_flow", "required": true}, + {"source": "mlp_head", "target": "class_prediction", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["CNN Backbone", "Recurrent Network", "Decoder", "Object Detection Head"], + "expected_reading_order": ["input_image", "image_patches", "linear_projection", "transformer_encoder", "mlp_head", "class_prediction"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 0, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/source.json b/benchmarks/paper-to-image/cases/101_vit_linear/source.json new file mode 100644 index 0000000..0b126fc --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale", + "venue": "ICLR", + "year": 2021, + "arxiv_id": "2010.11929", + "official_url": "https://openreview.net/forum?id=YicbFdNTTy", + "paper_urls": ["https://arxiv.org/pdf/2010.11929", "https://export.arxiv.org/pdf/2010.11929"], + "target_figure": "Figure 1", + "verified_caption": "Model overview. The image is split into fixed-size patches, linearly embedded, combined with position embeddings and processed by a Transformer encoder.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json new file mode 100644 index 0000000..67ba6cb --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json @@ -0,0 +1,14 @@ +{ + "summary": "ICCV 2017 Mask R-CNN case for shared-backbone multi-head branching.", + "case_id": "102_mask_rcnn_branch", + "suite": "paper-to-image", + "tier": "real-paper-offline-contract", + "topology": "branching_architecture", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json new file mode 100644 index 0000000..390a603 --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json @@ -0,0 +1,24 @@ +{ + "summary": "Human-authored semantic contract for Mask R-CNN.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["Image"], "role": "input", "required": true}, + {"id": "backbone", "label": "Backbone", "aliases": ["Backbone Architecture"], "role": "module", "required": true}, + {"id": "region_proposals", "label": "Region Proposals", "aliases": ["Region Proposal Network (RPN)", "RPN", "Proposals"], "role": "module", "required": true}, + {"id": "roi_align", "label": "RoIAlign", "role": "module", "required": true}, + {"id": "classification_head", "label": "Classification", "aliases": ["Class Label", "Classification Head"], "role": "output", "required": true}, + {"id": "box_head", "label": "Bounding-box Regression", "aliases": ["Bounding Box", "Box Branch", "Box Regression"], "role": "output", "required": true}, + {"id": "mask_head", "label": "Mask Branch", "role": "output", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "backbone", "type": "data_flow", "required": true}, + {"source": "backbone", "target": "region_proposals", "type": "data_flow", "required": true}, + {"source": "region_proposals", "target": "roi_align", "type": "data_flow", "required": true}, + {"source": "backbone", "target": "roi_align", "type": "feature_flow", "required": true}, + {"source": "roi_align", "target": "classification_head", "type": "branch", "required": true}, + {"source": "roi_align", "target": "box_head", "type": "branch", "required": true}, + {"source": "roi_align", "target": "mask_head", "type": "branch", "required": true} + ], + "forbidden_labels": ["Semantic Segmentation Only", "Single Output Head", "Transformer Decoder"], + "expected_reading_order": ["input_image", "backbone", "region_proposals", "roi_align"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 3, "convergence_count": 1, "feedback_loop_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json new file mode 100644 index 0000000..06453b2 --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Mask R-CNN", "venue": "ICCV", "year": 2017, "arxiv_id": "1703.06870", + "official_url": "https://openaccess.thecvf.com/content_ICCV_2017/html/He_Mask_R-CNN_ICCV_2017_paper.html", + "paper_urls": ["https://arxiv.org/pdf/1703.06870", "https://export.arxiv.org/pdf/1703.06870"], + "target_figure": "Figure 1", "verified_caption": "The Mask R-CNN framework for instance segmentation.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json new file mode 100644 index 0000000..e75cef8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json @@ -0,0 +1,8 @@ +{ + "summary": "NeurIPS 2023 Self-Refine case for an explicit iterative feedback loop.", + "case_id": "103_self_refine_feedback", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "feedback_loop", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json new file mode 100644 index 0000000..104b0bd --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json @@ -0,0 +1,25 @@ +{ + "summary": "Human-authored semantic contract for Self-Refine.", + "entities": [ + {"id": "task_input", "label": "Input", "aliases": ["Task Prompt", "Task Instruction"], "role": "input", "required": true}, + {"id": "initial_generator", "label": "Generate", "aliases": ["INIT", "Generator"], "role": "module", "required": true}, + {"id": "initial_output", "label": "Initial Output", "role": "module", "required": true}, + {"id": "feedback_module", "label": "Feedback", "aliases": ["FEEDBACK", "Feedback Provider"], "role": "module", "required": true}, + {"id": "feedback_text", "label": "Feedback", "aliases": ["Self-Feedback", "Self-Feedback (NL)"], "role": "control", "required": true}, + {"id": "refinement_module", "label": "Refine", "aliases": ["ITERATE / REFINE", "Refiner"], "role": "module", "required": true}, + {"id": "refined_output", "label": "Refined Output", "role": "output", "required": true} + ], + "relations": [ + {"source": "task_input", "target": "initial_generator", "type": "data_flow", "required": true}, + {"source": "initial_generator", "target": "initial_output", "type": "data_flow", "required": true}, + {"source": "initial_output", "target": "feedback_module", "type": "evaluation", "required": true}, + {"source": "feedback_module", "target": "feedback_text", "type": "feedback", "required": true}, + {"source": "initial_output", "target": "refinement_module", "type": "revision_input", "required": true}, + {"source": "feedback_text", "target": "refinement_module", "type": "feedback", "required": true}, + {"source": "refinement_module", "target": "refined_output", "type": "data_flow", "required": true}, + {"source": "refined_output", "target": "feedback_module", "type": "feedback_loop", "required": true} + ], + "forbidden_labels": ["Gradient Update", "Human Annotator Required", "External Reward Model", "Supervised Fine-tuning"], + "expected_reading_order": ["task_input", "initial_generator", "initial_output", "feedback_module", "refinement_module", "refined_output"], + "topology_requirements": {"primary_flow": "cyclic", "feedback_loop_count": 1, "iteration_must_be_visible": true} +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json new file mode 100644 index 0000000..b0c34da --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json @@ -0,0 +1,7 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Self-Refine: Iterative Refinement with Self-Feedback", "venue": "NeurIPS", "year": 2023, "arxiv_id": "2303.17651", + "official_url": "https://arxiv.org/abs/2303.17651", "paper_urls": ["https://arxiv.org/pdf/2303.17651", "https://export.arxiv.org/pdf/2303.17651"], + "target_figure": "Figure 1", "verified_caption": "Iterative generation, feedback and refinement without additional supervised training.", + "license_note": "Use the arXiv license shown on the source page; downloaded inputs remain local." +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json new file mode 100644 index 0000000..aa2f8cd --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json @@ -0,0 +1,8 @@ +{ + "summary": "CVPR 2023 ImageBind case for six-modality shared-embedding alignment.", + "case_id": "104_imagebind_multimodal", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "multimodal_system", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json new file mode 100644 index 0000000..49bbfa9 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json @@ -0,0 +1,27 @@ +{ + "summary": "Human-authored semantic contract for ImageBind.", + "entities": [ + {"id": "image", "label": "Image", "role": "input", "required": true}, + {"id": "text", "label": "Text", "role": "input", "required": true}, + {"id": "audio", "label": "Audio", "role": "input", "required": true}, + {"id": "depth", "label": "Depth", "role": "input", "required": true}, + {"id": "thermal", "label": "Thermal", "role": "input", "required": true}, + {"id": "imu", "label": "IMU", "role": "input", "required": true}, + {"id": "modality_encoders", "label": "Modality Encoders", "aliases": ["Transformer-based Encoders for Each Modality", "Encoders for Each Modality"], "role": "module", "required": true}, + {"id": "joint_embedding", "label": "Joint Embedding Space", "aliases": ["Single Shared Joint Embedding Space", "Single Shared Representation Space", "Joint Embedding"], "role": "module", "required": true}, + {"id": "emergent_alignment", "label": "Emergent Cross-modal Alignment", "aliases": ["Emergent Alignment", "Binding Agent", "Binding Property"], "role": "output", "required": true} + ], + "relations": [ + {"source": "image", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "text", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "audio", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "depth", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "thermal", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "imu", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "modality_encoders", "target": "joint_embedding", "type": "alignment", "required": true}, + {"source": "joint_embedding", "target": "emergent_alignment", "type": "enables", "required": true} + ], + "forbidden_labels": ["All Modality Pairs Are Directly Paired", "Single Shared Raw Encoder", "Late Fusion Classifier"], + "expected_reading_order": ["image", "text", "audio", "depth", "thermal", "imu", "modality_encoders", "joint_embedding", "emergent_alignment"], + "topology_requirements": {"primary_flow": "many_to_one_to_many", "input_modality_count": 6, "shared_space_required": true} +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json new file mode 100644 index 0000000..726abe3 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "ImageBind: One Embedding Space To Bind Them All", "venue": "CVPR", "year": 2023, "arxiv_id": "2305.05665", + "official_url": "https://openaccess.thecvf.com/content/CVPR2023/html/Girdhar_ImageBind_One_Embedding_Space_To_Bind_Them_All_CVPR_2023_paper.html", + "paper_urls": ["https://arxiv.org/pdf/2305.05665", "https://export.arxiv.org/pdf/2305.05665"], + "target_figure": "Figure 2", "verified_caption": "Different naturally aligned modality pairs are linked through image data into a common embedding space.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json new file mode 100644 index 0000000..21ca329 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json @@ -0,0 +1,8 @@ +{ + "summary": "ICCV 2023 Segment Anything case for a dense three-part task-model-data-engine overview.", + "case_id": "105_sam_dense_multiframe", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "dense_multi_panel", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json new file mode 100644 index 0000000..86f1af4 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json @@ -0,0 +1,32 @@ +{ + "summary": "Human-authored semantic contract for the Segment Anything overview.", + "entities": [ + {"id": "promptable_task", "label": "Promptable Segmentation Task", "role": "panel", "required": true}, + {"id": "image", "label": "Image", "role": "input", "required": true}, + {"id": "prompt", "label": "Prompt", "aliases": ["Sparse Prompts", "Dense Prompts", "Segmentation Prompt"], "role": "input", "required": true}, + {"id": "valid_mask", "label": "Valid Segmentation Mask", "aliases": ["Valid Masks", "Valid Mask"], "role": "output", "required": true}, + {"id": "sam_model", "label": "Segment Anything Model", "aliases": ["Segmentation Model (SAM)", "SAM"], "role": "panel", "required": true}, + {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, + {"id": "prompt_encoder", "label": "Prompt Encoder", "role": "module", "required": true}, + {"id": "mask_decoder", "label": "Mask Decoder", "role": "module", "required": true}, + {"id": "data_engine", "label": "Data Engine", "role": "panel", "required": true}, + {"id": "assisted_manual", "label": "Assisted-manual", "role": "module", "required": true}, + {"id": "semi_automatic", "label": "Semi-automatic", "role": "module", "required": true}, + {"id": "fully_automatic", "label": "Fully Automatic", "role": "module", "required": true}, + {"id": "sa1b", "label": "SA-1B", "role": "output", "required": true} + ], + "relations": [ + {"source": "image", "target": "image_encoder", "type": "data_flow", "required": true}, + {"source": "prompt", "target": "prompt_encoder", "type": "data_flow", "required": true}, + {"source": "image_encoder", "target": "mask_decoder", "type": "feature_flow", "required": true}, + {"source": "prompt_encoder", "target": "mask_decoder", "type": "conditioning", "required": true}, + {"source": "mask_decoder", "target": "valid_mask", "type": "data_flow", "required": true}, + {"source": "assisted_manual", "target": "semi_automatic", "type": "stage_transition", "required": true}, + {"source": "semi_automatic", "target": "fully_automatic", "type": "stage_transition", "required": true}, + {"source": "fully_automatic", "target": "sa1b", "type": "data_generation", "required": true}, + {"source": "sam_model", "target": "data_engine", "type": "annotation_support", "required": true} + ], + "forbidden_labels": ["Text-to-Image Generation", "Object Detector Only", "Single-stage Manual Annotation", "Language Model", "Class Descriptions", "Contrastive Learning", "Text Embeddings", "Zero-Shot Classifier"], + "expected_panels": ["promptable_task", "sam_model", "data_engine"], + "topology_requirements": {"panel_count": 3, "dense_multi_panel": true, "feedback_between_model_and_data_engine": true} +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json new file mode 100644 index 0000000..1065723 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Segment Anything", "venue": "ICCV", "year": 2023, "arxiv_id": "2304.02643", + "official_url": "https://openaccess.thecvf.com/content/ICCV2023/html/Kirillov_Segment_Anything_ICCV_2023_paper.html", + "paper_urls": ["https://arxiv.org/pdf/2304.02643", "https://export.arxiv.org/pdf/2304.02643"], + "target_figure": "Figure 1", "verified_caption": "Three interconnected components: promptable segmentation task, Segment Anything Model and data engine.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json new file mode 100644 index 0000000..94860f8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ECCV 2020 DETR case for learned queries, encoder-decoder flow, branched outputs, and a training-only matching objective.", + "case_id": "106_detr_set_prediction", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json new file mode 100644 index 0000000..56d332d --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for DETR's end-to-end set prediction architecture.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["Image"], "role": "input", "required": true}, + {"id": "cnn_backbone", "label": "CNN Backbone", "aliases": ["Convolutional Backbone", "Backbone"], "role": "module", "required": true}, + {"id": "transformer_encoder", "label": "Transformer Encoder", "aliases": ["Encoder"], "role": "module", "required": true}, + {"id": "object_queries", "label": "Object Queries", "aliases": ["Learned Object Queries"], "role": "input", "required": true}, + {"id": "transformer_decoder", "label": "Transformer Decoder", "aliases": ["Decoder"], "role": "module", "required": true}, + {"id": "prediction_ffn", "label": "Prediction Feed-Forward Network", "aliases": ["Prediction FFN", "Feed Forward Network", "FFN"], "role": "module", "required": true}, + {"id": "class_predictions", "label": "Class Predictions", "aliases": ["Class Labels", "Classes"], "role": "output", "required": true}, + {"id": "box_predictions", "label": "Bounding Box Predictions", "aliases": ["Bounding Boxes", "Box Predictions"], "role": "output", "required": true}, + {"id": "bipartite_matching", "label": "Bipartite Matching", "aliases": ["Hungarian Matching", "Set Prediction Loss"], "role": "training", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "cnn_backbone", "type": "data_flow", "required": true}, + {"source": "cnn_backbone", "target": "transformer_encoder", "type": "data_flow", "required": true}, + {"source": "transformer_encoder", "target": "transformer_decoder", "type": "data_flow", "required": true}, + {"source": "object_queries", "target": "transformer_decoder", "type": "conditioning", "required": true}, + {"source": "transformer_decoder", "target": "prediction_ffn", "type": "data_flow", "required": true}, + {"source": "prediction_ffn", "target": "class_predictions", "type": "branch", "required": true}, + {"source": "prediction_ffn", "target": "box_predictions", "type": "branch", "required": true}, + {"source": "class_predictions", "target": "bipartite_matching", "type": "training_objective", "required": true}, + {"source": "box_predictions", "target": "bipartite_matching", "type": "training_objective", "required": true} + ], + "forbidden_labels": ["Region Proposal Network", "RoIAlign", "Non-Maximum Suppression", "Anchor Boxes", "Mask Branch"], + "expected_reading_order": ["input_image", "cnn_backbone", "transformer_encoder", "object_queries", "transformer_decoder", "prediction_ffn", "class_predictions", "box_predictions"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json new file mode 100644 index 0000000..e800719 --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "End-to-End Object Detection with Transformers", + "venue": "ECCV", + "year": 2020, + "arxiv_id": "2005.12872", + "official_url": "https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/832_ECCV_2020_paper.php", + "paper_urls": ["https://arxiv.org/pdf/2005.12872", "https://export.arxiv.org/pdf/2005.12872"], + "target_figure": "Figure 2", + "verified_caption": "DETR uses a conventional CNN backbone, a transformer encoder-decoder, a fixed set of learned object queries, and parallel class and box predictions; bipartite matching supplies the set prediction training objective.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json new file mode 100644 index 0000000..30a56df --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ICML 2021 CLIP case requiring explicit separation of contrastive training and zero-shot inference.", + "case_id": "107_clip_contrastive", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "multimodal", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json new file mode 100644 index 0000000..d1f43c8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for CLIP contrastive pre-training and zero-shot inference.", + "entities": [ + {"id": "training_images", "label": "Images", "aliases": ["Image Batch", "Training Images"], "role": "input", "required": true}, + {"id": "training_text", "label": "Text", "aliases": ["Captions", "Text Batch", "Training Text"], "role": "input", "required": true}, + {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, + {"id": "text_encoder", "label": "Text Encoder", "role": "module", "required": true}, + {"id": "image_embeddings", "label": "Image Embeddings", "aliases": ["Image Features"], "role": "module", "required": true}, + {"id": "text_embeddings", "label": "Text Embeddings", "aliases": ["Text Features"], "role": "module", "required": true}, + {"id": "contrastive_objective", "label": "Contrastive Learning", "aliases": ["Contrastive Pre-training", "Contrastive Objective", "Image-Text Contrastive Loss", "Correct Pairings"], "role": "training", "required": true}, + {"id": "class_descriptions", "label": "Class Descriptions", "aliases": ["Class Names", "Text Prompts"], "role": "input", "required": true}, + {"id": "zero_shot_classifier", "label": "Zero-Shot Classifier", "aliases": ["Zero-shot Prediction", "Zero-shot Linear Classifier"], "role": "output", "required": true} + ], + "relations": [ + {"source": "training_images", "target": "image_encoder", "type": "encoding", "required": true}, + {"source": "training_text", "target": "text_encoder", "type": "encoding", "required": true}, + {"source": "image_encoder", "target": "image_embeddings", "type": "data_flow", "required": true}, + {"source": "text_encoder", "target": "text_embeddings", "type": "data_flow", "required": true}, + {"source": "image_embeddings", "target": "contrastive_objective", "type": "alignment", "required": true}, + {"source": "text_embeddings", "target": "contrastive_objective", "type": "alignment", "required": true}, + {"source": "class_descriptions", "target": "text_encoder", "type": "encoding", "required": true}, + {"source": "text_embeddings", "target": "zero_shot_classifier", "type": "classification", "required": true}, + {"source": "image_embeddings", "target": "zero_shot_classifier", "type": "classification", "required": true} + ], + "forbidden_labels": ["Bounding Box Regression", "Mask Decoder", "Autoregressive Decoder", "Image Generator", "Region Proposals"], + "expected_reading_order": ["training_images", "image_encoder", "image_embeddings", "contrastive_objective", "training_text", "text_encoder", "text_embeddings", "zero_shot_classifier"], + "topology_requirements": {"primary_flow": "multimodal_alignment", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": true} +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json new file mode 100644 index 0000000..10de429 --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Learning Transferable Visual Models From Natural Language Supervision", + "venue": "ICML", + "year": 2021, + "arxiv_id": "2103.00020", + "official_url": "https://proceedings.mlr.press/v139/radford21a.html", + "paper_urls": ["https://arxiv.org/pdf/2103.00020", "https://export.arxiv.org/pdf/2103.00020"], + "target_figure": "Figure 1", + "verified_caption": "CLIP jointly trains an image encoder and a text encoder to predict correct image-text pairings, then uses text embeddings of class descriptions as a zero-shot classifier at inference time.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json new file mode 100644 index 0000000..e344436 --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ECCV 2020 NeRF case for coordinate conditioning, branched field outputs, and differentiable rendering.", + "case_id": "108_nerf_volume_rendering", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json new file mode 100644 index 0000000..c7cb0ca --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for NeRF's neural scene representation and differentiable rendering pipeline.", + "entities": [ + {"id": "camera_ray", "label": "Camera Ray", "aliases": ["Camera Rays", "Ray"], "role": "input", "required": true}, + {"id": "sampled_points", "label": "Sampled 3D Points", "aliases": ["5D Coordinates", "Spatial Locations"], "role": "module", "required": true}, + {"id": "viewing_direction", "label": "Viewing Direction", "aliases": ["View Direction"], "role": "input", "required": true}, + {"id": "positional_encoding", "label": "Positional Encoding", "aliases": ["Fourier Features"], "role": "module", "required": true}, + {"id": "nerf_mlp", "label": "Neural Radiance Field", "aliases": ["NeRF", "MLP", "Neural Network"], "role": "module", "required": true}, + {"id": "volume_density", "label": "Volume Density", "aliases": ["Density", "Sigma"], "role": "module", "required": true}, + {"id": "color", "label": "Color", "aliases": ["RGB Color", "Radiance"], "role": "module", "required": true}, + {"id": "volume_rendering", "label": "Volume Rendering", "aliases": ["Differentiable Volume Rendering"], "role": "module", "required": true}, + {"id": "rendered_image", "label": "Rendered Image", "aliases": ["Synthesized Image", "Synthesized View", "Novel View"], "role": "output", "required": true} + ], + "relations": [ + {"source": "camera_ray", "target": "sampled_points", "type": "sampling", "required": true}, + {"source": "sampled_points", "target": "positional_encoding", "type": "encoding", "required": true}, + {"source": "positional_encoding", "target": "nerf_mlp", "type": "data_flow", "required": true}, + {"source": "viewing_direction", "target": "nerf_mlp", "type": "conditioning", "required": true}, + {"source": "nerf_mlp", "target": "volume_density", "type": "prediction", "required": true}, + {"source": "nerf_mlp", "target": "color", "type": "prediction", "required": true}, + {"source": "volume_density", "target": "volume_rendering", "type": "rendering_input", "required": true}, + {"source": "color", "target": "volume_rendering", "type": "rendering_input", "required": true}, + {"source": "volume_rendering", "target": "rendered_image", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Voxel Grid", "Mesh Reconstruction", "Convolutional Decoder", "GAN Discriminator", "Depth Sensor"], + "expected_reading_order": ["camera_ray", "sampled_points", "positional_encoding", "nerf_mlp", "volume_density", "color", "volume_rendering", "rendered_image"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json new file mode 100644 index 0000000..e3cb3ab --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis", + "venue": "ECCV", + "year": 2020, + "arxiv_id": "2003.08934", + "official_url": "https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/1473_ECCV_2020_paper.php", + "paper_urls": ["https://arxiv.org/pdf/2003.08934", "https://export.arxiv.org/pdf/2003.08934"], + "target_figure": "Figure 2", + "verified_caption": "NeRF samples 5D coordinates along camera rays, maps position and viewing direction through a neural radiance field to color and volume density, and applies differentiable volume rendering to synthesize an image.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json new file mode 100644 index 0000000..22cdb5e --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NeurIPS 2017 Transformer case for dual embedding paths, encoder-decoder conditioning, and output projection.", + "case_id": "109_transformer_encoder_decoder", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json new file mode 100644 index 0000000..2d079df --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json @@ -0,0 +1,30 @@ +{ + "summary": "Human-authored semantic contract for the Transformer encoder-decoder architecture.", + "entities": [ + {"id": "source_tokens", "label": "Inputs", "aliases": ["Input Sequence", "Source Tokens"], "role": "input", "required": true}, + {"id": "input_embedding", "label": "Input Embedding", "aliases": ["Input Embeddings"], "role": "module", "required": true}, + {"id": "position_encoding", "label": "Positional Encoding", "aliases": ["Position Encoding"], "role": "module", "required": true}, + {"id": "encoder_stack", "label": "Encoder Stack", "aliases": ["Encoder", "N× Encoder"], "role": "module", "required": true}, + {"id": "target_tokens", "label": "Outputs (shifted right)", "aliases": ["Target Tokens", "Target Sequence"], "role": "input", "required": true}, + {"id": "output_embedding", "label": "Output Embedding", "aliases": ["Output Embeddings"], "role": "module", "required": true}, + {"id": "decoder_stack", "label": "Decoder Stack", "aliases": ["Decoder", "N× Decoder"], "role": "module", "required": true}, + {"id": "linear", "label": "Linear", "aliases": ["Linear Layer"], "role": "module", "required": true}, + {"id": "softmax", "label": "Softmax", "role": "module", "required": true}, + {"id": "output_probabilities", "label": "Output Probabilities", "aliases": ["Next-token Probabilities", "Outputs"], "role": "output", "required": true} + ], + "relations": [ + {"source": "source_tokens", "target": "input_embedding", "type": "embedding", "required": true}, + {"source": "input_embedding", "target": "encoder_stack", "type": "data_flow", "required": true}, + {"source": "position_encoding", "target": "encoder_stack", "type": "conditioning", "required": true}, + {"source": "target_tokens", "target": "output_embedding", "type": "embedding", "required": true}, + {"source": "output_embedding", "target": "decoder_stack", "type": "data_flow", "required": true}, + {"source": "position_encoding", "target": "decoder_stack", "type": "conditioning", "required": true}, + {"source": "encoder_stack", "target": "decoder_stack", "type": "cross_attention", "required": true}, + {"source": "decoder_stack", "target": "linear", "type": "data_flow", "required": true}, + {"source": "linear", "target": "softmax", "type": "data_flow", "required": true}, + {"source": "softmax", "target": "output_probabilities", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Convolutional Backbone", "Recurrent Layer", "LSTM", "Region Proposals"], + "expected_reading_order": ["source_tokens", "input_embedding", "encoder_stack", "target_tokens", "output_embedding", "decoder_stack", "linear", "softmax", "output_probabilities"], + "topology_requirements": {"primary_flow": "encoder_decoder", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json new file mode 100644 index 0000000..be0df1d --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Attention Is All You Need", + "venue": "NeurIPS", + "year": 2017, + "arxiv_id": "1706.03762", + "official_url": "https://papers.nips.cc/paper/7181-attention-is-all-you-need", + "paper_urls": ["https://arxiv.org/pdf/1706.03762", "https://export.arxiv.org/pdf/1706.03762"], + "target_figure": "Figure 1", + "verified_caption": "The Transformer model architecture contains encoder and decoder stacks, embeddings with positional encoding, attention and feed-forward sublayers, and a linear-softmax output path.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json new file mode 100644 index 0000000..63d8280 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NAACL 2019 BERT case for embedding summation, pre-training objectives, and downstream fine-tuning.", + "case_id": "110_bert_pretrain_finetune", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "dense_multiframe", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json new file mode 100644 index 0000000..527cf05 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json @@ -0,0 +1,29 @@ +{ + "summary": "Human-authored semantic contract for BERT pre-training, input representation, and fine-tuning.", + "entities": [ + {"id": "input_tokens", "label": "Input Tokens", "aliases": ["Token Sequence", "Input Sequence"], "role": "input", "required": true}, + {"id": "token_embeddings", "label": "Token Embeddings", "role": "module", "required": true}, + {"id": "segment_embeddings", "label": "Segment Embeddings", "aliases": ["Segmentation Embeddings"], "role": "module", "required": true}, + {"id": "position_embeddings", "label": "Position Embeddings", "role": "module", "required": true}, + {"id": "input_representation", "label": "Input Representation", "aliases": ["BERT Input Representation"], "role": "module", "required": true}, + {"id": "bert_encoder", "label": "BERT", "aliases": ["BERT Encoder", "Bidirectional Transformer Encoder"], "role": "module", "required": true}, + {"id": "masked_lm", "label": "Masked LM", "aliases": ["Masked Language Model", "Masked Language Modeling"], "role": "training", "required": true}, + {"id": "next_sentence", "label": "Next Sentence Prediction", "aliases": ["NSP"], "role": "training", "required": true}, + {"id": "fine_tuning", "label": "Fine-tuning", "aliases": ["Fine-tune"], "role": "module", "required": true}, + {"id": "downstream_tasks", "label": "Downstream Tasks", "aliases": ["Task-specific Outputs", "Fine-tuning Tasks"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_tokens", "target": "token_embeddings", "type": "embedding", "required": true}, + {"source": "token_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "segment_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "position_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "input_representation", "target": "bert_encoder", "type": "data_flow", "required": true}, + {"source": "bert_encoder", "target": "masked_lm", "type": "training_objective", "required": true}, + {"source": "bert_encoder", "target": "next_sentence", "type": "training_objective", "required": true}, + {"source": "bert_encoder", "target": "fine_tuning", "type": "initialization", "required": true}, + {"source": "fine_tuning", "target": "downstream_tasks", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Autoregressive Decoder", "Convolutional Backbone", "Retrieval Index", "Image Encoder"], + "expected_reading_order": ["input_tokens", "token_embeddings", "segment_embeddings", "position_embeddings", "input_representation", "bert_encoder", "fine_tuning", "downstream_tasks"], + "topology_requirements": {"primary_flow": "pretrain_then_finetune", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": true} +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json new file mode 100644 index 0000000..f200c15 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", + "venue": "NAACL", + "year": 2019, + "arxiv_id": "1810.04805", + "official_url": "https://aclanthology.org/N19-1423/", + "paper_urls": ["https://arxiv.org/pdf/1810.04805", "https://export.arxiv.org/pdf/1810.04805"], + "target_figure": "Figures 1 and 2", + "verified_caption": "BERT uses the same bidirectional Transformer encoder for pre-training and fine-tuning; its input representation sums token, segment, and position embeddings.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json new file mode 100644 index 0000000..89c7667 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NeurIPS 2020 RAG case for parametric/non-parametric composition and retrieval-conditioned generation.", + "case_id": "111_rag_retrieval_generation", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json new file mode 100644 index 0000000..c54db48 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json @@ -0,0 +1,24 @@ +{ + "summary": "Human-authored semantic contract for retrieval-augmented generation.", + "entities": [ + {"id": "input_query", "label": "Input Query", "aliases": ["Query", "Input x"], "role": "input", "required": true}, + {"id": "query_encoder", "label": "Query Encoder", "role": "module", "required": true}, + {"id": "document_index", "label": "Document Index", "aliases": ["Non-parametric Memory", "Wikipedia Index"], "role": "module", "required": true}, + {"id": "retriever", "label": "Retriever", "aliases": ["Neural Retriever"], "role": "module", "required": true}, + {"id": "top_k_documents", "label": "Top-K Documents", "aliases": ["Retrieved Documents", "Retrieved Passages"], "role": "module", "required": true}, + {"id": "generator", "label": "Generator", "aliases": ["Seq2seq Generator", "Pre-trained Generator"], "role": "module", "required": true}, + {"id": "output_sequence", "label": "Output Sequence", "aliases": ["Generated Output", "Output y"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_query", "target": "query_encoder", "type": "encoding", "required": true}, + {"source": "query_encoder", "target": "retriever", "type": "retrieval_query", "required": true}, + {"source": "document_index", "target": "retriever", "type": "memory_lookup", "required": true}, + {"source": "retriever", "target": "top_k_documents", "type": "retrieval", "required": true}, + {"source": "input_query", "target": "generator", "type": "generation_input", "required": true}, + {"source": "top_k_documents", "target": "generator", "type": "conditioning", "required": true}, + {"source": "generator", "target": "output_sequence", "type": "generation", "required": true} + ], + "forbidden_labels": ["Image Encoder", "Object Queries", "Mask Decoder", "Convolutional Backbone", "Class Prediction", "Class Predictions", "Classification Head"], + "expected_reading_order": ["input_query", "query_encoder", "document_index", "retriever", "top_k_documents", "generator", "output_sequence"], + "topology_requirements": {"primary_flow": "retrieval_then_generation", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json new file mode 100644 index 0000000..e3435f1 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks", + "venue": "NeurIPS", + "year": 2020, + "arxiv_id": "2005.11401", + "official_url": "https://papers.nips.cc/paper/2020/hash/6b493230205f780e1bc26945df7481e5-Abstract.html", + "paper_urls": ["https://arxiv.org/pdf/2005.11401", "https://export.arxiv.org/pdf/2005.11401"], + "target_figure": "Figure 1", + "verified_caption": "RAG combines a query encoder and non-parametric document index with a pre-trained seq2seq generator; retrieved top-K documents condition output generation.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/human_rating_template.json b/benchmarks/paper-to-image/human_rating_template.json new file mode 100644 index 0000000..301efbc --- /dev/null +++ b/benchmarks/paper-to-image/human_rating_template.json @@ -0,0 +1,30 @@ +{ + "summary": "Blinded human rating record for one generated scientific figure.", + "case_id": "", + "candidate_id": "", + "reviewer_id": "anonymous_reviewer_01", + "reviewer_background": "paper_author|domain_expert|visual_design_expert|general_researcher", + "scores_1_to_5": { + "scientific_faithfulness": 0, + "information_completeness": 0, + "information_density_without_clutter": 0, + "reading_order_clarity": 0, + "arrow_and_grouping_clarity": 0, + "text_readability_at_target_size": 0, + "aesthetic_quality": 0, + "academic_professionalism": 0 + }, + "quick_comprehension": { + "input_identified": false, + "output_identified": false, + "core_innovation_identified": false, + "main_flow_direction_identified": false + }, + "hard_errors": [], + "pairwise_preference": { + "other_candidate_id": "", + "preferred": "this|other|tie", + "reason": "" + }, + "notes": "" +} diff --git a/benchmarks/paper-to-image/preferences/top_conference.json b/benchmarks/paper-to-image/preferences/top_conference.json new file mode 100644 index 0000000..bbdf8c7 --- /dev/null +++ b/benchmarks/paper-to-image/preferences/top_conference.json @@ -0,0 +1,9 @@ +{ + "summary": "Shared visual preferences for computer-science top-conference benchmark cases.", + "aspect_ratio": "16:9", + "language": "English", + "style_description": "clean top-conference method figure, compact academic layout, restrained color, clear arrows and grouping", + "preferred_palette": ["cool blue", "teal", "muted orange accent", "light neutral background"], + "visual_density": "high but readable", + "avoid": ["commercial poster", "photorealistic decoration", "cyberpunk", "generic robot imagery", "fake equations", "fake charts", "long paragraphs"] +} diff --git a/benchmarks/schema/case.schema.json b/benchmarks/schema/case.schema.json new file mode 100644 index 0000000..076f0ae --- /dev/null +++ b/benchmarks/schema/case.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ResearchFigureStudio benchmark case", + "type": "object", + "required": ["summary", "case_id", "suite", "thresholds", "run_config"], + "properties": { + "summary": {"type": "string", "minLength": 1}, + "case_id": {"type": "string", "minLength": 1}, + "suite": {"enum": ["paper-to-image", "image-to-ppt"]}, + "thresholds": {"type": "object"}, + "run_config": {"type": "object"} + }, + "additionalProperties": true +} diff --git a/benchmarks/schema/expected_objects.schema.json b/benchmarks/schema/expected_objects.schema.json new file mode 100644 index 0000000..f0119ea --- /dev/null +++ b/benchmarks/schema/expected_objects.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Image-to-PPT object ground truth", + "type": "object", + "required": ["summary", "objects", "relations"], + "properties": { + "summary": {"type": "string"}, + "objects": {"type": "array", "items": {"type": "object", "required": ["id", "type", "bbox_percent"]}}, + "relations": {"type": "array", "items": {"type": "object", "required": ["source", "target", "type"]}} + } +} diff --git a/benchmarks/schema/expected_semantics.schema.json b/benchmarks/schema/expected_semantics.schema.json new file mode 100644 index 0000000..9d98d43 --- /dev/null +++ b/benchmarks/schema/expected_semantics.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Paper-to-image semantic ground truth", + "type": "object", + "required": ["summary", "entities", "relations", "forbidden_labels"], + "properties": { + "summary": {"type": "string"}, + "entities": {"type": "array", "items": {"type": "object", "required": ["id", "label", "role"]}}, + "relations": {"type": "array", "items": {"type": "object", "required": ["source", "target", "type"]}}, + "forbidden_labels": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/benchmarks/schema/paper_image_review.schema.json b/benchmarks/schema/paper_image_review.schema.json new file mode 100644 index 0000000..e42afd9 --- /dev/null +++ b/benchmarks/schema/paper_image_review.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Frozen judge or human-calibrated paper-image review", + "type": "object", + "required": ["scientific", "paper_alignment", "topology", "ocr", "information", "clarity", "aesthetic", "stability"], + "properties": { + "scientific": { + "type": "object", + "required": ["score", "missing_modules", "missing_relations", "reversed_relations", "invented_items"] + }, + "paper_alignment": { + "type": "object", + "required": ["score", "passed", "priorities", "missing_priorities", "evidence_conflicts", "unsupported_visual_claims"] + }, + "topology": { + "type": "object", + "required": ["score", "passed", "missing_relations", "reversed_relations", "invented_relations"] + }, + "ocr": { + "type": "object", + "required": ["score", "missing_labels", "misspelled_labels", "forbidden_labels_found"] + }, + "information": { + "type": "object", + "required": ["coverage", "density", "redundancy", "unsupported_information", "score"] + }, + "clarity": { + "type": "object", + "required": ["reading_order", "arrow_ambiguity", "grouping", "target_size_readability", "cognitive_load", "score"] + }, + "aesthetic": { + "type": "object", + "required": ["hierarchy", "balance", "whitespace", "color", "icon_consistency", "professionalism", "score"] + }, + "stability": { + "type": "object", + "required": ["seed_count", "mean_score", "worst_case_score", "standard_deviation", "production_pass_rate"] + } + } +} diff --git a/codex-skills/research-figure-making/SKILL.md b/codex-skills/research-figure-making/SKILL.md deleted file mode 100644 index fe1efa6..0000000 --- a/codex-skills/research-figure-making/SKILL.md +++ /dev/null @@ -1,456 +0,0 @@ ---- -name: research-figure-making -description: >- - Use this skill when the user provides a research paper, manuscript, method - section, experiment section, LaTeX/Word/PDF/Markdown draft, result table, CSV, - figure sketch, or old paper figure and asks to create, redesign, or improve - publication-quality scientific figures, including paper-driven framework - figures, system overview diagrams, model architecture figures, pipeline - diagrams, data plots, main-result figures, multi-panel figures, or PPT/Lark - editable figures. Prioritize AI/ML/NLP research papers, paper-grounded visual - extraction, AI-generated visual blocks for framework figures, reproducible - Python plots for data figures, and PPTX-first editable outputs with optional - Lark/SVG exports. Also use for - Chinese-language requests that mean drawing figures from a paper, making paper - framework diagrams, system diagrams, data figures, main paper figures, PPT - figures, Lark whiteboard figures, method figures, result figures, or - top-tier-publication figures. ---- - -# Research Figure Making - -## Summary - -Create publication-oriented research figures from the user's actual paper content. -Do not use generic architecture templates until the paper has been read and the -paper-specific concepts, variables, modules, and claims have been extracted. -For image-rich framework figures, default to many slot-level image2 blocks, -PPTX-first editable composition, front Summary sections, no semantic cropping, -and hard asset-fill validation. - -## Trigger - -Use this skill when the user asks for figures based on: - -- a paper draft, method section, experiment section, abstract, related work, or notes -- LaTeX, Word, PDF, Markdown, plain text, tables, CSV/Excel, logs, or existing figures -- requests such as "draw Figure 1 from my paper", "make a system overview", - "create a framework diagram", "turn this paper into PPT figures", "improve my - data plot", or "redesign this figure for a top-tier paper" -- Chinese-language requests that mean drawing framework figures from a paper, - making system diagrams from method sections, turning experimental results into - data figures, creating main paper figures, PPT figures, or top-tier paper figures - -## Mandatory Framework Protocol - -For system architecture, framework, pipeline, model overview, or Figure 1 style -requests where the user wants an image-rich figure, this protocol is mandatory: - -## Front Summary Rule - -Every text artifact must start with a `Summary` section before details. This -applies to the figure brief, slot inventory, style sheet, figure program, -prompts, asset quality report, asset visual review, layout plan, visual critic, -alignment review, critic report, and final response. For Markdown, the first -non-empty heading must be `# Summary` or `## Summary`. For JSON, use a top-level -non-empty `summary` field. - -## Local RFS Route - -When `D:\ResearchFigureStudio` and the `rfs` CLI are available, use them as the -default implementation for image-rich framework figures: - -```powershell -rfs make-framework --paper --reference --out --slot-count 40 --slot-source reference-primary --complexity-profile reference-dense --candidates-per-slot 4 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 8 --asset-mode image2 --asset-workers 6 --asset-retries 3 --asset-review-mode heuristic --critic-mode heuristic -``` - -Use `--asset-mode image2` for real image generation through the Yunwu -OpenAI-compatible Images API. The logical `image-2` target maps to Yunwu's listed -`gpt-image-2` model unless `RFS_IMAGE_MODEL` or `IMAGE_MODEL` overrides it. Keep -`--asset-mode gemini` as a fallback and use `--asset-mode placeholder` only for -engineering validation. Use `--locator-mode vlm` when a -VLM should borrow the LiveFigure-style positioning idea by returning -`layout_plan.json` coordinates. The VLM must not write arbitrary PPT code or -generate a single full diagram. For API-cost control, run a small real pass first -with `--slot-count 25 --candidates-per-slot 1 --asset-workers 3`, then a full -pass after the contract validates. Yunwu image2/Gemini slot generation may run -concurrently through RFS workers; do not fall back to manual one-by-one image2 -generation unless the API path is unavailable. -Prompt planning is not a cost-saving heuristic stage by default: use -`--prompt-plan-mode vlm` so the model inspects the reference image and each -local slot before producing `slot_prompt_plan.json`. Use heuristic prompt -planning only for offline engineering validation. Use `--prompt-plan-workers` -to parallelize per-slot VLM prompt planning; use `--asset-workers` separately to -parallelize Yunwu image2/Gemini image generation. -Use `--control-localizer-mode hybrid` by default for arrow and connector -localization. This AutoFigure-inspired stage writes -`reference_control_candidates.json`, `slot_overlay.png`, and -`reference_control_overlay.png`; VLM binding may assign source-target semantics -from those overlays, while the fallback heuristic keeps the workflow offline. -The VLM may patch arrow IDs, anchors, and normalized paths only; it must not -write PPT code, rasterize arrows, or redraw the full figure. -Use `--arrow-style-mode reference` by default. This stage may soften PPT -connector line caps, vary stroke widths, assign bundle/lane metadata, and score -crossing/bend/overlap quality, but the reference image remains the hard -constraint. It must not replace reference-derived flow logic with a generic -graph router. Orthogonal obstacle-aware routing is allowed only for missing -paths or arrows explicitly marked `route_policy=fallback_reroute_allowed`; it -must never rewrite reference-locked `path_percent` values. -Use `--arrow-style-mode aesthetic` only for an explicit experimental -beautification pass. In that mode, the router may apply curve connectors and -halo underlays by default. Bundle lane offsets require explicit route opt-in -and must stay inside `reference_tunnel_percent`; the router must record -`reference_original_path_percent`, `reference_path_delta_max`, and -`reference_tunnel_preserved`, and it must not change source-target logic. -Use `rfs presentations-qa --out ` only as an optional QA pass when -the Presentations plugin is available. Presentations may import/render the PPTX, -extract layout JSON, and report connector/font/rendering drift, but it must not -mutate the PPTX, rewrite arrows, regenerate layout, or replace RFS as the -authoritative compiler for reference-locked connector-heavy figures. - -Before execution, read these references in this order: - -1. `references/paper_to_figure.md` -2. `references/visual_framework_figures.md` -3. `references/reference_image_alignment.md` when a reference image is provided -4. `references/stylist_stage.md` before writing image2 prompts -5. `references/figure_program.md` before composition or scripting -6. `references/image_block_prompting.md` before writing image2 prompts -7. `references/critic_stage.md` before final delivery -8. `references/journal_quality_checklist.md` before final delivery - -Execution checklist: - -1. Read the paper and write a paper-grounded figure brief. -2. If the user provides a reference image, parameterize it before generation: - identify slots, canvas ratios, safe areas, and fit policies. - When the user says the reference image should guide the figure, treat its - slot positions and flow logic as layout source-of-truth. The paper calibrates - terminology and scientific labels; it must not cause a generic paper-derived - grid to replace the reference image's spatial logic. - In reference-primary mode, the reference image is the highest authority for - layout, visual object choice, style, framework colors, arrow logic, and - visual rhythm. The paper only assists terminology and scientific mapping. - Write `reference_geometry.json` with precise panel/slot geometry before any - prompt generation. Every panel and slot must include `bbox_percent`, - `center_percent`, `width_percent`, `height_percent`, - `aspect_ratio_decimal`, `aspect_ratio_w_h`, `target_pixels`, - `target_pixels_exact`, and `generation_min_pixels` with at least three - decimals where numeric precision matters. `target_pixels` must equal - `target_pixels_exact`; use `generation_min_pixels` only for the image - generator's minimum resolution safeguard. - Also write `reference_control_candidates.json`, - `slot_overlay.png`, `reference_control_overlay.png`, and - `reference_controls.json` for measured arrows, connector lines, dashed - loops, transition arrows, framework shapes, and text regions. Control - candidates are analogous to a boxlib/overlay layer: they identify possible - arrows and connectors from the reference image before semantic binding. - Bound controls must record geometry, `source_id`, `target_id`, - `source_anchor`, `target_anchor`, multi-point `path_percent`, color token, - and `render_policy: ppt_shape_not_image_asset`. - Then write `arrow_style_profile.json`, `selected_arrow_routes.json`, and - `arrow_quality_report.json`. These files must state - `reference_image_hard_constraint`, preserve locked reference paths, and only - synthesize missing or explicitly fallback-allowed paths. If fallback routing - is used, record `routing_algorithm`, `route_generation_status`, candidate - count, and obstacle/crossing metrics. - If `--arrow-style-mode aesthetic` is used, also record original reference - paths, tunnel width, path delta, halo settings, connector type, and whether - each adjusted route stayed inside the reference tunnel. -3. Create a slot inventory before using image2. Default to 25-50 image slots for - a normal paper system figure. This count means non-arrow image slots only; - arrows, connector lines, dashed loops, panel frames, and titles must not be - counted as image2 assets. -4. Write `reference_style_profile.json` and `style_sheet.md` before writing - image2 prompts. They must define reference-derived color tokens, palette, - line weight, shadows, viewpoint, icon complexity, background, visual density, - font-layer rules, and image2 text policy. -5. Write `layout_plan.json` and `figure_program.json` before composition. - `layout_plan.json` records normalized reference-guided coordinates; it may be - produced by heuristic rules or VLM positioning, but never by freeform PPT - code. `figure_program.json` must record `canvas`, - `panels`, `slots`, `assets`, `labels`, `arrows`, `groups`, and - `export_targets`; slots must bind paper concepts, reference bboxes, target - ratios, safe areas, fit policies, and asset IDs. `export_targets` must - include the main editable PPTX target, usually `editable_composition.pptx`. - In reference-primary mode, `bbox_percent` values extracted from the reference - image must be preserved into `layout_plan.json`, `figure_program.json`, and - PPT placement. A VLM locator may refine arrows or identify missing slots, but - it must not freely redraw the macro framework or overwrite reference-derived - slot positions. Do not convert exact reference ratios into coarse presets - like `1:1`, `4:3`, `3:4`, `16:9`, or `9:16`; use precise decimal ratios such - as `0.538:1.000`. - Text size, position, color, and hierarchy are reference-primary too. Do not - create a default `publication_scale.json`, do not enforce a fixed - `paper_double_column` target width, and do not fail a figure only because a - reference-matched label would be small after manuscript scaling. Instead, - write `reference_text_geometry.json`, `text_program.json`, and - `text_alignment_report.json`. These files must preserve the reference - image's text bbox, center, relative font height, color, role, and hierarchy. - The paper may adapt terminology, but the PPT text layer must stay editable - and must not override the reference layout with a generic typography rule. -6. Write `slot_visual_spec.json`, `reference_slot_prompt_brief.json`, and - `slot_prompt_plan.json` before image generation. `slot_visual_spec.json` - records each slot's reference crop objects, foreground subject, secondary - objects, micro details, background fill elements, scientific mechanism - detail, required visual complexity, and forbidden simplifications. Except - explicit legend/badge slots, normal image slots must be planned as dense mini - scientific scenes/cards with 2-5 layered objects, not simple standalone - icons. The brief records what each reference slot does and what paper concept - it carries. `slot_prompt_plan.json` must be generated by a VLM by default and - include per-slot `slot_function`, local reference style, concrete - `visual_metaphor`, `must_show`, `avoid_showing`, visual-complexity fields, - and `image_prompt_core`. Every image slot must have - `reference_slot_crops/.png`, and the prompt planner must use the - full reference image, the local slot crop, the global style profile, local - color token ids, precise geometry, and the paper concept. Arrows and - connector controls never receive image prompts. For normal 25-50 slot - figures, run VLM prompt planning in parallel unless rate limits force a lower - worker count. -7. Generate image assets per slot, preferably through RFS Gemini/Yunwu - concurrent workers when configured. Manual image2 generation is only a - fallback. Do not generate one full architecture image as the final figure. -8. Create `asset_quality_report.json`, `asset_complexity_report.json`, - `composition_quality_report.json`, `asset_visual_review.json`, selected asset - contact sheet, and candidate contact sheet. The quality report must record - `content_fill_percent`, `empty_margin_percent`, `edge_cutoff_status`, - `ratio_status`, selected candidate, and `action` for every image block. The - complexity report must record `detail_score`, `object_count_estimate`, - `simple_icon_risk`, `reference_crop_match`, `style_match`, and - `selected_reason`; unresolved `too_simple`, `generic_icon`, - `reference_crop_ignored`, `single_object_on_blank_background`, or - `style_drift` must block delivery. - `composition_quality_report.json` must prove that PPT insertion used - frameless image slots, no extra white tile, no caption inside the image slot, - and at least 95% image-slot area fill. -9. Compose the final figure in `editable_composition.pptx` by default. Use PPT - editable layers for containers, arrows, connector lines, dashed loops, labels, - formulas, legends, and panel IDs. Image slots are direct frameless assets; - arrows are PPT shapes/connectors, not generated PNGs. -10. Write an alignment review noting source grounding, slot count, rejected assets, - and reference-image mismatches. -11. Write `visual_critic_iter_0.json` and `critic_report.md` or - `critic_report.json` before final delivery. - It must check paper faithfulness, slot count, text controllability, whether - the output used a single full diagram, vector-only fallback, semantic - cropping, incomplete image blocks, asset fill quality, asset complexity, - reference-crop match, style match, and PPT editability. - Major failures must be fixed or the delivery must stop. -12. Run `scripts/validate_framework_outputs.py ` before final - delivery when a local output directory exists. -13. Optionally run `rfs presentations-qa --out ` after validation. - Treat `autoRouteConnectorPx failed` or similar plugin connector fallback - warnings as QA evidence. Do not let the Presentations plugin rebuild the - figure or override RFS connector geometry. - -Hard workflow order: - -`input archive -> paper brief -> reference slot/control analysis -> reference_geometry.json -> -reference_control_candidates.json -> slot_overlay.png/reference_control_overlay.png -> -reference_controls.json -> arrow_style_profile.json/selected_arrow_routes.json/arrow_quality_report.json -> -reference_style_profile.json/style sheet -> -layout_plan.json -> figure_program.json -> reference_text_geometry.json/text_program.json/text_alignment_report.json -> slot_visual_spec.json -> -reference_slot_prompt_brief.json -> slot_prompt_plan.json -> image2/Gemini slot prompts -> generated -assets/asset_quality_report/asset_complexity_report/composition_quality_report/asset_visual_review/contact sheets -> -editable_composition.pptx -> PDF/PNG export -> visual critic -> critic report -> -final validation/export -> optional presentations QA report` - -Image block fill rules: - -- Target useful visual content fill: 90%-97% of the canvas. -- Minimum passing content fill: 85% unless the slot is explicitly marked as a - sparse symbol with a documented reason. -- Maximum pure empty margin in any direction: 10%. -- Safe area means the key subject is not cut off; it does not mean leaving a - large blank border. Background texture, secondary details, card surfaces, and - non-critical supporting marks should extend close to the canvas edge. -- If a block has too much whitespace, regenerate or redesign the prompt. Padding - is only for ratio matching, not for fixing a tiny subject on a blank canvas. - -Forbidden shortcuts: - -- Do not satisfy an image-rich framework request with vector-only shapes unless - the user explicitly asks for a vector-only diagram. -- Do not silently replace image2 blocks with PPT shapes, SVG, or Lark shapes - because that is faster or easier. -- Do not deliver only an SVG source file for an editable research figure unless - the user explicitly asks for SVG-only or Illustrator/Inkscape-first work. -- Do not generate a single full diagram image and then screenshot or crop it as - the final result. -- Do not use a browser or canvas screenshot as the primary artifact unless it is - a rendered composition assembled from generated slot assets. -- Do not use the Presentations plugin as the primary compiler for image-rich - research figures; it is QA-only unless the user explicitly asks for a - separate presentation deck workflow. -- If image2/image generation is unavailable, say so and stop or ask for a - fallback; do not silently switch to a vector-only workflow. - -Required output files for image-rich framework figures: - -- `slot_inventory.json` or `slot_inventory.md` -- `reference_geometry.json` -- `reference_control_candidates.json` -- `slot_overlay.png` -- `reference_control_overlay.png` -- `reference_controls.json` -- `arrow_style_profile.json` -- `selected_arrow_routes.json` -- `arrow_quality_report.json` -- `reference_style_profile.json` -- `style_sheet.md` or `style_sheet.json` -- `input_manifest.json` -- `layout_plan.json` -- `figure_program.json` -- `reference_text_geometry.json` -- `text_program.json` -- `text_alignment_report.json` -- `slot_visual_spec.json` -- `reference_slot_prompt_brief.json` -- `slot_prompt_plan.json` -- `reference_slot_crops/.png` for every non-arrow image slot -- `prompts.md` -- at least 25 generated image assets for normal system figures -- `asset_candidate_contact_sheet.png` -- `asset_quality_report.json` -- `asset_complexity_report.json` -- `composition_quality_report.json` -- `asset_visual_review.json` -- `asset_contact_sheet.png` -- `editable_composition.pptx` -- final figure export -- `visual_critic_iter_0.json` -- `alignment_review.md` -- `critic_report.md` or `critic_report.json` - -Optional Presentations-plugin QA outputs: - -- `presentations_plugin_qa_report.json` -- `presentations_plugin_qa_report.md` -- `presentations_plugin_qa_workspace/` - -Validation must fail if the output uses only a single generated full-diagram -image, lacks slot assets, lacks prompts, lacks a contact sheet, or records -semantic cropping as the fitting strategy. Validation must also fail when the -style sheet, layout plan, figure program, reference text geometry, text program, -text alignment report, slot visual spec, reference slot -prompt brief, slot prompt plan, reference geometry, asset quality report, asset -complexity report, composition quality report, asset visual review, visual -critic report, critic report, or editable PPTX source is missing. -Validation must fail when any unresolved image block has `content_fill_percent` -below its minimum or `empty_margin_percent` above its maximum. -Validation must fail when a slot uses a coarse preset ratio, when PPT insertion -adds an extra white tile, or when an inserted image fills less than 95% of its -slot area. -Validation must fail when PPT text is not bound to -`reference_text_geometry.json`, when text center/bbox/font-ratio drift from the -reference exceeds the recorded alignment tolerance, or when critical text is -not editable in PPTX. Validation must not fail only because the text would be -small under a default paper-double-column scaling assumption. -Validation must fail when a normal non-legend slot lacks `secondary_objects` or -`micro_details`, or when a selected asset has unresolved `too_simple`, -`generic_icon`, `reference_crop_ignored`, `single_object_on_blank_background`, -or `style_drift`. -Validation must fail when arrow, dashed loop, transition, or connector elements -appear in `slots`/`assets`; they must appear in `reference_controls.json` and -`figure_program.json` as editable PPT controls with source/target logic and -style tokens. Validation must also fail when `reference_control_candidates.json` -or the slot/control overlay images are missing, when a bound control lacks -`source_id`, `target_id`, `source_anchor`, `target_anchor`, or at least two -`path_percent` points, or when composition reports that a control was not -rendered as an editable PPT connector. Validation must fail when -`arrow_style_profile.json`, `selected_arrow_routes.json`, or -`arrow_quality_report.json` are missing, or when an arrow style stage overrides -a locked reference path without a documented reason. Validation must also fail -when fallback obstacle routing is applied to a reference-locked path, or when -route artifacts omit `routing_algorithm` / `route_generation_status`. -For aesthetic mode, validation or critic review must fail when -`reference_tunnel_preserved` is false, when source-target binding changes, or -when a curve/bundle/halo style is baked into raster images instead of PPT -editable connector shapes. -Validation must also fail when image slots lack local reference crops, local color token ids, or -`reference_style_profile.json` grounding. - -## Core Workflow - -1. Read the source material first. Extract the research problem, method story, - inputs/outputs, named components, training/inference flow, datasets, baselines, - metrics, and main claims before planning any figure. -2. Create a candidate figure inventory. Include only figures that carry paper - information: method overview, system pipeline, model architecture, data - construction, main results, ablation, case study, error analysis, or appendix - figures. -3. Route each figure: - - **Framework / architecture / pipeline**: use the visual-block workflow in - `references/visual_framework_figures.md`. - If the user provides a full reference image first, - also use `references/reference_image_alignment.md`. - Before image2 prompting, write a style sheet using - `references/stylist_stage.md` and a structured program using - `references/figure_program.md`. - Then create `reference_slot_prompt_brief.json` and VLM-generated - `slot_prompt_plan.json`; for image2 visual blocks, use - `references/image_block_prompting.md` to - generate slot-level prompts with target aspect ratios and low-blank-space - requirements. - Before delivery, run the critic stage in `references/critic_stage.md`. - - **Data / statistical / result plots**: use `references/data_figures.md`. - - **Mixed paper figure or multi-panel layout**: combine both routes, then use - PPTX-first composition for the editable working file. -4. Keep scientific structure editable. Use generated images for visual blocks, - but add labels, formulas, arrows, panel marks, grouping boxes, legends, and - captions with editable PPT tools by default. -5. For reference-image-driven framework figures, parameterize the reference image - first: identify visual slots, estimate each slot's relative bounding box and - aspect ratio, then generate many small blocks for those slots instead of a few - large macro-panel images. -6. Run the journal-quality check before final delivery. See - `references/journal_quality_checklist.md`. - -## Required Grounding Rules - -- Every framework module name must come from the user's paper or be explicitly - introduced as an inferred simplification. -- Never insert stock modules such as encoder, retriever, memory, or decoder unless - the paper actually uses that concept or the user approves the abstraction. -- AI-generated visual blocks may illustrate scientific objects, data flow, - model components, reasoning stages, or experimental settings, but must not be - trusted for exact text, formulas, numbers, axes, or citations. -- For data figures, Python remains the source of truth for computation and plots. - PPT editing is for layout, annotation, panel assembly, and presentation polish, - not for changing plotted values. -- Keep source artifacts: raw input, extracted figure brief, generated prompts, - full reference image when provided, original image blocks, enhanced blocks, - slot inventory, layout plan, style sheet, figure program, asset quality - report, asset visual review, visual critic report, critic report, alignment - review notes, editable composition, and final exports. - -## Output Policy - -Default outputs: - -- editable working file: `editable_composition.pptx` by default -- publication export: PDF plus 600 DPI PNG/TIFF by default -- optional SVG/PDF vector export only when the venue, user, or downstream - Illustrator/Inkscape workflow explicitly needs it -- reproducibility files: Python plotting script and cleaned data for data plots -- prompt record: visual-block prompts and selected/rejected image notes -- for image-rich framework figures: slot inventory, image2 prompts, generated - assets, asset quality report, asset visual review, contact sheets, style - sheet, layout plan, figure program, editable composition, visual critic, - critic report, final export, and alignment review that pass - `rfs validate` or `scripts/validate_framework_outputs.py` when outputs are - local - -Use `references/paper_to_figure.md` for paper analysis and figure planning. Use -`references/enhancement_pipeline.md` when generated images need upscaling, -sharpening, background removal, or artifact cleanup. Use -`references/reference_image_alignment.md` only when the user supplies a complete -reference framework image as the visual blueprint for the final editable -composition. Do not create a full reference image yourself unless the user -explicitly asks for one. Use `references/stylist_stage.md` to lock style before -prompting. Use `references/figure_program.md` to define the structured -intermediate layout before scripting or composition. Use -`references/image_block_prompting.md` whenever image2 prompts are needed for -framework sub-blocks. Use `references/critic_stage.md` to block final delivery -when the workflow violates the required image-rich, editable, no-crop rules. - diff --git a/codex-skills/research-figure-making/agents/openai.yaml b/codex-skills/research-figure-making/agents/openai.yaml deleted file mode 100644 index e4b5c2e..0000000 --- a/codex-skills/research-figure-making/agents/openai.yaml +++ /dev/null @@ -1,7 +0,0 @@ -interface: - display_name: "Research Figure Making" - short_description: "Create paper-grounded scientific figures and editable framework diagrams." - default_prompt: "Use $research-figure-making to turn my paper draft into publication-quality framework and data figures." - -policy: - allow_implicit_invocation: true diff --git a/codex-skills/research-figure-making/references/critic_stage.md b/codex-skills/research-figure-making/references/critic_stage.md deleted file mode 100644 index 44bdd08..0000000 --- a/codex-skills/research-figure-making/references/critic_stage.md +++ /dev/null @@ -1,127 +0,0 @@ -# Critic Stage - -## Summary - -Use this stage after the contact sheet and editable composition are produced, -but before final delivery. The critic is a gate, not a summary. Major failures -must be fixed automatically when possible or reported as blockers. - -## Required Output - -Write `critic_report.md` or `critic_report.json` in the output directory. -For local `rfs` runs, also require `asset_visual_review.json` and -`asset_complexity_report.json` plus `visual_critic_iter_0.json`. - -Minimum checks: - -- `paper_faithfulness`: all modules, labels, formulas, and arrows trace back to - the paper or are explicitly marked as inferred -- `slot_count`: normal system figures use at least 25 slot-level image assets -- `input_archive`: `input_manifest.json` exists and records archived paper and - reference paths -- `layout_plan`: `layout_plan.json` exists, has at least 25 slots, and was used - as the positioning layer before `figure_program.json` -- `image2_usage`: visual content is generated as slot assets, not one full - architecture image -- `not_vector_only`: vector/PPT/SVG elements are structure and editable layers, - not a replacement for requested image blocks -- `text_controllability`: critical terms, variables, formulas, metrics, panel - IDs, and arrow labels are editable layer objects -- `reference_text_alignment`: `reference_text_geometry.json`, - `text_program.json`, and `text_alignment_report.json` exist; text size, - position, color, and hierarchy follow the reference image; no default - publication-width or minimum-font threshold overrides the reference -- `ppt_editability`: `editable_composition.pptx` exists and is the main editable - source; labels, formulas, arrows, group boxes, legends, and panel IDs are PPT - editable objects, not baked into image2 blocks, SVG, PNG, or screenshots -- `no_semantic_crop`: no center-crop, cover-crop, forced crop, or crop-to-ratio - that removes semantic content -- `block_integrity`: no cut-off icons, cards, characters, charts, or meaningful - visual details -- `asset_fill_quality`: every image block has `content_fill_percent` at or above - its `min_content_fill_percent`, pure empty margin at or below - `max_empty_margin_percent`, and no unresolved `too_much_whitespace`, - `tiny_centered_subject`, or `large_blank_canvas` issue -- `asset_complexity_quality`: every normal non-legend image block has enough - visual detail to function as a research mini scene/card, not a sparse generic - icon. Check `detail_score`, `object_count_estimate`, `simple_icon_risk`, - `reference_crop_match`, `style_match`, and unresolved `too_simple`, - `generic_icon`, `reference_crop_ignored`, - `single_object_on_blank_background`, or `style_drift` -- `style_consistency`: generated blocks follow `style_sheet` -- `program_consistency`: final figure matches `figure_program.json` -- `locator_boundary`: VLM positioning, when used, outputs coordinates and arrow - routes only; it does not write arbitrary PPT code -- `arrow_control_localization`: `reference_control_candidates.json`, - `slot_overlay.png`, and `reference_control_overlay.png` exist; controls have - source/target IDs, source/target anchors, multi-point normalized paths, and - reference color tokens; final PPT renders them as editable connector shapes -- `arrow_style_quality`: `arrow_style_profile.json`, - `selected_arrow_routes.json`, and `arrow_quality_report.json` exist; styling - preserves reference-locked paths, improves softness/variation through PPT - connector properties, reports crossings, bends, and obstacle overlaps, and - uses obstacle-aware routing only for missing or explicitly fallback-allowed - paths. If `--arrow-style-mode aesthetic` is used, every adjusted route must - remain inside its reference tunnel, keep source-target binding unchanged, and - render curve/bundle/halo effects as editable PPT shapes -- `reference_alignment`: when a reference image exists, major flow, group - positions, visual density, and resource regions remain aligned unless the - paper requires a deviation -- `asset_visual_review`: selected image blocks passed heuristic or VLM review -- `visual_critic`: reference-vs-render critic has no unresolved blocking issue - -## Severity - -- `major`: wrong paper mechanism, missing key module, missing PPTX source, - PPTX that contains only one flattened full-figure screenshot, single - full-diagram output, vector-only fallback, SVG-only delivery, semantic crop, - baked labels, critical text generated by image2, unresolved severe whitespace - failure, unresolved severe complexity failure such as `too_simple` or - `reference_crop_ignored`, arrows baked into raster assets, missing control - overlays, controls without source-target binding, arrow styling that - overrides reference-locked paths, fallback routing applied to a reference - locked path, aesthetic routing with `reference_tunnel_preserved=false`, text - not bound to reference text geometry, critical text not editable in PPTX, or - fewer than 25 slot assets for a normal system figure -- `medium`: inconsistent style, poor slot density, unclear arrow route, weak - reference alignment, or incomplete prompt records -- `minor`: spacing, padding, color balance, small visual cleanup, or export - polish issue - -## Required Actions - -For major failures: - -- regenerate the offending image block -- regenerate or reprompt blocks whose useful content fill is below 80% -- regenerate or reprompt blocks whose pure empty margin exceeds 10%-12% -- regenerate or reprompt normal non-legend blocks that are simple standalone - icons, ignore the local reference crop, drift from the reference style, or - lack layered objects/micro-details -- revise `figure_program.json` -- adjust editable labels, arrows, or groups -- rebuild `reference_text_geometry.json` and `text_program.json` when text - bbox, relative font height, color, or center position drift from the reference - image; do not fix this by applying a generic publication-font threshold -- patch arrows only through `arrow_id`, `source_anchor`, `target_anchor`, and - `path_percent`; do not let the critic rewrite the whole figure or emit PPT - code -- rebuild the PPT editable layer when labels, formulas, arrows, group boxes, or - panel IDs are baked into an image or SVG -- create `editable_composition.pptx` when it is missing -- replace vector-only placeholders with image2 slot assets -- stop delivery if image generation is unavailable and the user did not approve - a fallback - -For medium failures: - -- adjust layout, style sheet, prompts, padding, or arrow routing before export -- document any intentional deviation from the reference image - -For minor failures: - -- fix when practical, or record as accepted residual polish issue - -The final response should not claim completion if the critic report contains -unresolved major failures. - diff --git a/codex-skills/research-figure-making/references/stylist_stage.md b/codex-skills/research-figure-making/references/stylist_stage.md deleted file mode 100644 index be84130..0000000 --- a/codex-skills/research-figure-making/references/stylist_stage.md +++ /dev/null @@ -1,83 +0,0 @@ -# Stylist Stage - -## Summary - -Use this stage after the paper brief and reference-image slot analysis, but -before writing any image2 prompt. The goal is to make all generated image blocks -look like one coherent figure instead of unrelated illustrations. - -## Required Output - -Write `reference_style_profile.json` and `style_sheet.md` or -`style_sheet.json` in the output directory. The JSON profile is the machine -contract; the Markdown sheet is the human-readable summary. - -Minimum fields: - -- `style_name`: short name for the visual system -- `palette`: 4-8 colors with roles such as panel fill, accent, warning, cache, - data, model, and neutral line -- `color_tokens`: exact reference-derived color records with `token_id`, `hex`, - `rgb`, `hsl`, `source_region`, and `usage`; panel headers, fills, strokes, - arrows, and local slot palettes must refer to these token IDs -- `reference_priority`: use `reference_image_primary`; paper concepts only - adapt terminology and scientific meaning -- `line_style`: outline color, line weight, corner radius, connector style -- `shadow_style`: shadow softness, direction, opacity, and when to avoid shadow -- `viewpoint`: flat, slight isometric, front-facing card, or mixed -- `icon_complexity`: simple, medium, or detailed, with size rules -- `background`: transparent, white, paper tint, or flat color -- `visual_density`: expected density for macro panels, meso cards, and micro icons -- `content_fill_target`: default 88%-95% useful visual content per slot -- `margin_policy`: maximum pure empty margin, usually 10%-12% on any side -- `background_fill_strategy`: how background detail, card surfaces, or texture - extend near the canvas edge without changing scientific meaning -- `icon_scale_rule`: micro icons should be large and full-frame, not tiny objects - centered on a blank canvas -- `card_density_rule`: meso cards should use full-bleed card surfaces and - supporting detail rather than sparse white panels -- `font_layer_rules`: font family or style for editable labels, minimum size, - weight, and label placement rules -- `image2_text_policy`: none, decorative_only, or editable_overlay_required -- `forbidden_style_drift`: styles, colors, or rendering effects to reject - -## Prompt Inheritance - -Every image2 prompt must reference both: - -- the relevant `slot_inventory` or `figure_program.json` slot entry -- the `reference_style_profile.json` fields and color token IDs that control - palette, viewpoint, line style, density, background, and text policy -- the local `reference_slot_crops/.png` crop path used by the prompt - planner - -Do not let each prompt invent a new style. If a block needs a special treatment, -record the exception in the style sheet or in that slot's `style_override` field -inside `figure_program.json`. - -## Text Rules - -The style sheet controls what text image2 may create: - -- `none`: no visible text -- `decorative_only`: tiny non-critical marks, unreadable UI strokes, or vague - card headings are acceptable -- `editable_overlay_required`: all meaningful text is added by the PPTX layer by - default; Lark or SVG text layers are only for explicit collaboration or vector - export needs - -Critical scientific text is never generated by image2. This includes method -names, module labels, formulas, variables, metrics, axis labels, branch labels, -panel IDs, citations, and numeric results. - -## Acceptance - -Before prompting, check that the style sheet is specific enough to prevent: - -- inconsistent icon shape language -- mixed 2D and 3D perspectives without intent -- unrelated color palettes across slots -- unplanned text inside generated blocks -- visual density that makes some slots look unfinished or overcrowded -- undersized icons or cards with large blank canvas -- prompt language that asks for generous margins or small centered objects diff --git a/codex-skills/research-figure-making/references/visual_framework_figures.md b/codex-skills/research-figure-making/references/visual_framework_figures.md deleted file mode 100644 index 4d3ab04..0000000 --- a/codex-skills/research-figure-making/references/visual_framework_figures.md +++ /dev/null @@ -1,245 +0,0 @@ -# Visual Framework Figures - -## Summary - -Use this route for system overviews, model architecture figures, method pipelines, -algorithm diagrams, Figure 1 style figures, and PPT-ready framework diagrams. - -## Principle - -Build the figure as an editable composition whose visual mass comes from -AI-generated image blocks. Do not generate the full figure as one bitmap unless -the user explicitly wants a non-editable concept mockup. - -This route is not a vector-only diagram route. For image-rich framework -requests, PPT elements are the default structure layer: containers, arrows, -labels, formulas, legends, and panel IDs. SVG/Lark are optional auxiliary routes, -not the default main editable source. The visual content must come from -generated slot-level image blocks unless the user explicitly requests a -vector-only figure or image generation is unavailable and the user accepts a -fallback. - -Target composition: - -- 60%-80% visual area: AI-generated image blocks -- 20%-40% structure layer: editable arrows, labels, formulas, panel letters, - grouping boxes, small captions, legends, and callouts -- For detailed paper system figures, default to 25-50 generated non-arrow image - blocks. Simple figures may use 15-25 blocks; complex system figures may use - 40-50. Arrows, connector lines, dashed loops, transition arrows, panel frames, - and titles are PPT controls and do not count as image blocks. - Do not stop at 4-10 macro blocks unless the user explicitly asks for a quick - draft. - -## Process - -1. Start from the paper-derived figure brief. -2. If the user provides a full reference image, treat it as - a visual blueprint and follow `reference_image_alignment.md` before writing - block prompts. In `reference-primary` mode, split reference elements into - `image_slots`, `ppt_arrows`, `ppt_shapes`, and `text_regions`. -3. Create the positioning and control artifacts before generation and delivery: - - **Stylist**: write `reference_style_profile.json` and `style_sheet.md` or - `style_sheet.json` using `stylist_stage.md` before image2 prompting. - - **Locator**: write `layout_plan.json` before PPT composition. This file - records reference-guided normalized coordinates. A VLM may estimate these - coordinates, but it must not write arbitrary PPT code or generate the - figure as one bitmap. - - **Program**: write `figure_program.json` using `figure_program.md` before - composition or layout scripting. - - **Reference Text Layer**: write `reference_text_geometry.json`, - `text_program.json`, and `text_alignment_report.json` before PPT - composition. Text size, position, color, and hierarchy must follow the - reference image. Do not apply a default publication-width or minimum-font - rule that overrides the reference. - - **Complexity Spec**: write `slot_visual_spec.json` before prompt - planning. It records reference crop objects, foreground subject, - secondary objects, micro details, background fill elements, scientific - mechanism detail, required visual complexity, and forbidden - simplifications. - - **Prompt Planner**: write `reference_slot_prompt_brief.json` and - `slot_prompt_plan.json` before image generation. The default route is a - VLM/API prompt planner that inspects the reference image, local slot crop, - and `slot_visual_spec.json`, then outputs a concrete `image_prompt_core` - for every non-arrow image slot. - - **Critic**: write `critic_report.md` or `critic_report.json` using - `critic_stage.md` before final delivery. -4. Divide the framework into three levels: - - **Macro panels**: layout containers only, such as current context, - conditioning, speculation, gate, cache, or resource library. - - **Meso cards**: paper-specific functional units inside macro panels, such - as a scene card, state board, anchor pin, candidate branch card, cache card, - or evaluation card. - - **Micro icons**: small reusable objects, status tokens, metrics, archives, - model icons, memory chips, option markers, and visual cues. -5. Generate image blocks mainly at the meso-card and micro-icon levels. Macro - panels should usually be drawn by the editable composition layer, not produced - as large AI images. -6. For each slot, first write the slot's `slot_visual_spec.json` entry. Then - let the prompt planner write the slot's `image_prompt_core` from paper - content, reference-local style, slot geometry, and the visual complexity - spec. Then write 3-5 targeted prompts using `image_block_prompting.md`. - Each final prompt must inherit its slot entry from `figure_program.json`, its - `image_prompt_core` from `slot_prompt_plan.json`, and style constraints from - the style sheet. Allow only very small non-critical decorative text. Key - scientific labels, formulas, metrics, variables, arrows, and panel numbers - must be added by the editable composition layer. -7. Generate multiple candidates per block. Prefer consistent aspect ratios and - visual grammar: same camera angle, line weight, material, lighting, and color - family. Reject simple standalone icons for normal slots, local reference crop - mismatches, and style drift. -8. Select blocks by scientific fit first, style second. Reject beautiful blocks - that imply a wrong mechanism. -9. Enhance, pad, or regenerate selected blocks if needed. See - `enhancement_pipeline.md`. Do not crop semantic content. If the generated - block does not match the slot ratio, prefer regeneration or pad-to-fit over - cropping. -10. Write `asset_quality_report.json`, `asset_complexity_report.json`, - `asset_visual_review.json`, selected asset contact sheet, and candidate - contact sheet. Reject or regenerate blocks below 85% useful content fill, - with pure empty margin above 10%, with cut-off content, with unresolved - whitespace issues, with `too_simple`/`generic_icon`, or with - `reference_crop_ignored`/`style_drift`. -11. Assemble in `editable_composition.pptx` by following `figure_program.json`. - Add all scientific labels, arrows, connector lines, and dashed loops as PPT - editable objects. Text objects must come from `text_program.json`, with - geometry and relative font height bound to `reference_text_geometry.json`. - Use contain-fit for image blocks; do not crop semantic content. -12. If a reference image was provided, compare the assembled result against it, - write `visual_critic_iter_0.json`, and automatically adjust large coordinate - mismatches before final export when the fix is mechanical. -13. Run the critic stage. Fix unresolved major failures before final export. -14. Export a review PNG plus final PDF and 600 DPI raster file. Export SVG only - when the user, venue, or downstream Illustrator/Inkscape workflow explicitly - needs it. - -## Required Artifacts - -For image-rich system figures, leave these artifacts in the output directory: - -- `slot_inventory.json` or `slot_inventory.md` -- `reference_geometry.json` -- `reference_controls.json` -- `reference_style_profile.json` -- `style_sheet.md` or `style_sheet.json` -- `input_manifest.json` -- `layout_plan.json` -- `figure_program.json` -- `reference_text_geometry.json` -- `text_program.json` -- `text_alignment_report.json` -- `slot_visual_spec.json` -- `reference_slot_prompt_brief.json` -- `slot_prompt_plan.json` -- `reference_slot_crops/.png` for every non-arrow image slot -- `prompts.md` -- generated slot assets -- `asset_quality_report.json` -- `asset_complexity_report.json` -- `asset_visual_review.json` -- selected asset contact sheet -- candidate asset contact sheet -- `editable_composition.pptx` -- final PDF/PNG export -- `visual_critic_iter_0.json` -- alignment review -- `critic_report.md` or `critic_report.json` - -Do not mark the task complete if the only visual artifact is a single generated -full-diagram image, a pure SVG/PPT shape diagram, or a screenshot without -slot-level image assets. Do not mark the task complete with unresolved major -critic failures. - -## Prompt Requirements - -Each prompt should include: - -- paper concept being visualized -- slot function from `slot_prompt_plan.json` -- VLM-planned `image_prompt_core` -- slot visual spec fields: foreground subject, secondary objects, micro - details, background fill elements, scientific mechanism detail, and forbidden - simplifications -- visual metaphor -- desired medium or style -- viewpoint and composition -- background and transparency needs -- target slot aspect ratio and fill requirement -- style sheet constraints -- figure program slot ID -- forbidden elements, especially critical text, fake numbers, fake axes, fake - formulas, and incorrect symbols - -Prompt pattern: - -```text -Create a dense mini scientific scene/card for [paper concept]. Show [visual -metaphor] representing [paper-specific function]. Recreate the local reference -crop object first, then adapt minor details to the paper concept. Foreground: -[foreground_subject]. Secondary objects: [secondary_objects]. Micro details: -[micro_details]. Background fill: [background_fill_elements]. Mechanism: -[scientific_mechanism_detail]. Style: polished academic paper illustration, -consistent line weight, crisp edges, reference-derived color palette, high -detail. Composition: canvas aspect ratio must be exactly [ratio]. Use a -full-frame composition with a large complete subject, minimal blank canvas, and -supporting detail close to the edge. Useful visual content should fill 90-97% -of the canvas, minimum 85%. Include 2-5 layered objects and edge-to-edge -support detail; do not create a standalone pictogram, simple centered icon, or -single object on a blank canvas. Keep key content complete and uncut inside the -safe area without adding a large blank border. No cut-off edges, no object -extending outside the frame, no tiny centered object. Allow at most tiny -decorative placeholder text; no critical labels, equations, numbers, axes, or -fake charts. -``` - -## Layout Patterns - -Choose based on the paper's method, not habit: - -- **Left-to-right pipeline**: sequential data or inference flow -- **Two-stage split**: offline/online, training/inference, construction/evaluation -- **Hub-and-spoke**: central model or memory with multiple interacting modules -- **Loop**: iterative refinement, feedback, self-correction, active learning -- **Stacked panels**: architecture on top, data/results/case below -- **Before/after**: baseline vs proposed mechanism or failure vs correction - -## Editable Layer Rules - -Always add these outside AI images: - -- module labels and method terminology -- arrows and arrow labels -- formulas, tensor shapes, metric names, table values -- panel letters such as A/B/C -- legends and color keys -- section dividers such as training, inference, evaluation, or deployment - -For reference-primary figures, editable text must be planned through -`reference_text_geometry.json` and `text_program.json`. Match the reference -image's text bbox, center, relative height, color, and hierarchy. Do not -introduce a default `paper_double_column` or fixed minimum publication font -threshold unless the user explicitly asks to prioritize readability over -reference matching. - -Use simple but precise labels. If the paper's terminology is long, keep the full -term in the caption and use a short visible label in the figure. - -## PPT-First Routing - -- Use PPTX as the default main editable source for image-rich framework figures. -- Put generated image blocks into PPT slots with contain-fit, no semantic crop. -- Draw containers, arrows, labels, formulas, legends, panel IDs, dividers, and - grouping boxes as editable PPT objects. Do not generate arrows, connector - lines, dashed loops, or transition arrows as image2 PNG assets. -- Use native PPT charts or PPT shapes for simple editable data panels when this - does not compromise data correctness. -- Use SVG only as an optional intermediate or export when a venue asks for vector - files, a Python data figure needs vector embedding, or the user explicitly asks - for SVG/Illustrator/Inkscape editing. -- Use Lark whiteboard only when cloud collaboration is explicitly more important - than PPT supervisor editing. - -Do not deliver only SVG, PDF, PNG, or a browser screenshot for an image-rich -framework figure unless the user explicitly accepts that the main editable PPTX -will not be produced. - diff --git a/docs/architecture.md b/docs/architecture.md index bf376f3..c4a2a7b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,69 +1,105 @@ -# Summary +# Summary -ResearchFigureStudio is organized as a small deterministic pipeline, not a patched copy of LiveFigure. The borrowed ideas are reference-image-based positioning and AutoFigure-style structured control localization: the system writes overlay/candidate JSON for visual elements, then renders editable PPT objects from deterministic programs. +ResearchFigureStudio has one product goal: let Image2 generate a scientifically faithful complete paper figure, then reconstruct that approved image into editable PowerPoint structure. -## Architecture +## Product architecture + +```mermaid +flowchart LR + P["Paper"] --> G["PaperGroundTruth"] + R["Optional visual references"] --> B["Full layout blueprint and Prompt"] + G --> B + B --> I["Image2 complete raster figure"] + I --> Q["Scientific, text, topology, and aesthetic review"] + Q --> A["Approved selected_image.png"] + A --> D["Image-to-PPT reconstruction program"] + D --> PPT["Editable PPTX"] + PPT --> V["Deterministic and optional VLM QA"] +``` + +The semantic path never passes through OCR alone. OCR may help recover geometry from an image, but exact labels, entity IDs, relations, and evidence links are restored from `paper_review.json` and `figure_specification.json` before compilation. + +## Repository map ```text -Input Loader - -> Paper Analyzer - -> Reference Analyzer - -> Control Localizer - -> Arrow Stylist/Router - -> Stylist - -> Layout Locator - -> Figure Program Builder - -> Prompt Planner - -> Asset Generator - -> Asset Reviewer - -> PPT Compiler - -> Exporter - -> Visual Critic - -> Validator +.codex-plugin/ Codex plugin manifest +skills/research-figure-studio Thin agent instructions; no machine paths +rfs/ Installable, agent-independent Python engine + contracts/ Paper semantic contracts and stable schemas + providers/ VLM and external model integrations + analysis/ Paper, layout, and text analysis entry points + planning/ Paper and rebuild planning entry points + generation/ Reference and slot-asset generation entry points + workflows/ Product-level workflow entry points + composition/ PPTX compilation and fallback preview rendering + evaluation/ Deterministic rebuild quality checks + paper_to_image/ Paper review, grounding, planning, image candidates + coevolution/ Experimental creator/judge research + paper_to_editable.py Product workflow orchestrator + semantic_contract.py Paper truth → editable object bindings + editable_rebuild.py Reference image → editable figure program/PPTX + ppt_compiler.py Deterministic PowerPoint renderer + rebuild_visual_critic.py Deterministic rebuild QA +tests/ Unit, integration, and future end-to-end tests +benchmarks/ Real product acceptance cases and metrics +experiments/ Research prototypes and historical scripts +docs/ Architecture, workflows, decisions, roadmaps +scripts/ Reusable installation/maintenance scripts only ``` -## Modules - -- `rfs/input_archive.py`: copies paper and reference into `out/inputs/` and writes `input_manifest.json`. -- `rfs/input_loader.py`: extracts text from source documents. -- `rfs/paper_analyzer.py`: creates paper-grounded figure brief. -- `rfs/reference_analyzer.py`: creates 25-50 non-arrow slot inventory from the reference image geometry and paper figure goal; it also detects arrow/control candidates, writes overlays, and emits `reference_control_candidates.json` / `reference_controls.json`. -- `rfs/arrow_router.py`: assigns reference-preserving arrow aesthetics, bundle metadata, line softness, and route QA reports without changing reference-image flow logic. It also provides reference-constrained orthogonal fallback routing for missing or explicitly fallback-allowed connectors only. In aesthetic mode it records original paths and applies reference-tunnel bundle offsets only for explicitly opted-in routes. -- `rfs/stylist.py`: writes the style sheet before image prompting. -- `rfs/layout_locator.py`: creates `layout_plan.json`; heuristic mode is local, VLM mode returns JSON coordinates only. -- `rfs/program_builder.py`: creates `figure_program.json`, the single source for PPT compilation, preserving control source/target anchors and normalized routes from `reference_controls.json`. -- `rfs/prompt_planner.py`: creates `reference_slot_prompt_brief.json` and `slot_prompt_plan.json`; default VLM mode inspects the full reference image plus local slot crops and writes `image_prompt_core` for each slot. Per-slot calls can run in parallel via `--prompt-plan-workers`. -- `rfs/asset_generator.py`: generates 1-5 candidates per slot, selects the best block, and writes contact sheets and QA metrics. -- `rfs/asset_reviewer.py`: performs heuristic or VLM visual review of selected assets. -- `rfs/ppt_compiler.py`: renders editable PPTX from `figure_program.json` with contain-fit images and editable labels/arrows/groups; multi-segment arrows and dashed loops are rendered as PPT connector shapes, not image assets. -- `rfs/exporter.py`: exports PDF/PNG where local tooling supports it. -- `rfs/visual_critic.py`: compares reference and final render; VLM mode can propose JSON coordinate corrections and arrow-only patches without rewriting the whole figure. -- `rfs/validator.py`: blocks delivery when required artifacts or hard constraints are missing. - -## Non-Goals - -- Do not import LiveFigure as a dependency. -- Do not let a VLM write arbitrary PowerPoint code. -- Do not generate a single full architecture image and crop it into pieces. -- Do not make SVG the main editing source. -- Do not bake critical scientific text, formulas, arrows, or labels into image blocks. -- Do not count arrows, connector lines, dashed loops, or transition symbols as generated image slots. - -## Data Contracts - -- `slot_inventory.json`: paper concept plus candidate slot metadata. -- `reference_control_candidates.json`: AutoFigure-inspired boxlib-like candidate list for arrows, connectors, loops, and branch routes. -- `slot_overlay.png`: visual overlay of detected slot IDs for human/VLM binding. -- `reference_control_overlay.png`: visual overlay of control candidate IDs such as `AR01`. -- `reference_controls.json`: bound editable PPT controls with `source_id`, `target_id`, anchors, path, style token, and render policy. -- `arrow_style_profile.json`: reference-first arrow style rules, fallback routing policy, and routing algorithm metadata for main flow, branch, convergence, feedback loops, and module flow. -- `selected_arrow_routes.json`: selected route/style assignments, bundle IDs, lane indices, reference-lock status, route generation status, and routing algorithm. -- `arrow_quality_report.json`: crossing, bend, obstacle-overlap, and aesthetic-score diagnostics for PPT connectors. -- `composition_quality_report.json`: records PPT arrow connector type, halo usage, route style, and editability alongside slot composition checks. -- `layout_plan.json`: normalized positions, panels, arrows, and z order. -- `figure_program.json`: final composition program consumed by the PPT compiler. -- `reference_slot_prompt_brief.json`: per-slot paper concept, local reference role, geometry, and function briefing sent before prompt planning. -- `slot_prompt_plan.json`: VLM-generated per-slot prompt plan, including `image_prompt_core`, `must_show`, and `avoid_showing`. -- `asset_quality_report.json`: fill/margin/ratio/cutoff/candidate-selection metrics. -- `asset_visual_review.json`: semantic and visual review of selected image blocks. -- `visual_critic_iter_*.json`: reference alignment and optional coordinate correction proposals. +The current package remains under `rfs/` to avoid a disruptive all-at-once move. A future `src/rfs/` migration should be a packaging-only change after the paper-to-editable contracts and benchmarks stabilize. + +## Authority model + +| Concern | Source of truth | +|---|---| +| Scientific entities and exact labels | `paper_review.json`, `figure_specification.json` | +| Scientific relations and evidence | Paper semantic contract | +| Entity roles, provenance, and evaluation boundaries | Normalized evidence-backed contract completion | +| Paper-to-image draft node and connector geometry | `layout_blueprint.json.semantic_plan` | +| Complete visible raster result | Approved `selected_image.png` generated by Image2 | +| Reconstructed layout, controls, and style | Approved complete image plus paper semantic contract | +| Editable object geometry | `figure_program.json` | +| PowerPoint rendering | `ppt_compiler.py` | +| Production eligibility | Candidate, semantic, visual, and PPTX validation reports | + +The scientific contract cache is keyed by paper hash, planner model, compiler +version, and scientific domain profile. Canvas ratio and display-language +preferences are intentionally excluded because they are rendering concerns and +must not produce competing scientific interpretations of the same paper. + +Topology normalization also owns role-consistent branch heads and shortcut +removal, while deterministic template rendering owns exact nested connector +anchors. Candidate review measures raster enrichment inside the detected active +blueprint region and records both active-region and whole-canvas ratios. + +## Stable workflow boundaries + +- `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. +- `paper-to-image`: produces complete Image2 figures by default, including visible text and arrows, then selects only candidates that pass scientific and visual review. +- `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. +- `paper-to-editable`: defaults to `rebuild`: generate the complete figure first, then reconstruct the approved image. +- `paper-to-image`: always produces a complete Image2-owned raster figure; post-generation text/connector overlays are not supported. +- Topology-aware initialization uses benchmarked blueprint-conditioned full redraw for feedback, linear, branch, multimodal, and dense-multiframe figures; unbenchmarked topologies remain on direct generation. +- `make-framework --slot-source paper-native`: consumes a validated semantic contract, generates a content-free seed, expands real entities into grounded facets, and compiles slot assets plus editable structure. +- Plugin skill: selects and invokes workflows; it does not contain the engine or hardcoded repository paths. +- Benchmark layer: independently scores paper-to-image scientific/visual quality and image-to-PPT fidelity/editability. + +## Production rules + +- Failed candidates never become `selected_image.png`. +- Whole-image Image2 candidates must pass visible-label, scientific-entity, relation-direction, topology, information-density, and aesthetic review before VLM approval can count. +- Engineering previews are opt-in and never production-approved. +- The complete Image2 source visibly contains critical text and relations; the later reconstruction must recover them as editable PPT objects wherever possible. +- Paper-native geometry/style seeds and slot assets belong only to the optional framework experiment. +- PowerPoint PDF/600-DPI PNG export runs in a separate process with a hard timeout so COM cannot stall the whole workflow indefinitely. +- Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. +- Deterministic completion is graph-scoped: rich VLM contracts only gain overview-local or directly connected evidence-backed entities, while heuristic contracts grow from overview seeds or the largest connected method chain. Late appendix-only diagram labels, ungrounded innovations, and input shortcuts that bypass an explicit intermediate node are removed before planning. +- `semantic_blueprint.py` compiles 2-16 visible contract entities into normalized graph geometry. It ranks DAG layers, places source-only conditioning nodes immediately before their targets, assigns distinct fan-in/fan-out ports, routes skip-layer edges around intermediate nodes, and sends feedback loops through an outer lane. +- Benchmark entity matching uses both label fidelity and relation consistency so relation labels and variable suffixes cannot steal mappings from real contract entities. +- VLM correction is optional and bounded; deterministic reports remain available offline. +- Historical sample-specific scripts stay under `experiments/legacy_reference_rebuilds/` and are not imported by production code. + +## Planned package refinement + +As interfaces stabilize, split `rfs/` internally into `contracts`, `providers`, `analysis`, `planning`, `generation`, `composition`, `evaluation`, and `workflows`. Do this incrementally with compatibility imports, not as a mass rename that obscures behavior changes. diff --git a/docs/code-structure.md b/docs/code-structure.md new file mode 100644 index 0000000..bb18e97 --- /dev/null +++ b/docs/code-structure.md @@ -0,0 +1,68 @@ +# Paperimage 代码结构 + +本文是仓库中代码归属的唯一导航页。它描述的是**应修改的位置**，而不是 +Python 历史导入路径；历史路径仍由兼容层保留。 + +## 产品主线 + +```text +论文 / 参考图 + -> contracts（科学语义） + -> analysis + planning（理解和规划） + -> generation（局部图像资产） + -> composition（figure program 与 PPTX） + -> evaluation（验证与报告） +``` + +`workflows` 只编排这些阶段。`providers` 只负责外部模型/API。论文到候选图 +的领域实现保留在 `paper_to_image`；它生产并消费上述稳定合同。 + +## 目录职责 + +| 状态 | 位置 | 负责内容 | 新代码规则 | +|---|---|---|---| +| Production | `rfs/contracts` | 语义合同、绑定与兼容转换 | 合同字段先在此定义和验证 | +| Production | `rfs/analysis` | 论文、布局、控件、OCR 与文本分析 | 不编译 PPTX | +| Production | `rfs/planning` | 设计、布局和语义规划 | 不调用资产 API | +| Production | `rfs/generation` | 候选图、槽资产和资产策略 | 不拥有 PPTX 几何 | +| Production | `rfs/composition` | figure program、PPTX、预览和导出 | 唯一编译权威 | +| Production | `rfs/evaluation` | validator、视觉检查和 benchmark | 只检查和报告，不修写合同 | +| Production | `rfs/providers` | VLM/API、模型选择、重试 | 不包含业务规划 | +| Production | `rfs/workflows` | 产品级跨阶段编排 | 不复制领域实现 | +| Production | `rfs/professional` | controlled professional DSL | 与普通 rebuild 分开 | +| Production | `rfs/paper_to_image` | 论文 review、grounding、候选图路线 | 保持领域内聚 | +| Experimental | `rfs/coevolution`、`rfs/judge_training`、`experiments/` | Creator/Judge 与研究原型 | 不被默认产品 workflow 隐式导入 | +| Compatibility | 根目录的已迁移 alias | 旧 import 路径 | 仅转发，禁止新增逻辑 | +| Transitional infrastructure | 根目录的 `cli` 等 | 尚未完全下沉的公共入口或实现 | 维护行为兼容；新功能进入对应 canonical 包 | + +## 根目录文件的阅读规则 + +根目录 `rfs/*.py` 不表示一个独立的业务层： + +- 薄 alias（例如 `editable_rebuild`、`ppt_compiler`、`asset_generator`、 + `program_builder`、`layout_planner`）只为外部兼容而保留。 +- `cli.py` 是唯一命令行入口；它不拥有算法或输出合同。 +- 仅 `cli.py` 与历史 `workflow.py` 属于 *transitional infrastructure*。 + 修改其行为时，必须同时在本页对应的 canonical 目录建立或使用明确入口，不能再创建第二套实现。 +- `workflow.py` 是历史 `make-framework` 路线；新增产品编排进入 + `rfs/workflows`。 + +## 命令分类 + +| 分类 | 命令 | +|---|---| +| Product | `paper-to-image`、`paper-to-editable`、`rebuild-editable`、`make-framework`、`fast-framework-prompt` | +| Experimental | `rebuild-editable-pro`、`rebuild-evolve`、`rebuild-editable-eval`、`coevolve-image`、`coevolution-report`、`rebuild-judge` | +| Maintenance / QA | `doctor`、`inspect-pdf`、`benchmark`、`validate`、`presentations-qa` 和参考图库辅助命令 | + +命令名、参数、输出 JSON 以及输出合同是公共兼容面；结构整理不能改变它们。 + +## 修改后的最低验证 + +```powershell +python -m compileall -q rfs +python -m unittest discover -s tests -q +rfs doctor --json +``` + +涉及 workflow、合同或 CLI 时，还必须运行相应离线 smoke 和兼容导入测试。 diff --git a/docs/gpt-reproduction-workflow.md b/docs/gpt-reproduction-workflow.md index e80ba79..38c670f 100644 --- a/docs/gpt-reproduction-workflow.md +++ b/docs/gpt-reproduction-workflow.md @@ -21,7 +21,7 @@ rfs rebuild-editable --reference input.png --out output\demo --asset-mode api -- For higher-quality reference-only reproduction, use the scripted professional workflow. It asks the VLM to generate a controlled Figure DSL that mimics the best one-off rebuild scripts, then compiles that DSL safely: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --repair-rounds 2 --export-preview +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --asset-policy smart-api --repair-rounds 2 --export-preview ``` The evaluation command is: @@ -57,7 +57,7 @@ Existing one-off scripts in `scripts/` are historical examples. Do not treat the Install locally: ```powershell -cd D:\ResearchFigureStudio +cd python -m pip install --upgrade pip python -m pip install -e . ``` @@ -80,7 +80,8 @@ VLM planning uses an OpenAI-compatible chat-completions endpoint: $env:API_BASE='https://your-openai-compatible-provider/v1' $env:API_KEY='' $env:MODEL_VLM='your-vision-language-model' -$env:RFS_REBUILD_LAYOUT_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_DESIGN_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_LAYOUT_MODEL='gemini-3.5-flash' $env:RFS_REBUILD_CONTROL_MODEL=$env:MODEL_VLM $env:RFS_REBUILD_SEMANTIC_MODEL=$env:MODEL_VLM $env:RFS_PROFESSIONAL_REBUILD_MODEL=$env:MODEL_VLM @@ -99,7 +100,7 @@ Never write API keys into source files, docs, fixtures, or committed outputs. ### 1. Start With A Cost-Safe Evaluation -Use crop assets first. This validates layout/control/semantic VLM quality without spending image-generation credits: +Use crop/placeholder assets first only as an engineering check. The evaluator may use crop for cost-safe structure comparison; production/pro runs should use `smart-api` so crop is not inserted as a final PPT asset: ```powershell rfs rebuild-editable-eval ` @@ -136,6 +137,7 @@ rfs rebuild-editable ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild" ` --asset-mode api ` + --asset-policy smart-api ` --layout-mode hybrid ` --control-mode hybrid ` --text-mode ocr ` @@ -215,6 +217,7 @@ A result is acceptable only if: - Complex visuals are slot-level assets under `assets/`. - `rebuild_vlm_validation_report.json` has `status: pass` or only explainable warnings. - `asset_generation_report.json` records API request counts and fallbacks. +- `asset_decision_report.json`, `text_asset_filter_report.json`, and `api_asset_plan.json` explain which slots are API-generated, reused, or rejected as text. - `reference_geometry_overlay.png` and `reference_controls_overlay.png` visually match the reference closely enough for the current stage. For `rfs rebuild-editable-pro`, additionally check: @@ -258,10 +261,10 @@ Use these modes deliberately: rfs rebuild-editable --reference input.png --out output\placeholder --asset-mode placeholder --layout-mode heuristic --control-mode heuristic # VLM structure test without image generation cost. -rfs rebuild-editable --reference input.png --out output\crop --asset-mode crop --layout-mode hybrid --control-mode hybrid +rfs rebuild-editable --reference input.png --out output\crop --asset-mode crop --asset-policy smart-api --layout-mode hybrid --control-mode hybrid # Full quality run. -rfs rebuild-editable --reference input.png --out output\api --asset-mode api --layout-mode hybrid --control-mode hybrid +rfs rebuild-editable --reference input.png --out output\api --asset-mode api --asset-policy smart-api --layout-mode hybrid --control-mode hybrid ``` Default economy behavior: @@ -325,7 +328,7 @@ Action: - Inspect `asset_generation_specs.json` for `generation_aspect_ratio`, `main_subject_fill_target`, and `prompt_subject`. - Regenerate only failed slots with `--regenerate-slots`. -- Do not globally crop assets unless the user explicitly approves a controlled trim experiment. +- Do not use reference crops as final PPT assets in smart-api/pro runs. Crops are context for API generation only. ## Test Commands diff --git a/docs/help-wanted.md b/docs/help-wanted.md index 2000b47..63e80e0 100644 --- a/docs/help-wanted.md +++ b/docs/help-wanted.md @@ -67,8 +67,8 @@ We welcome help on: - `rfs/ppt_compiler.py` - `rfs/visual_critic.py` - `rfs/validator.py` -- `codex-skills/research-figure-making/references/reference_image_alignment.md` -- `codex-skills/research-figure-making/references/figure_program.md` +- `skills/research-figure-studio/references/reference_image_alignment.md` +- `skills/research-figure-studio/references/figure_program.md` ## Desired End State diff --git a/docs/judge-shadow-deployment.md b/docs/judge-shadow-deployment.md new file mode 100644 index 0000000..ca1ecd9 --- /dev/null +++ b/docs/judge-shadow-deployment.md @@ -0,0 +1,42 @@ +# Judge Shadow deployment + +This interface activates only after the training team has produced a checkpoint +and exported JSONL predictions. It does not execute arbitrary checkpoint code +or silently substitute a Bootstrap VLM. + +## Register a candidate + +Create a manifest outside source control. A Shadow candidate needs its model +identifier, dataset version, Git commit and allowed role: + +```json +{ + "model_id": "judge-v1-rc1", + "dataset_version": "judge-dataset-v001", + "roles": ["shadow"], + "git_commit": "" +} +``` + +```powershell +rfs judge register --checkpoint models/judge-v1-rc1 --manifest manifest.json --registry models/judge_registry.json +``` + +`online` and `frozen` roles require `promotion.approved`, +`promotion.hidden_test_passed`, and `promotion.adversarial_passed`. `frozen` +also requires `promotion.shadow_stable`. This prevents a newly trained model +from replacing both Judge roles at once. + +## Evaluate and Shadow compare + +The training/runtime system exports one JSONL row per `example_id`; either the +top-level row or its `prediction` object may contain `scientific_score`. + +```powershell +rfs judge evaluate --checkpoint models/judge-v1-rc1 --dataset datasets/judge-v001 --predictions outputs/judge-v1-hidden.jsonl --suite hidden_test +rfs judge shadow --checkpoint models/judge-v1-rc1 --dataset datasets/judge-v001 --predictions outputs/judge-v1-shadow.jsonl --bootstrap-predictions outputs/bootstrap-shadow.jsonl +``` + +Shadow comparison records all divergent decisions but does not control the +Creator or acceptance path. Promotion remains a human-and-metric decision; +the deterministic scientific Grounding Gate continues to be mandatory. diff --git a/docs/judge_model_training_plan.md b/docs/judge_model_training_plan.md new file mode 100644 index 0000000..6ab08a5 --- /dev/null +++ b/docs/judge_model_training_plan.md @@ -0,0 +1,456 @@ +# 论文框架图质量 Judge 模型训练计划（落地执行版） + +> 状态：v1.1（2026-08-23） +> 定位：本文是「现在、用现有数据、具体怎么做」的执行计划。与两份既有文档互补： +> - `docs/judge_model_training_roadmap.md` —— 总路线 + 决策记录（为什么这么做） +> - `docs/Final_target.md` —— 产品目标蓝图（最终系统长什么样） +> 本文只回答一件事：**先做什么、用什么、做到什么程度算完成。** + +--- + +## 0. 一句话目标 + +本阶段训练一个**论文条件化的共享底座多任务 Judge**，让模型通过同时阅读论文证据与候选框架图，学习判断图是否科学地表达论文方法，以及什么样的构图在信息组织、可读性和审美上更好。 + +本阶段的模型目标是： + +```text +Judge(paper, evidence, figure, preferences) + → scientific grounding + → method representation quality + → figure design quality + → blocking issues and evidence + → preserve / repair + → pairwise preference +``` + +Judge 只负责评价、诊断、排序和提出可执行的改进建议，**不负责生成框架图**。Creator Agent 的训练不属于本计划，只有 Judge 通过科学门禁后才进入后续独立计划。 + +训练数据仍可覆盖图像和可编辑 PPTX，但两者是先后阶段，不代表本阶段同时训练 Creator： + +```text +论文 Ground Truth + → 像素图（paper-to-image 生成） + → 可编辑图（rebuild-editable 生成 PPTX） +``` + +即同一个底座，支撑两个评价任务： + +| 任务 | 输入 | 输出 | +|---|---|---| +| **图像 Judge**（本计划） | 论文证据/GT + 候选 PNG | 科学方法表达 / 审美偏好 / 视觉质量 / 成对排序 / 阻塞问题 / 修改建议 | +| **可编辑图 Judge**（后续计划） | GT + 目标图 + PPT 对象树 + 渲染预览 | 视觉还原度 / 文字·箭头·分组可编辑率 / 科学保留度 | + +**首版明确不做什么**：不从零训练、不全参微调、不训练 Creator Agent、不把人工好坏标签当成唯一监督、不让模型自行决定总分权重。图像 Judge 未过科学门禁前不开 PPT Judge。 + +### 0.1 训练边界 + +本计划要回答的是“什么样的框架图科学、完整、清晰、好看，以及为什么”，而不是“如何从论文直接生成一张图”。 + +因此本阶段不以最终图片生成率作为目标，也不把 Judge 的高分直接当作 Creator 的训练奖励。后续 Creator 训练必须使用冻结、通过门禁的 Judge，并另行设计论文到语义合同、拓扑和布局的训练任务。 + +--- + +## 1. Judge 的最终形态（三层，先建立心智） + +```text +Trained VLM Judge（7B~10B 多模态，QLoRA 微调） + 只输出分项分数 + 证据引用 + 错误定位 + 建议；禁止自己算总分 + ↓ +Deterministic Grounding Gate（纯代码） + OCR + 实体/关系覆盖 + 禁止内容 + 术语；硬错误直接 gate fail + ↓ +Score Aggregator（代码） + 按 GT 权重算总分与通过状态；硬错误不允许被审美分抵消 +``` + +**为什么必须三层**：你们自己的 AutoFigure 实验已证明——纯 VLM Judge 会放行"美观但科学错误"的图（最终图仍虚构 Encoder/Prediction 等无关模块却判无阻塞）。**科学硬错误漏检是当前第一风险，Grounding Gate 是安全层。** + +--- + +## 2. 现状盘点（已有 vs 缺失） + +### 2.1 已有资产 + +| 资产 | 位置 | 用途 | +|---|---|---| +| 闭环代码 | `rfs/coevolution/`（orchestrator/creator/judges/groundtruth/storage） | 生成候选 → Online Judge → Frozen Judge → 轨迹 | +| Bootstrap Judge | `rfs/coevolution/judges.py::VLMJudge` | 当前 Prompt 版 Judge，只能当基线和数据生产 | +| GT schema | `rfs/coevolution/groundtruth.py` | 已定义 scientific_truth + aesthetic_preferences + weights + thresholds | +| 轨迹 jsonl | 每次 `rfs coevolve-image` 输出 `preference_pairs/repair_trajectories/critique_outcomes.jsonl` | 真实生成轨迹，转训练数据 | +| 已有偏好 | `output/paper_to_image_autofigure/preference_pairs.jsonl` | 2 条人工 A/B（Sen-illion 标注） | +| **papers-dataset** | github.com/Sen-illion/papers-dataset | 1000 元数据 + 996 PDF + **491 条人工粗标注** | +| 审美问卷模板 | `docs/aesthetic_ground_truth_questionnaire.md` | 收集更细的人工偏好 | +| 文档 | `judge_model_training_roadmap.md` / `Final_target.md` | 决策依据 | + +### 2.2 缺失（本计划要补的） + +1. **论文—框架图级训练数据集**（现在只有论文级粗标注） +2. `corruption_generator.py`（主动造"已知错误"科学负样本） +3. `grounding_gate.py`（确定性硬门禁） +4. `rfs/judge_training/` 训练包（schema/dataset_builder/trainer/evaluator/registry） +5. 云端 QLoRA 训练脚本与配置 + +--- + +## 3. 数据资产深度分析（决定第 1 步怎么做） + +### 3.0 数据落盘位置与目录约定（所有脚本据此读写） + +| 用途 | 绝对路径 | 说明 | +|---|---|---| +| 源数据（已 clone） | `D:\文件夹\papers-dataset` | 含 `classified/分类清单.csv`、`classified/{优,好,坏,未筛}/`、`pdf/`、`_raw/` | +| 图级数据集根目录 | `D:\文件夹\Paperimage\datasets\` | 抽图/转换/造错/切分的产物，**需 gitignore** | +| 图资产 | `D:\文件夹\Paperimage\datasets\figures\\fig_.png` | 抽图输出 | +| 训练记录 | `D:\文件夹\Paperimage\datasets\judge-v001\*.jsonl` | 统一 `JudgeExample` 格式 | +| 模型权重 | `D:\文件夹\Paperimage\models\` | 下载的底座 + QLoRA checkpoint，**需 gitignore** | + +> 约定：源数据只读，不写回 `papers-dataset`；一切中间产物进 `datasets/`；`datasets/` 与 `models/` 都不提交到 git（`pdf/` 与论文原图属私有数据）。 + +### 3.1 `分类清单.csv` 精确结构（491 条） + +表头：`分类,风格,优,arxiv_id,标题,备注,被引,arXiv` + +| 字段 | 取值 | 含义 | +|---|---|---| +| 分类 | 坏 / 好 / 未筛 | 论文级三档质量 | +| 风格 | 科研风 / 生动活泼 / 空 | 仅「好」有；两种审美风格 | +| 优 | 优 / 空 | 是否额外标记为优选 | +| 备注 | 自由文本 | 人工原因，含**图级线索** | +| 被引 | 数字 | 引用数 | + +备注里已出现的关键词（可直接转成缺陷标签）： + +- 审美缺陷：`配色不好看 / 配色难看` +- 信息量缺陷：`信息密度低 / 信息密度太低 / 太简单 / 较简单 / 好看，但信息密度低` +- 可读性缺陷：`字体太小，信息密度低` +- 图级优选：`图1：优 / 图2：优 / 图3：优 / 图2更好 / 图3` + +### 3.2 关键差距（必须讲清楚） + +> **现有标注是“论文级审美/风格”标注，不是“论文条件化的图级科学表达”标注。** + +| 维度 | 现有数据能训 | 现有数据不能训 | +|---|---|---| +| 审美质量（优/好/坏） | ✅ 可做正/负例 | — | +| 风格偏好（科研风 vs 生动活泼） | ✅ 可做偏好对 | — | +| 缺陷分类（配色/密度/字体） | ✅ 备注可转标签 | — | +| **科学正确性（虚构模块/关系反转/术语错）** | ❌ 完全没有 | 需另一路数据 | + +仅有“好/不好”不能告诉模型失败原因。正式样本必须同时提供论文证据、结构化方法合同、图中实体/关系和人工原因；否则模型容易把视觉风格误学成科学质量。 + +**科学正确性数据只能用两路补**：① `coevolve-image` 真实轨迹（有 GT 对照）；② `corruption_generator` 主动注入已知错误（标签 100% 确定）。 + +### 3.3 分布估计（以脚本统计为准，约数） + +```text +优 ≈ 50 篇（正例最强信号） +好·科研风 ≈ 90 篇 +好·生动活泼 ≈ 60 篇 +坏 ≈ 200 篇（负例） +未筛 ≈ 50 篇（暂不用于训练，留作待标注池） +``` + +--- + +## 4. 第 1 步：数据工程（最需人力，一半可脚本化） + +### 4.1 要产出的四个数据集 + +| 数据集 | 内容 | 训练用途 | +|---|---|---| +| `fig_quality` | 论文证据 + 图级 优/好/坏 + 科学/叙事/视觉缺陷 | 综合质量与缺陷识别 | +| `fig_style_pairs` | 同一主题「科研风 vs 生动活泼」偏好对 | 风格偏好排序（Pairwise） | +| `fig_scientific` | GT/论文证据 + 正确图 + 注入错误图 + 确定标签 | 科学方法表达与硬错误召回 | +| `fig_feedback` | 修改前图 + 建议 + 修改后图 + 是否有效 | 建议有效性预测 | + +### 4.2 具体动作（按顺序） + +**A. 抽图脚本 `rfs/judge_training/figure_extractor.py`** +- 用 PyMuPDF 遍历 `classified/{优,好/*,坏}/` 的 PDF，提取框架图（按渲染的图片对象 + 图注上下文裁出，过滤小图/图标）。 +- 输出：`datasets/figures//fig_.png` + `figures_manifest.json`（记录页码、bbox、所属 PDF）。 +- 验收：能稳定抽出 >80% 论文的主框架图。 + +**B. 转换脚本 `rfs/judge_training/classified_csv_to_dataset.py`** +- 读 `分类清单.csv`，把论文级标签投影到抽出的每张图： + - `分类=好/优` → positive；`坏` → negative + - `风格` → style 标签；`优` → excellent 标记 + - `备注` 按关键词表 → 缺陷标签（配色/信息密度/字体/太简单） + - `备注` 含 `图N` 的 → 打在图级（更高置信） +- 输出统一为 `JudgeExample` 格式（见 4.3）。 +- 验收：每条记录可回溯到 `arxiv_id + 页码 + bbox + 原始 CSV 行`。 + +**C. 造错误脚本 `rfs/judge_training/corruption_generator.py`（科学数据命根子）** +- 对每张"正确"框架图，用图像编辑 + 文字覆盖主动注入已知错误： + 1. 删除一个核心模块（遮挡/裁掉） + 2. 添加不存在的模块（贴一个假模块卡片） + 3. 反转关键箭头方向 + 4. 替换核心术语（改文字） + 5. 保持美观但篡改事实（只改语义不改构图）→ 这就是 `beautiful_but_wrong` 对抗集 +- 每个注入错误记录：注入位置 + 类型 + 预期标签（`invented_module/relation_reversal/...`）。 +- 验收：注入错误召回率 ≥95%（门禁能抓）。 + +**D. 切分 `rfs/judge_training/split.py`** +- **按 arxiv_id 切分**：train 70% / val 15% / hidden_test 15%，同一篇论文绝不跨集。 +- 另建 8 个对抗集：`beautiful_but_wrong / correct_but_ugly / relation_reversal / missing_core_stage / invented_module / terminology_corruption / style_preference_conflict / reward_hacking`。 + +### 4.3 统一样本格式 `JudgeExample`（`schemas.py`） + +```json +{ + "example_id": "fig-2505.20824-1", + "dataset_version": "judge-dataset-v001", + "ground_truth_id": "arxiv-2505.20824", + "task_type": "method_grounding | quality_classification | style_preference | pairwise_preference | scientific_error | feedback", + "paper_evidence": { + "title": "...", + "abstract": "...", + "method": "...", + "citations": [{"text": "...", "page": 3, "section": "Method"}] + }, + "ground_truth": { + "must_show": ["..."], + "must_not_invent": ["..."], + "entities": ["..."], + "relations": [["source", "target", "relation"]], + "terminology": ["..."] + }, + "candidate_a": "datasets/figures/2505.20824/fig_1.png", + "candidate_b": null, + "labels": { + "scientific_status": "correct|incomplete|incorrect|unverifiable", + "quality": "excellent|good|insufficient|bad", + "style": "research|vivid|null", + "defects": ["配色不好看"], + "scientific_score": null, + "method_coverage_score": null, + "relation_correctness_score": null, + "narrative_clarity_score": null, + "readability_score": null, + "aesthetic_score": null, + "blocking_issues": [], + "missing_required_entities": [], + "ungrounded_entities": [], + "missing_relations": [], + "incorrect_relations": [], + "terminology_errors": [], + "injected_error_type": null, + "preferred_candidate": "A|B|tie", + "preserve": [], + "repair": [], + "confidence": 0.0 + }, + "provenance": {"arxiv_id": "2505.20824", "csv_row": 123, "page": 2, "bbox": [..], "human_audited": true} +} +``` + +### 4.4 规模目标 + +| 版本 | 论文数 | 图数 | 记录数 | 用途 | +|---|---|---|---|---| +| D0 协议验证 | 30~50 | 150~400 | ≥500 单图 + ≥500 配对 | 验证流程，不上线 | +| D1 正式训练 | 300~500 | 3k~5k | 8k~15k | Judge v1 训练 | + +### 4.5 第 1 步里"必须人来"的部分（不可脚本化） + +1. **审核论文—图对应关系**：确认图中模块和关系确实能由论文证据支持。 +2. **审核注入错误**：确认每张错误图“错对了、错干净了”，且没有引入未标注副作用。 +3. **A/B 综合裁决**：分别说明科学表达、叙事、可读性和审美原因，不能只给总偏好。 +4. **硬错误人工复核**：所有高置信科学错误样本人工确认。 + +### 4.6 人工标注协议：不能只标“好/不好” + +每张单图至少分开标注以下维度；总质量标签只作为汇总结果，不能替代分项原因： + +| 维度 | 必填标签 | 作用 | +|---|---|---| +| 科学状态 | `correct / incomplete / incorrect / unverifiable` | 区分正确、不完整、错误和证据不足 | +| 方法覆盖 | 必须模块、缺失模块、允许简化项 | 判断是否完整展示论文方法 | +| 关系正确性 | 正确、缺失、反向或无依据的关系 | 监督输入输出、依赖和训练/推理流程 | +| 贡献突出度 | 核心贡献是否被突出、是否被次要内容淹没 | 学习框架图叙事重点 | +| 叙事清晰度 | 阶段划分、阅读顺序、分组是否清楚 | 学习科学内容如何组织成图 | +| 可读性 | 字体、拥挤、遮挡、箭头、信息密度 | 学习表达质量 | +| 审美质量 | 配色、风格、一致性、视觉层级 | 学习人工偏好 | +| 修复信息 | `preserve`、`repair`、依据和目标区域 | 训练可执行诊断 | + +Pairwise 标注必须使用同一论文或等价科学合同下的 A/B 图，并分别记录： + +```text +preferred_candidate +scientific_reason +method_representation_reason +narrative_reason +visual_reason +confidence +``` + +如果 A 更美观但 B 更科学，不能强行压成单一标签；应分别保存分项偏好，并由硬门禁保证科学错误不能被审美优势抵消。 + +--- + +## 5. 第 2 步：门禁 + 基线 + 底座选型（8G 本地可做） + +### 5.1 写 Grounding Gate v0（`rfs/judge_training/grounding_gate.py`） + +输入 GT（实体/关系/must_show/must_not_invent/terminology）+ 候选图 + OCR 文本 + VLM 抽取的实体关系，输出： + +```json +{ "grounded_entities":[], "ungrounded_entities":[], "missing_required_entities":[], + "incorrect_relations":[], "terminology_errors":[], "blocking_issues":[], "gate_pass": false } +``` + +**硬失败规则**（直接 gate fail，不参与加权）：核心模块缺失 / 新增无依据模块 / 关键关系方向错 / 命中 must_not_invent / 术语被替换 / Judge 无法给出 GT 依据的高置信科学判断。 + +### 5.2 跑 Bootstrap 基线 + +用 `VLMJudge` 把 D0 全量评一遍，得到**未训练基线**分数（后续所有训练版本都对比它）。 + +### 5.3 底座 bake-off（云端小规模） + +候选（7B~10B 多模态，能读长 GT + 高清图 + 多图 Pairwise + JSON 稳定）： +- Qwen2.5-VL-7B-Instruct（首选，中文/长上下文强） +- InternVL2-8B +- MiniCPM-V-2.6（8B） +- LLaVA-OneVision-7B + +各训一小轮，按「JSON 合法率 + 科学错误召回 + 多图能力」排序，**选定后全程锁定**（本数据版本消融结束前不许换）。 + +--- + +## 6. 第 3 步：云端 QLoRA 分阶段微调（租 24~48G） + +### 6.1 训练顺序（图像 Judge，5 阶段） + +| Stage | 训练任务 | 目的 | 数据 | +|---|---|---|---| +| A | 论文条件化结构化 SFT | 稳定 JSON + 论文证据引用 + 方法覆盖/关系核验 + 阻塞识别 | 全量单图样本 | +| B | Pairwise 排序 | 在同一论文/GT 下判断哪张图更科学、更清晰、更符合审美 | fig_style_pairs + 质量对 | +| C | 硬错误加权 | 提高错误样本损失权重，高召回优先 | fig_scientific 注入错误 | +| D | 建议有效性 | 预测"建议是否可执行/有效/有副作用" | fig_feedback | +| E | 分数校准 | 分项分 → 代码聚合总分；置信度校准 | 独立验证集 | + +### 6.2 可编辑图 Judge（不属于 Judge v1） + +Judge v1 只训练论文条件化的图像 Judge。可编辑图 Judge 是后续独立阶段：复用已经通过门禁的底座，再增加「PPT 对象树 + 渲染预览 vs 目标图」样本，评价视觉还原度、文字/箭头/分组可编辑率和科学保留度。 + +该阶段不得和 Judge v1 混合验收，也不得在图像 Judge 通过科学门禁前启动。 + +### 6.3 训练配置（起点，非定稿） + +- QLoRA：4-bit NF4，LoRA rank 32~64，alpha=2×rank，target 全线性层 +（可选）视觉投影 +- lr 1e-4~2e-4，cosine 衰减，epochs 1~3，warmup 3% +- batch 1~2 × gradient accumulation → 有效 batch 16~32 +- 图像 max pixel ≈ 1280×1280（高分辨率科研图必需） +- 每 epoch 存 checkpoint，只留验证最优为候选 + +### 6.4 上线门禁（训练完 ≠ 能用） + +```text +离线训练 → val 选候选 → hidden_test 评测 → 对抗集 → Shadow 影子运行 +→ 与当前 Judge 分歧分析 → 小流量 Online → Frozen 仍用旧稳定版 → 达标后升 Frozen +``` + +**Judge v1 硬指标（不达标不得上线，不许调低阈值）**： + +| 指标 | 门槛 | +|---|---| +| 科学硬错误召回率 | ≥95% | +| 科学硬错误漏检率 | ≤5% | +| 美观但错误对抗集拦截率 | ≥95% | +| 必须方法模块召回率 | ≥90% | +| 虚构科学模块召回率 | ≥95% | +| 关键关系方向准确率 | ≥90% | +| 方法表达完整度与人工评分相关性 | Spearman ≥0.70 | +| Pairwise 排序准确率 | ≥80% | +| 审美偏好排序准确率 | ≥75% | +| 修改建议有效率 | ≥70% | +| 重复评分标准差 | ≤0.03 | +| 新论文跨领域排序 | ≥70% | +| JSON 合法率 | ≥99% | + +隐藏测试必须以**未见论文**为单位，要求模型在没有见过该论文框架图的情况下完成：论文证据理解、图中模块与关系核验、方法表达质量判断和 A/B 排序。仅在训练论文或近重复图片上得分高，不算达到目标。 + +--- + +## 7. 算力与分工 + +### 7.1 算力（已实测你机器：RTX 5060 8GB，CUDA 13.1） + +| 阶段 | 硬件 | 说明 | +|---|---|---| +| 第 1 步数据 | 本机 CPU / 云 CPU | 几乎不用 GPU | +| 第 2 步门禁+推理 | 本机 8G（4-bit 推理）或云端 | 8G 够推理、不够训练 | +| 第 3 步训练 | 云 **24G 起，推荐 48G** | RTX4090/L4/A10(24G)，L40S/A6000(48G) | + +### 7.2 分工 + +| 事项 | 谁 | +|---|---| +| 全部脚本/训练代码/配置/评测 | Kira1（本助手） | +| 审核注入错误 + 拍板 A/B 偏好 | Sen-illion（人工） | +| 租云卡 + 在云上执行命令 | Sen-illion（或给 Kira1 远程访问） | + +### 7.3 人力估算（粗） + +- 第 1 步：脚本 ~3~5 人天（助手），人工审核 ~2~4 人天（你）→ **总瓶颈在此** +- 第 2 步：~2~3 人天（助手） +- 第 3 步：代码 ~2~3 人天（助手）+ 云 GPU 每轮实验几小时~几十小时（算力，非人力） + +--- + +## 8. 风险与防护 + +| 风险 | 防护 | +|---|---| +| Reward Hacking（后续生成系统迎合 Judge 骗分） | 独立 Gate + 冻结对抗集 + Online/Frozen 分离 + 人工抽查 | +| Judge 与后续生成系统共谋漂移 | Judge 先独立训练和冻结；后续不得同周期互证 | +| 审美模板坍缩 | 多风格 Profile + 多样性指标 | +| 科学错误被总分掩盖 | 硬错误 gate fail，不参与加权抵消 | +| 数据泄漏 | 按 arxiv_id 切分 + 图像感知哈希去重 + 冻结 manifest | +| 私有论文泄漏 | 数据/模型目录 gitignore，本地训练优先 | + +--- + +## 9. 名词速查 + +- **Ground Truth（GT）**：人工确认的标准答案（论文事实 + 审美偏好），训练与打分的唯一权威。 +- **VLM / MLLM**：多模态大模型，能同时看图 + 读文字，Judge 底座必须是它。 +- **Backbone（底座）**：选来微调的开源基础模型；"7B"≈70 亿参数。 +- **LoRA / QLoRA**：只训练少量附加参数（LoRA），QLoRA 再把底座压成 4-bit 更省显存。 +- **SFT（监督微调）**：给"输入→标准答案"教模型模仿。 +- **Pairwise / 成对排序**：只判"A 和 B 谁更好"，比绝对打分稳定。 +- **合成硬负例**：主动制造的错误图（删模块/反转箭头），用于监督错误召回。 +- **Grounding Gate（落地门禁）**：独立代码检查层，硬错误直接判失败。 +- **Bootstrap Judge**：当前靠 Prompt 调 API、未训练的 Judge，只做基线和数据生产。 +- **Reward Hacking（刷分作弊）**：生成模型迎合评分器而非满足真相。 +- **Checkpoint**：训练中保存的权重快照，可回滚。 +- **Shadow / Online / Frozen**：影子（只跑不决策）/ 在线（指导生成）/ 冻结（独立验收），三者不能同时换新模型。 +- **VRAM / 显存**：显卡训练专用内存；8G/24G/48G 均指此。 +- **数据泄漏**：同一论文同时进训练和测试集，导致虚高分数。 +- **bake-off**：多个候选模型"打擂选拔"，选定后锁定。 +- **对象树**：PPTX 内部结构化描述（文本框/箭头/分组/坐标），可编辑 Judge 读它判断可编辑性。 +- **PDF 抽图（figure extraction）**：用 PyMuPDF 把 PDF 内嵌插图裁成独立 PNG。 + +--- + +## 10. 立即执行清单（下一步动作） + +1. 冻结论文证据包、科学 Ground Truth 和 `JudgeExample` schema，确保每个科学标签都能追溯到论文证据。 +2. 建 `rfs/judge_training/` 包骨架：`schemas.py / dataset_builder.py / figure_extractor.py / classified_csv_to_dataset.py / corruption_generator.py / grounding_gate.py / split.py`。 +3. 先写并跑通三个不依赖训练 GPU 的环节： + - 论文证据与方法合同构建； + - `classified_csv_to_dataset.py`（分类清单 → 待复核图级数据）； + - `corruption_generator.py`（基于已确认 GT 生成科学负样本）。 +4. 统计 `分类清单.csv` 精确分布，抽图验证，并完成第一批论文—图对应关系人工复核。 +5. 建立 D0：同时覆盖正确图、不完整图、科学错误图、视觉不足图和同论文 A/B 图。 +6. 冻结 D0、manifest 和隐藏测试后，云端 bake-off 选底座，再进入 QLoRA。 + +--- + +## 11. 与既有文档的关系（避免重复） + +- 决策/为什么：`docs/judge_model_training_roadmap.md` +- 产品蓝图：`docs/Final_target.md` +- 闭环用法：`docs/coevolution.md` +- 人工审美收集：`docs/aesthetic_ground_truth_questionnaire.md` +- 本文件：落地执行（做什么、用什么、验收标准） diff --git a/docs/module-map-and-migration.md b/docs/module-map-and-migration.md new file mode 100644 index 0000000..d7482ee --- /dev/null +++ b/docs/module-map-and-migration.md @@ -0,0 +1,198 @@ +# Paperimage 现状模块地图与重构迁移清单 + +> 本文是代码导航和迁移计划，不改变任何运行行为。迁移期间以现有 CLI、输出合同和测试结果为准；旧模块保留兼容导入，直到对应阶段完成验收。 + +## 1. 先记住这一条主线 + +```text +论文/参考图 + -> 语义合同（paper_review / figure_specification） + -> 分析与规划（layout / design / controls / semantic binding） + -> 局部资产生成（按 asset policy） + -> figure_program.json + -> PPTX 编译与预览 + -> deterministic QA / benchmark +``` + +语义合同决定“画什么、文字是什么、关系是什么”；参考图决定“怎么排、什么风格”；本地编译器决定“如何生成可编辑 PPTX”。任何重构都不能改变这三个权责边界。 + +## 2. 当前仓库地图 + +### 2.1 用户可见入口 + +唯一 CLI 入口是 `rfs/cli.py`，对应以下产品/实验命令： + +| CLI 命令 | 当前实现 | 定位 | +|---|---|---| +| `paper-to-image` | `rfs.paper_to_image.workflow.run_paper_to_image` | 论文到候选图与论文 grounding 产物 | +| `fast-framework-prompt` | `rfs.paper_to_image.preparation.run_fast_framework_prompt` | 快速语义合同/提示词路线 | +| `paper-to-editable` | `rfs.workflows.paper_to_editable.run_paper_to_editable` | 产品级组合工作流 | +| `rebuild-editable` | `rfs.workflows.rebuild_editable.rebuild_editable`（旧 `rfs.editable_rebuild` 兼容） | 参考图到可编辑 PPTX 的基线路线 | +| `rebuild-editable-pro` | `rfs.professional.rebuild.rebuild_editable_pro`（`rfs.professional_rebuild` 兼容） | controlled professional DSL 实验路线 | +| `rebuild-editable-eval` | `rfs.rebuild_eval.evaluate_rebuild_vlm` | heuristic/hybrid rebuild 对比实验 | +| `coevolve-image` | `rfs.coevolution.orchestrator.run_image_coevolution` | whole-image Creator/Judge 实验 | +| `coevolution-report` | `rfs.coevolution.analysis.analyze_coevolution_run` | co-evolution 指标汇总 | +| `benchmark` / `validate` / `presentations-qa` | `rfs.evaluation.*`、`rfs.presentations_qa` | 验收、评分、QA，不拥有编译权 | + +### 2.2 目标子包（已存在，优先作为新代码落点） + +| 包 | 清晰职责 | 主要现有入口 | +|---|---|---| +| `rfs/contracts` | 稳定 JSON 合同、语义绑定、兼容转换 | `semantic.py` | +| `rfs/analysis` | 论文、参考图、文字/布局分析入口 | `parse_paper`、`extract_reference_text`、`plan_reference_layout` | +| `rfs/planning` | 论文图、重建设计与布局规划 | `plan_paper_image`、`plan_rebuild_design` | +| `rfs/generation` | 参考图候选和槽级 raster 资产生成 | `generate_and_select`、`generate_assets` | +| `rfs/composition` | figure program/PPTX 编译、预览渲染 | `compile_ppt`、`render_rebuild_preview` | +| `rfs/evaluation` | validator、视觉检查、benchmark、回归评分 | `run_rebuild_visual_quality_check`、`run_benchmark_case` | +| `rfs/workflows` | 面向产品的跨阶段编排 | `run_paper_to_editable` | +| `rfs/paper_to_image` | 论文路线的领域实现与中间合同生产 | `run_paper_to_image`、`run_fast_framework_prompt` | +| `rfs/coevolution` | Creator/Judge 研究实验 | `run_image_coevolution` | +| `rfs/providers` | VLM/API 适配、凭据解析、重试 | `call_vlm_json`、`build_rebuild_vlm_adapters` | + +### 2.3 `rfs/` 根目录的现状 + +根目录仍有约 46 个 Python 文件，主要是三类混合物： + +1. **产品核心但尚未完全归位**：`workflow.py` 与少量历史基础设施；`paper_to_editable.py`、`ppt_compiler.py`、`validator.py`、`editable_rebuild.py` 与 `program_builder.py` 均保留为兼容入口。 +2. **重建专用模块**：`layout_*`、`control_*`、`reference_*`、`rebuild_*`、`text_*`、`arrow_router.py`。 +3. **兼容/基础设施/历史接口**：`workflow.py`、`semantic_contract.py`、`asset_generator.py`、`vlm_client.py`、`presentations_qa.py` 等。 + +这些文件不是都应立即移动；当前最重要的是先建立“权威入口”，禁止新增同职责的第三套实现。 + +## 3. 修改代码时应该改哪里 + +| 需求 | 首选修改位置 | 不要直接改的地方 | +|---|---|---| +| 论文实体、标签、关系、证据合同 | `rfs/contracts/`、`rfs/paper_to_image/review_contract.py` | OCR 输出、PPTX XML、视觉 critic | +| 论文解析/grounding | `rfs/paper_to_image/analyzer.py`、`review.py`、`preparation.py` | `rfs/cli.py` | +| 论文到图的布局蓝图/提示词 | `rfs/paper_to_image/planner.py`、`semantic_blueprint.py` | 资产生成器 | +| 参考图全局设计、层、槽、控件 | `rfs/rebuild_design_planner.py`、`layout_planner.py`、`control_localizer.py` | 语义合同编译器 | +| OCR/文本分组/文本层 | `rfs/reference_text_extractor.py`、`text_grouping.py`、`text_layer*.py` | 资产 API client | +| 箭头、连接器和避让 | `rfs/arrow_router.py`，最终由 figure program 消费 | raster 资产生成 | +| 槽级图片策略和生成 | `rfs/asset_generator.py`、`rfs/generation/` | PPTX 编译器 | +| figure program 生成/校验 | `rfs/program_builder.py`、`rfs/paper_to_image/editable_ppt.py` | 参考图 OCR | +| PPTX/预览渲染 | `rfs/composition/`；根目录 `ppt_compiler.py` 仅作兼容 wrapper | workflow 中复制渲染逻辑 | +| 产品级流程编排 | `rfs/workflows/` | 在 CLI 分支中塞业务逻辑 | +| 质量检查/benchmark | `rfs/evaluation/` | 用 presentations QA 反向改合同或 PPTX | +| VLM/API 调用 | `rfs/providers/`（根目录 `vlm_client.py` 仅兼容） | 在业务模块中散落 requests 调用 | +| professional DSL | `rfs/professional_*.py`（迁移期），目标为独立 professional 子包 | 普通 rebuild 模块 | +| Creator/Judge 演化实验 | `rfs/coevolution/`、`experiments/` | 产品 workflow 默认路径 | + +## 4. 权威性规则 + +- CLI 只负责参数解析、依赖组装、结果打印；业务逻辑必须进入 workflow 或领域模块。 +- `rfs/workflows/` 是跨阶段编排层，不重新实现合同、规划或编译。 +- `rfs/contracts/` 是稳定边界；新增字段先改合同/validator，再改生产者和消费者。 +- `rfs/composition/` 是 PPTX 编译权威；`ppt_compiler.py` 不再增加新功能。 +- `rfs/evaluation/` 只检查和报告，不偷偷修写生产合同。 +- `experiments/` 可以试错，但不得被生产包隐式 import。 +- 同一职责只能有一个“新实现”；兼容文件只能转发并标注迁移目标。 + +## 5. 不改变功能的迁移清单 + +### 当前已落地的 rebuild 权威边界 + +后续定位可编辑重建问题时，优先从下表的 canonical 模块进入；根目录旧模块和 +`rfs.editable_rebuild` 仍是兼容入口，不应新增业务逻辑。 + +| 关注点 | Canonical 位置 | 兼容入口 | +|---|---|---| +| 产品编排 | `rfs.workflows.rebuild_editable` | `rfs.editable_rebuild` | +| 参考图布局、控件、文本分析 | `rfs.analysis.rebuild.{layout,controls,reference_text}` | 原 `layout_*` / `control_*` / `reference_*` 模块 | +| 设计和语义规划 | `rfs.planning.rebuild.{design,semantics}` | 原 rebuild planner 模块 | +| 箭头、文本层、figure program | `rfs.composition.rebuild.{arrows,text,program}` | 原 `arrow_router` / `program_builder` 路径 | +| 槽级资产策略和生成 | `rfs.generation.{rebuild_policy,rebuild_assets}` | `rfs.asset_generator`、`rfs.editable_rebuild` | +| 组合覆盖审计与质量检查 | `rfs.evaluation.rebuild_coverage`、`rfs.evaluation` | `rfs.editable_rebuild` | +| PPTX 编译、预览和导出 | `rfs.composition`、`rfs.composition.rebuild.export` | `rfs.ppt_compiler` | +| controlled professional DSL | `rfs.professional` | `rfs.professional_*` | + +这些边界不改变三条产品路线：`paper-to-image`、`paper-to-editable`、 +`rebuild-editable`。Creator/Judge co-evolution 与 Judge 训练仍保留在 +`rfs.coevolution`、`rfs.judge_training` 和 `experiments/`，不被产品默认路径调用。 + +### Phase 0：冻结边界（当前批次） + +- [x] 建立本文档。 +- [ ] 为每个 CLI 命令补充一条“入口 → 产物 → 权威合同 → 编译器 → QA”链路测试。 +- [x] 给已迁移的根目录兼容模块增加模块级 docstring：标明 canonical target 和禁止新增逻辑。 +- [ ] 在 README 增加本文档链接，作为代码定位入口。 + +### Phase 1：先收口入口，不移动文件 + +- [x] `cli.py` 保留总 parser、dispatch 与结果输出；命令参数注册和 rebuild 参数装配迁到 `cli_commands/`。 +- [x] 为 `rebuild-editable` 建立显式 `rfs/workflows/rebuild_editable.py` facade，内部暂时调用旧实现。 +- [x] 为 `rebuild-editable-pro` 建立显式 `rfs/workflows/professional_rebuild.py` facade。 +- [x] 为 workflow facade 增加稳定入口和长期兼容导入测试。 + +### Phase 2：迁移重建实现（兼容导入） + +- [ ] 将 `layout_*`、`control_*`、`reference_*`、`arrow_router.py`、`text_*` 按职责迁入 `analysis/`、`planning/` 或 `composition/` 的明确子模块。（已迁入 analysis/rebuild：layout/control/reference text；已迁入 composition/rebuild：arrows/text/program。） +- [x] 将 `editable_rebuild.py` 拆成：输入/分析、资产策略、figure program、编译 orchestration；总编排已迁入 `workflows/rebuild_editable.py`，figure program、资产基础操作和 combined coverage audit 已分别归入 `composition/`、`generation/`、`evaluation/`。 +- [x] 根目录旧路径保留 re-export，并加入 deprecation 注释；所有内部 import 已改用新路径。 + +### Phase 3：迁移编译和基础设施 + +- [x] 将 `program_builder.py`、`validator.py`、`exporter.py`、`rebuild_preview_renderer.py` 的权威实现归入 `composition/` 或 `evaluation/`。（`program_builder.py` 已迁入 `composition/rebuild/`；`validator.py` 已迁入 `evaluation/`；rebuild export/coverage canonical boundaries 已建立。） +- [x] 将 `vlm_client.py`、模型解析和重试逻辑归入 `providers/`；根目录 `vlm_client.py` 保留兼容 alias。 +- [ ] 保持 `rfs.ppt_compiler`、`rfs.vlm_client` 等旧导入可用，至少跨一个版本周期。 + +### Phase 4：professional 与实验隔离 + +- [x] 将 `professional_*` 收口为一个显式子包和 workflow facade（旧根模块保留兼容 alias）。`professional_rebuild.py` 的权威编排已迁入 `rfs.professional.rebuild`。 +- [ ] 将只服务研究实验的 co-evolution、legacy rebuild、文献脚本与产品代码隔离。 +- [x] CLI help 和文档明确标记 production、experimental、compatibility 三种状态。 + +### Phase 5：删除兼容层（最后做） + +- [ ] 连续两个版本无内部代码使用旧路径。 +- [ ] 全量 unit/integration/e2e、benchmark smoke、CLI help 通过。 +- [ ] 更新插件 skill、README、architecture 文档。 +- [ ] 单独提交删除兼容 wrapper；不与功能修改混合。 + +## 6. 每次迁移的验收门槛 + +迁移只允许改变 import 路径和目录归属，不允许同时改变： + +- CLI 参数名、默认值和命令行为； +- JSON/Markdown/PPTX 输出合同； +- asset policy、smart-api 默认策略和 fallback 记录； +- 语义绑定、箭头路由、文本可编辑性； +- 生产/实验路线的默认选择。 + +每个迁移 PR/commit 必须至少运行： + +```powershell +python -m compileall -q rfs +python -m unittest discover -s tests -q +rfs doctor --json +``` + +涉及 workflow 或合同时，再运行对应 offline smoke、validator 和 benchmark case。 + +## 7. 推荐的提交顺序 + +1. 文档与模块级说明（无行为变化）。 +2. facade/re-export 与测试（无行为变化）。 +3. 内部 import 迁移（无行为变化）。 +4. 小批量文件移动（每批一个职责域）。 +5. 删除空兼容层（单独提交）。 + +不要把“文件移动 + 逻辑重写 + 输出合同变化”放在同一个 commit；这样才能在任何阶段安全回退。 + +## 8. 当前迁移状态（2026-08） + +生产代码的目标归属已经确定，且不改变 CLI 行为或输出合同。已迁移领域的根目录 +模块是薄 alias；`cli.py` 和历史 `workflow.py` 仍是过渡基础设施。 +新代码使用以下位置： + +| Status | Location | Rule | +|---|---|---| +| Production | `rfs.contracts`, `analysis`, `planning`, `generation`, `composition`, `evaluation`, `providers`, `workflows`, `professional` | New behavior belongs here. | +| Experimental | `rfs.coevolution`, `rfs.judge_training`, `experiments/` | May support research experiments, but is not implicitly imported by a product workflow. | +| Compatibility | root-level `rfs` aliases such as `editable_rebuild`, `ppt_compiler`, and `asset_generator` | Preserve imports only; do not add business logic. | +| Transitional infrastructure | root-level `cli` and historical `workflow` | Preserve behavior while new work goes to the owning production package. | + +See [code-structure.md](code-structure.md) for the concise, current navigation +page. Retain compatibility aliases for at least one version cycle, keep +route-level contract tests, and make any eventual alias removal a standalone +backwards-incompatible change. diff --git a/docs/paper-to-image-reference-learning-plan.md b/docs/paper-to-image-reference-learning-plan.md new file mode 100644 index 0000000..032f683 --- /dev/null +++ b/docs/paper-to-image-reference-learning-plan.md @@ -0,0 +1,241 @@ +# Paper-To-Image Reference Learning Plan + +This plan records the staged roadmap for adding a human-reference learning layer to `paper-to-image`. + +The goal is to let Paperimage learn reusable design knowledge from strong scientific framework figures before generating a new paper figure. The system should learn layout grammar, structural division, information density, hierarchy, color use, text strategy, and connector behavior. It should not copy paper-specific labels, logos, or scientific content from reference figures. + +## Product Principle + +Paper content decides what the figure says. + +Reference learning decides how the figure is visually organized. + +The image model generates candidates from the paper contract plus retrieved design knowledge. + +The critic checks both scientific correctness and alignment with human-quality design standards. + +## Target Pipeline + +```text +project reference figures +-> reference analysis workflow +-> API-backed library build +-> paper_to_image_reference_library.jsonl +-> exemplar selection for the current paper +-> exemplar_style_brief.json +-> paper-to-image prompt +-> image candidates +-> scientific / OCR / template / aesthetic / exemplar-alignment review +-> selected_image.png +``` + +## Reference Figure Storage + +Reference images should live in project-level folders instead of requiring users to pass arbitrary local paths for the built-in reference set. + +Stage 1 uses this local folder: + +```text +reference-figures/ + excellent/ + negative/ + user/ +``` + +The real image files should remain local-only by default because paper figures may have copyright or private-data constraints. The repository should commit reusable structured contracts and documentation, not the original reference images. + +## Stage 1: Build The Analysis Workflow + +Purpose: create the code path and schema for analyzing reference figures, before spending API calls on the actual library. + +Deliverables: + +- A reference-learning schema for reusable design contracts. +- A module such as `rfs/paper_to_image/reference_learning.py`. +- A CLI command such as `rfs analyze-reference-figures`. +- Offline placeholder or fixture-backed tests that verify schema shape and output paths. +- Documentation for expected input folders and outputs. +- Output files named `reference_analysis_records.jsonl` and `reference_analysis_report.json`. + +Important: this stage creates the analysis workflow only. It should not require real API calls to pass tests. + +Commit boundary: one commit for the Stage 1 workflow/scaffold. + +## Stage 2: API-Backed Library Build + +Purpose: run the analysis workflow with a VLM/API to analyze the prepared excellent and negative figures. + +Deliverables: + +- `paper_to_image_reference_library.jsonl` generated from project reference folders. +- A build report recording model, input counts, failures, skipped files, and API status without secrets. +- Safety fields for copied terms, copied objects, and content that must not be reused. +- Optional human-editable review notes for correcting VLM analysis. + +Important: this stage is where API analysis and actual library creation happen. + +Stage 2 command: + +```powershell +rfs analyze-reference-figures ` + --references reference-figures ` + --out output\reference_learning_stage2 ` + --mode vlm ` + --model gpt-5.6-sol ` + --json +``` + +The command writes both `reference_analysis_records.jsonl` and `paper_to_image_reference_library.jsonl`. API credentials come from environment variables already loaded into the shell, not from automatic `.env` parsing in code. + +Commit boundary: one commit for API-backed library building support and any non-private generated contract files that are safe to commit. + +## Stage 3: Exemplar Selection For Current Paper + +Purpose: before `paper-to-image` generation, select relevant design examples from the built library. + +Selection should consider: + +- domain profile, such as AI method, system, dataset, empirical science, or survey; +- module count; +- expected topology, such as linear, tree, loop, tripanel, or dense multimodal; +- aspect ratio; +- information density; +- retrieval, multimodal, feedback, or comparison structure; +- positive versus negative quality labels. + +Deliverables: + +- An exemplar retrieval module. +- `exemplar_selection.json`. +- `exemplar_style_brief.json`. +- Tests for deterministic selection from small fixture libraries. + +Stage 3 command: + +```powershell +rfs select-reference-exemplars ` + --paper-review output\paper_to_image\paper_review.json ` + --library output\reference_learning_stage2\paper_to_image_reference_library.jsonl ` + --out output\paper_to_image\reference_exemplars ` + --selected-template output\paper_to_image\selected_template.json ` + --top-k 5 ` + --json +``` + +`paper-to-image` may also receive `--reference-library` and `--reference-top-k` to write the two Stage 3 artifacts during a run. Stage 4 feeds the resulting brief into image prompts. + +Commit boundary: one commit for exemplar selection and brief generation. + +## Stage 4: Prompt Integration + +Purpose: make the selected design knowledge actively guide image generation. + +The generated prompt should include: + +- layout rules; +- structure and section-division rules; +- information-density target; +- hierarchy and emphasis rules; +- palette and contrast rules; +- text length and label-style rules; +- connector grammar; +- hard anti-copy constraints. + +Deliverables: + +- `paper-to-image` loads `exemplar_style_brief.json`. +- `compile_image_prompt()` includes the brief in a structured section. +- `image_prompt.txt` records the reference-derived design guidance. +- Candidate prompt files include the same brief so all generated candidates receive consistent visual guidance. +- Tests confirm the prompt contains the exemplar brief and anti-copy rules. + +Commit boundary: one commit for prompt integration. + +## Stage 5: Exemplar-Aware Critic + +Purpose: make candidate review judge whether the output matches human-quality design standards learned from the reference library. + +Add a review section similar to: + +```json +{ + "exemplar_alignment": { + "layout": 0.0, + "structure": 0.0, + "density": 0.0, + "hierarchy": 0.0, + "color": 0.0, + "text": 0.0, + "connectors": 0.0, + "copied_reference_content": [], + "score": 0.0, + "passed": false + } +} +``` + +Deliverables: + +- Critic prompt includes `exemplar_style_brief.json`. +- Aggregated review writes an exemplar-alignment report. +- Production gate rejects candidates when exemplar alignment fails, and the violations feed the existing repair round. +- Tests cover pass/fail normalization. + +Commit boundary: one commit for critic integration. + +## Stage 6: User-Provided Style Learning + +Purpose: allow a user to upload one style reference image and ask Paperimage to learn visual style preferences without replacing the project-wide design method. + +Suggested flow: + +```text +user style reference image +-> user_style_brief.json +-> combined_style_brief.json +-> paper-to-image generation +``` + +The user style brief is additive and style-only. It can influence color, line, shape, shadow/depth, typography, texture, icon/object rendering, panel surfaces, and overall mood. It must not decide layout topology, section structure, scientific relations, paper modules, visible labels, or copy reference text/content. + +Deliverables: + +- A command such as `rfs learn-reference-style --style-reference`. +- A `--user-style-brief` option for `paper-to-image`. +- `combined_style_brief.json` that distinguishes project-library design rules from user-provided style preferences. +- Tests for style-only learning and combining project exemplar brief with user style brief. + +Commit boundary: one commit for user-provided style learning. + +## Stage 7: Preference Feedback Loop + +Purpose: preserve human judgments over generated candidates so the system can improve later. + +Deliverables: + +- A `record-paper-image-preference` command that records one human-chosen candidate from an existing `paper-to-image` run. +- `preference_pairs.jsonl` for chosen-vs-rejected candidate comparisons generated from the chosen candidate and the other candidates in the run. +- `human_preference_feedback.json` with the reason, reviewer, tags, candidate metadata, and run metadata. +- Optional future training or tuning path for a stronger judge model. + +Commit boundary: one commit for preference logging. + +## Stage 8: Human-Preference-Aware Selection + +Purpose: separate scientific hard gates from human visual preference so visually stronger candidates are not rejected only because template or exemplar alignment is imperfect. + +Deliverables: + +- A critic schema with `scientific_gate`, `visual_preference`, `selection_status`, `hard_gate_errors`, and `soft_alignment_warnings`. +- Soft template and exemplar alignment scoring; copied reference content remains a hard failure. +- Candidate selection by weighted `selection_score`, with repair source selection preferring visually strong repairable candidates. +- Optional `--preference-history` input for prompt-safe summaries of prior `preference_pairs.jsonl`. +- Reports for human visual preference, selection breakdown, selection policy, and repair source selection. + +Commit boundary: one commit for human-preference-aware selection. + +## First Implementation Rule + +Implementation should start with Stage 1. Each stage must be completed and committed separately before moving to the next stage. + +Do not let later-stage work silently enter an earlier-stage commit. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 190b341..3bc73a3 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -1,6 +1,86 @@ # Summary -`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a content-free architecture blueprint, reference-conditioned Image2 edit generation, and four production quality gates. +`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Image2 itself renders the complete visible figure: exact paper terminology, arrows, modules, illustrations, grouping, and layout. Programmatic label/connector composition is not supported in this route. + +## Three-Minute Fast Contract + +```powershell +rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --json +``` + +The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/raw-semantic-plan caches. Layout, style, prompt, and audit/rebuild contracts are compiled deterministically after the raw semantic graph. A warm paper normally completes in well under a second. Older compiled semantic caches are deliberately not migrated because applying current normalization to an already compiled contract can duplicate or mutate relations. An affected paper therefore needs one fresh VLM compilation; later runs reuse the versioned raw-plan cache and remain stable and repeatable. Cached strings are recursively repaired for common UTF-8 mojibake before any JSON or Markdown artifact is written. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. + +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Quality gates count all Unicode letters and numbers. English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean section aliases are recognized; localized `Figure/Table` prefixes and Chinese `图/表` captions are indexed. Caption prefixes must include a real numeric, Roman, or appendix identifier, so ordinary prose such as `figura editable` is not misclassified. CJK cross-extractor agreement uses character bigrams so per-character spacing differences do not resemble corruption. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. + +If a long fully scanned paper cannot reach the 60% full-document readability gate within the deadline, the workflow may continue only when at least six high-confidence sampled pages cover both Abstract and Method. The result records `semantic_scope: sampled_pages_only`, includes an explicit uncertainty about unprocessed pages, and can never be production-ready. Missing Abstract/Method coverage or weak OCR still stops semantic planning. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. + +Run all or selected paper benchmarks with aggregate reliability metrics: + +```powershell +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast-suite --planner-mode heuristic --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case-id 106_detr_set_prediction --case-id 107_clip_contrastive --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json +``` + +For OCR performance diagnosis, `extraction_report.json` and `benchmark pdf-suite` now record page rendering, text detection, orientation classification, text recognition, and post-processing time separately. RapidOCR defaults are deliberately conservative: detector limit `512`, recognition batch `6`, and an adaptive page-worker cap (4 on 8+ logical CPUs, 2 on 4-7, 1 below 4). Advanced local experiments can override these without changing repository defaults: + +```powershell +$env:RFS_RAPIDOCR_DET_LIMIT = "512" +$env:RFS_RAPIDOCR_BATCH = "6" +$env:RFS_OCR_WORKERS = "3" +rfs benchmark pdf-suite --out output/benchmarks/pdf_profile --ocr-engine rapidocr --json +``` + +Lower detector limits are not assumed to be faster. On dense two-column scans, recognition usually dominates and smaller detector settings can have unstable latency; accept a change only after text agreement, section recovery, caption recovery, confidence, and wall-clock gates all pass. + +Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedule_complete=false` / `ocr_run_complete=false`. Such document models are never written to the global document cache, and scientifically partial scan results are never written to the semantic contract cache; a transient slow or failed OCR run therefore cannot poison later fast runs. + +When a deadline is active, every local OCR page runs in an isolated worker process. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. Single-page RapidOCR retains two intra-op threads; multi-page waves use one thread per worker. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. + +For a fully scanned long paper with no native section signals, the six-page fallback schedule covers pages 1-4 first, then a page near 85% of the document and finally a middle page. Under a deadline, the first wave uses adaptive page concurrency; the two coverage-rescue pages run one at a time with two OCR threads whenever at least 45 seconds remain. This biases scarce OCR time toward the abstract, overview figure, early method details, and likely appendix/conclusion architecture evidence without weakening the hard cutoff. + +In the 180-second fast path with `planner_mode=vlm`, rescue pages require at least 90 seconds remaining so the workflow can preserve a model-call and validation budget. Heuristic planning and PDF inspection retain the 45-second rescue threshold. The effective value is recorded as `ocr_rescue_min_remaining_seconds`. + +Planner and review retries share one absolute provider deadline rather than receiving a fresh timeout per attempt. If the service is slow or unavailable, each request is clamped to the remaining budget, further retries stop at the cutoff, and any later semantic stage switches to the deterministic evidence-grounded fallback. + +Born-digital and OCR-recovered pages also pass through a conservative repeated-margin filter. Short header or footer lines are removed only when the same digit-normalized pattern appears in the same margin on at least three pages and at least one quarter of the document. The report records `repeated_margin_noise_removed_count` so benchmark runs can detect publisher-header and page-number contamination without hiding the cleanup. + +For pages carrying a PDF rotation flag, native blocks are ordered in the unrotated media-box coordinate system and only then transformed into displayed page coordinates. This preserves Abstract-to-Method-to-Figure reading order on 90/180/270-degree pages while keeping overlay coordinates aligned with rendered output. + +Fast VLM planning preserves the paper's original writing system for every visible scientific label. Entity names and relation labels must either occur verbatim in their cited evidence or remain in the same source script; cross-script translations such as `文档编码器` to `Document encoder` fail planning validation and cannot be marked production-ready. Explanatory prose may follow the requested output language, but Image2-visible labels do not. + +For Spanish, French, German, and Portuguese papers, validation also detects same-script English translation. An English-looking visible label that does not occur verbatim anywhere in the paper evidence fails production validation even though both strings use Latin characters. This closes the gap where `codificador de documentos` could otherwise be silently changed to `Document Encoder`. + +Relation labels are optional: if a model supplies a connector label that is not verbatim in the relation's cited evidence, normalization clears the label while preserving the grounded direction and type. Contract normalization also requires an exact scientific label only once when the same evidence-backed concept is both a module and an innovation callout. + +Native PDF lines split by publisher hyphenation are joined inside the same source text block before evidence IDs are created. Known scientific words such as `trans-` + `former` and `de-` + `coder` lose the layout-only hyphen, while unknown long compound parts retain a literal hyphen. The extraction report records `native_hyphenation_repair_count`. + +OCR normalization also repairs fused numbered section boundaries such as `2方法`, `2Metodo`, `3Experimentos`, and `4Conclusion` before heading classification. The same rule covers common French, German, and Portuguese headings. English word segmentation now requires at least two scientific anchor words, preventing a single English-looking suffix from splitting valid non-English words such as Spanish `procesamiento`. + +Production validation treats visible inputs, outputs, and innovations without evidence IDs as hard errors. Before validation, exact entity labels may be grounded deterministically to matching evidence; innovation labels are grounded only when the same evidence also contains an explicit novelty cue such as `we propose`, `we introduce`, or `本文提出`. Otherwise the result remains engineering-only instead of silently presenting an unsupported contribution. + +Repeated-margin filtering interprets page edges in the PDF's semantic orientation. On a 90-degree page, the original header is detected at the displayed right edge and the original footer at the left edge; corresponding mappings are applied for 180 and 270 degrees. Mixed-orientation papers therefore use one header/footer signature space without leaking rotated publisher furniture into evidence. + +Section headings are deduplicated only within their concrete page/block occurrence, not globally by title text. Repeated `Methods`, `Experiments`, or appendix headings therefore create new evidence boundaries after publisher headers have been removed. + +PDF inputs receive a lightweight container preflight before pypdf, Poppler, or PyMuPDF extraction. Missing `%PDF-` headers and missing tail `%%EOF` markers return stable `ValueError` messages immediately; encrypted documents retain the explicit password-required error. This keeps `--json` failures concise and prevents damaged files from entering either cache. + +When `benchmark pdf-suite` runs with a real OCR engine, it now includes fully rasterized two-column and 2-degree skewed two-column pages in addition to the mixed native/scan fixture. These cases enforce column reconstruction, left-column-first reading order, section recovery, caption recovery, and OCR confidence using the actual runtime engine. + +RapidOCR pages render at 84 DPI by default. This retains the detector limit and recognition batch used by the fast path while recovering long Latin-script lines that could disappear at 72 DPI. The generated stress suite includes native and rasterized Spanish papers and records `render_dpi` for each OCR page; the current suite contains 10 parser-only cases and 16 cases with runtime OCR. + +English OCR pages with at least two explicit English section anchors also receive a lexical plausibility check backed by the bundled word-frequency model. A known-word ratio below 0.72 raises a warning and below 0.50 fails extraction even when the OCR engine reports high confidence. The gate is not applied to CJK or unrecognized non-English Latin documents, preventing an English dictionary from rejecting unrelated languages. + +Table captions now seed coordinate-based table regions. Header cells define column anchors, later cells are assigned by row and nearest column, and the document index stores `columns`, `rows`, `cells`, and a full table bounding box. Individual table cells are excluded from narrative evidence and replaced by one row-major structured table evidence record, preventing OCR detection order from turning `Model | Depth | Accuracy` into misleading prose. + +Table reconstruction is intentionally conservative: a candidate needs a plausible labeled header and at least two data rows. Pure numeric rows and lowercase prose fragments such as `are | obtained` are rejected without changing the original block kinds, avoiding false table structure on captions, equations, and surrounding sentences. + +If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. + +`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. +The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. + +When a paper exceeds the evidence character budget, the extractor reserves representative evidence from every page, then prioritizes topology-defining statements and Abstract, Conclusion, Method, Introduction, and Experiments content. This prevents long introductions or appendices from silently excluding later conclusions and framework definitions. ## Production Command @@ -27,6 +107,48 @@ rfs paper-to-image ` --json ``` +Resume a failed candidate with one localized repair instead of generating a new +initial batch: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_repair" ` + --planner-mode vlm ` + --asset-mode image2 ` + --review-mode vlm ` + --repair-source "output\paper_to_image\candidates\candidate_01.png" ` + --repair-rounds 1 ` + --image-retries 0 ` + --ocr-engine off ` + --json +``` + +`--repair-source` skips fresh initial candidate generation. The source is first +reviewed against the current exact-label and scientific contract. If it fails, +the best source is edited once and reviewed again. The request manifest marks +the reused source separately from the real Image2 repair request. Because the +primary route treats labels and arrows as part of the complete Image2 figure, +an explicit external repair source is reviewed and repaired as a whole image; +deterministic overlay mode stays disabled. + +Re-audit an existing complete or failed run without regenerating an image: + +```powershell +rfs benchmark audit ` + --run output/paper_to_image/existing_run ` + --case benchmarks/paper-to-image/cases/101_vit_linear ` + --candidate-image output/paper_to_image/existing_run/candidates/repair_02.png ` + --json +``` + +The audit rebuilds `review_ground_truth.json`, scans the selected image against +quoted paper evidence, exact labels, visible connector topology, and aesthetics, +then writes `benchmark_review.json`, `paper_alignment_report.json`, +`reaudit_repair_plan.json`, and `paper_image_reaudit.json`. If neither +`selected_image.png` nor a selected candidate exists, the highest-scoring +recorded candidate with an existing image is audited automatically. + Never reuse a key that has appeared in chat, source control, an issue, or a terminal transcript. The command records only `api_key_present`; it never records a key value. ## Paper Review Contract @@ -44,18 +166,315 @@ Available profiles: ## Template Contract -Positive references are classified as `arbor`, `linear`, `tripanel`, or `dense-multimodal`. Each becomes a content-free profile containing normalized panels, topology, connector rhythm, density, palette, and forbidden copied content. Automatic selection uses module count, loops/tree structure, multimodality, retrieval structure, and requested ratio. +Built-in templates include `dense-multiframe`, `multimodal`, `branch`, +`feedback`, `arbor`, `linear`, `tripanel`, and `dense-multimodal`. Dense +task/model/data-engine overviews with nested local flows use `dense-multiframe`; +multi-input systems that converge through a modality encoder bank into one joint +space use `multimodal`; compact iterative +generation/feedback/refinement systems use `feedback`; shared-trunk systems with +parallel output heads use `branch`; true search-tree systems use `arbor`. +Positive references are classified into +the closest ratio-based reference archetype and become +content-free profiles containing normalized panels, topology, connector rhythm, +density, palette, and forbidden copied content. Automatic selection uses module +count, loop/tree/parallel-branch structure, multimodality, retrieval structure, +and requested ratio. + +The selected profile is rendered as `layout_blueprint.png`. By default the CLI +uses direct Image2 generation with the semantic blueprint embedded in the Prompt; +Image2 must render the final visible labels, +connectors, modules, and visual explanations itself. Reference-derived profiles +remain free of copied reference text and reference-specific objects. + +## Complete Image2 Ownership and Density + +Image2 is explicitly required to render the complete figure, including exact +visible labels, arrows, arrowheads, loops, modules, illustrations, and grouping. +`overlay_spec.json` only audits what the image must already contain and supplies +semantic guidance to later image-to-PPT reconstruction. It is never painted onto +the primary raster result. + +The generation prompt targets a balanced compact layout: normally 92-96% +canvas-width and 90-95% canvas-height utilization with 2-4% functional breathing +room. It explicitly forbids adding modules, charts, labels, repeated pipelines, +or decorations merely to increase density. Production review still requires at +least 0.10 active-region change from the bare blueprint. The detailed-tile ratio +defaults to 0.80, while feedback-loop figures use 0.75 because necessary return +arrow corridors otherwise create a systematic false whitespace penalty. Once +the applicable threshold passes, extra density is not treated as an improvement; +readability, balance, exact text, and connector correctness decide between +candidates. Flat color-filled cards and label-only boxes still fail. + +The deterministic density audit records `active_detail_grid_stddev` on a 24x16 +active-region grid and `semantic_node_density` on a 4x4 grid inside every +contracted node. Card borders are excluded from the node interior measurement, +so a crisp outline around an empty card does not count as useful detail. When +the global density gate fails, nodes below the 0.80 diagnostic target are added +to `sparse_semantic_nodes` and `repair_regions`; the repair compiler then emits +node-specific Image-2 instructions and localized masks. + +Semantic normalization also separates external inputs from learned internal +conditioning. Class/query tokens and positional embeddings feeding an internal +model move to the module contract with `role=conditioning`. A generic leaf such +as `Head` is removed when it duplicates its upstream head and has one concrete +sibling output; the exact short output label printed in the selected overview +figure is preferred. Topology is re-inferred after late normalization so a +removed false branch cannot leave stale `topology=branch` metadata. + +The connector compiler uses global candidate routing rather than independent +per-edge elbows. It sorts fan-in/fan-out ports by geometry, reserves distinct +lanes, routes skip-layer and feedback edges through outside channels, and scores +every candidate against node obstacles, crossings, near-collinear shared +segments, bend count, and path length. `semantic_plan.route_quality` records: + +- `crossing_count` +- `shared_segment_overlap` +- `node_obstacle_intersections` +- per-connector `lane_assignments` +- the routing algorithm and pass state + +For supported normalized topologies (`linear`, `branch`, `multimodal`, +`feedback`, and `dense_multiframe`), contracts with 2-16 visible entities are +compiled into `layout_blueprint.json.semantic_plan`. The plan records exact +paper labels, normalized node boxes, graph ranks, distinct source/target ports, +relation types, connector labels, multi-point paths, and route styles. The +generic compiler: + +- puts independent inputs in one layer and separates their fan-in ports; +- keeps parallel branch outputs in one layer with separate fan-out ports; +- moves source-only conditioning nodes immediately before their target; +- routes skip-layer edges around intermediate nodes instead of through them; +- routes backward/feedback edges through an outer canvas lane; +- avoids connector/node intersections and near-overlapping long connector lanes; +- assigns fan-in and fan-out ports in geometric order to prevent avoidable crossings; +- records global route quality and per-connector lane ownership; +- refuses to force more than 16 visible nodes into one raster guide. + +When the contract matches a stricter built-in topology, the renderer uses a +specialized semantic layout instead: + +- `feedback`: the forward generation chain, two independent inputs to refinement, + the self-feedback node, and the return loop terminating at feedback; +- `dense-multiframe`: task/model/data-engine panels, separate image and prompt + encoder paths, decoder-to-mask output, the data-engine stage chain, and a + separate model-to-data-engine support arrow. + +This semantic blueprint is still generated from the evidence-backed contract; +it does not copy paper figures or invent labels. It reduces topology drift by +making Image2 preserve an explicit scientific source of truth instead of +inferring node mapping from empty placeholder boxes. + +Entity roles are part of that source of truth. Data sources, modalities, +processing modules, innovations, and outputs are reviewed separately even when +all their words are present. Explicit provenance text is normalized into a +source-to-modality-to-method chain, and multiple named sources may be collapsed +into one evidence-backed source group to avoid redundant nodes and crossed +edges. Explicit internal/external evaluation boundaries and named paper methods +or techniques are repaired deterministically when overview evidence states +them. + +Image-2 candidates must enrich the semantic blueprint rather than merely +restyling its boxes. Candidate review records `blueprint_enrichment_ratio` and +`blueprint_mean_abs_difference`; production defaults require at least `0.08` +changed-pixel enrichment inside the blueprint's detected active figure region. +Reports also retain `blueprint_global_enrichment_ratio`, +`blueprint_active_region_bbox`, and `blueprint_active_region_fraction` for +diagnosis. This keeps the threshold strict without penalizing feedback or +branch layouts that intentionally leave large white margins. The VLM aesthetic review separately scores visual +information density, mechanism visualization, and publication polish. Sparse +text-only nodes therefore fail even when labels, arrows, spacing, and colors +are otherwise correct. + +For branch figures, evidence-backed parallel leaf heads are normalized to the +same output-head role. Relations from an upstream ancestor directly into one +head are removed when they bypass the shared branch point already feeding that +head. If an innovation label is identical to an existing module or output, the +role contract highlights the existing node rather than requesting a duplicate. +For nested feedback layouts, the deterministic blueprint starts the feedback-to- +refinement connector at the inner feedback artifact, not the outer container +boundary. + +Dense multi-frame overview matching uses scientific head terms rather than +requiring exact bare labels. Descriptive variants such as `heavyweight image +encoder`, `prompt encoder`, and `lightweight mask decoder` are normalized to +their canonical overview nodes before the 13-entity SAM-style contract is +rebuilt. This prevents detailed implementation entities from changing the +intended Figure 1 topology. + +The same generic semantic plan can now bypass raster generation entirely: + +```powershell +rfs fast-framework-prompt ` + --paper paper.pdf ` + --out output\fast_result ` + --editable-ppt +``` + +This writes `figure_program.json` and `editable_composition.pptx`. Nodes are +native PowerPoint shapes with exact editable labels; multi-segment routes and +feedback loops are native connectors created behind the nodes. The compiler +also writes `semantic_ppt_report.json` and records every editable node and +connector in `composition_quality_report.json`. The same provenance contract is +used here, so data sources feed their declared modalities instead of appearing +as peer sensor inputs. + +Before semantic layout, contract completion prefers true early model-overview +figures over late scaling, timing, attention, or visualization figures. It also +removes isolated appendix-only diagram labels, innovations that cannot be +grounded to novelty evidence, duplicate aliases such as `MLP`/`MLP Head`, and +input shortcuts that bypass an explicit intermediate representation. + +The general production review and focused topology review execute concurrently +for every contract that declares visible relations, including linear pipelines. +Both default to `gemini-2.5-flash` when no review-specific model environment +variable is set. The focused review uses two passes: first verify all mandatory +directed relations, then exhaustively enumerate visible solid, dashed, curved, +faint, bundled, and cross-panel connectors. The second-pass inventory is +compared deterministically with the contract, so an undeclared visible edge can +only become an invented relation and cannot be reinterpreted as a missing edge. +Their default request timeout is 30 seconds with no automatic retry, preventing +two independent judges from creating a multi-minute sequential stall. Configure +`RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT` when a slower provider requires a larger +budget. Existing candidates can be passed through `--repair-source` so review or +localized repair does not regenerate the correct parts of the image. + +The focused topology review is authoritative only for visible connector +geometry. The default production review now asks one VLM call for OCR, +paper alignment, aesthetics, and structured topology. Every relation must return +`source_departure_side`, `target_arrival_side`, and `arrowhead_contact`; complex +feedback, return-loop, and cross-panel edges cannot pass on a vague description +such as `direct arrow`. If the integrated response omits endpoint evidence, one +short focused topology call is used as a fallback. Set +`RFS_PAPER_TO_IMAGE_FOCUSED_TOPOLOGY=1` only when an independent topology call +should always run. Paper semantics, entity roles, OCR, and aesthetics remain +hard gates. + +OCR review records one item per physical text occurrence in +`visible_text_occurrences`. This lets the deterministic gate reject duplicated +non-repeatable labels even when the critic's distinct-string summary lists the +word only once. Labels in `repeatable_labels` remain explicitly exempt. + +Every candidate review writes `paper_image_comparison_report.json`. This single +report combines exact-text findings, paper-evidence alignment, declared relation +coverage, missing/reversed/bypassed/invented connectors, template alignment, +information density, aesthetics, blocking issues, and an explicit next repair +action while recording that Image2 owns the complete visible raster. + +`paper-to-image` defaults to topology-aware initialization through +`--image-init-mode topology`. Feedback-loop, linear, branch, multimodal, and dense-multiframe figures use +blueprint-conditioned full redraw after passing their five-seed stability +benchmarks; other topologies use direct generation until their own multi-seed +benchmarks support a safer default. Explicit `generate`, `edit`, and +timeout-fallback `auto` remain available. + +The SAM dense-multiframe benchmark is recorded in +`output/evolution/sam_dense_stability_v21.json`: five completed Image2 seeds, +five production passes, and zero incorrect selections. Its blueprint keeps the +Task, Model, and Data graphs separate, anchors panel-level dashed arrows on the +outer panel boundary, and fills only existing cards with mandatory unlabeled +paper-grounded visual cues. The density gate is frozen: a candidate just below +the generic texture threshold may pass only when it also shows material Image2 +enrichment over the engineering blueprint; near-blank blueprint copies remain +blocked. +The CLI default per-candidate timeout is 45 seconds through `--image-timeout`. +Low-level edit and generation timeouts may also be controlled by +`RFS_IMAGE2_TIMEOUT` and `RFS_IMAGE2_GENERATION_TIMEOUT`. Network timeouts stop +the candidate immediately instead of blindly regenerating the entire image. Retryable HTTP or provider failures may +retry up to `--image-retries`; the default is one. + +`RFS_IMAGE_QUALITY` optionally sends `low`, `medium`, or `high` to the +OpenAI-compatible Image2 endpoint. When unset, the provider default is preserved. +The selected value is recorded in `generation_parameters.json` and every request +record in `image2_request_manifest.json`. Quality affects only generation cost +and latency; exact text, paper alignment, topology, density, and aesthetics use +the same production gates at every quality level. + +A full provider read/connect timeout also stops the multi-round repair loop with +`repair_stop_reason=provider_timeout`; another nominal repair round must not +repeat the same provider wait. Fast failures such as HTTP 429 remain +provider failures rather than semantic non-improvement and may use later repair +rounds. + +## Stability Audit + +Production stability requires repeated independent candidates rather than a +single selected best image. Run at least three candidates: + +```powershell +rfs paper-to-image ` + --paper paper.pdf ` + --out output/stability ` + --asset-mode image2 ` + --review-mode vlm ` + --candidates 3 ` + --repair-rounds 0 +``` + +The workflow writes `stability_report.json` with `seed_count`, +`production_pass_rate`, `mean_score`, `worst_case_score`, +`standard_deviation`, per-candidate scores, and failure-mode counts. Repair +candidates and reused `--repair-source` images are excluded because they are not +independent generations. `rfs benchmark score` consumes the generated report +automatically. + +The review records every `unexpected_label`, but only +`unsupported_unexpected_labels` are blocking. A visible detail outside +`required_labels` may remain when the scientific critic or paper-evidence critic +supports it; a label explicitly identified as invented or unsupported still +fails OCR/scientific production gates. Duplicate required labels remain errors, +while repeated non-required implementation details are recorded as +`noncritical_duplicate_labels` instead of causing a false failure. + +`training_flow` and `inference_flow` remain part of the paper contract, but they +are contextual by default. They become mandatory visual priorities only when an +item explicitly sets `required`, `must_appear_in_figure`, or +`visible_label_required`, or when the same content is listed in `must_show` or +`required_labels`. + +The focused topology judge also supports explicit containment semantics. If a +declared operation container visibly contains an evidence-supported repeatable +shared component, an arrow from that nested component to the target may satisfy +the container's outgoing relation. This is limited to repeatable labels inside +the declared source container; it cannot excuse missing cross-panel arrows, +shortcuts, reversed edges, or invented nodes. + +If a stability run is interrupted or one provider request fails, resume it in +place: + +```powershell +rfs paper-to-image ` + --paper paper.pdf ` + --out output/stability ` + --candidates 3 ` + --resume-candidates +``` + +Existing complete candidates are re-reviewed with the current gates, and only +missing candidates are generated. A three-seed stability run also permits +one bounded replacement for a provider-failed seed; successful candidates are +never regenerated by that replacement path. -The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. +When `--aspect-ratio auto` is used, the template keeps its internal normalized +geometry while the generation canvas uses the nearest native Image2 ratio +(`3:2`, `2:3`, or `1:1`). This avoids semantic cropping after generation. ## Production Gates -Every Image2 candidate must pass: +Every Image2 candidate in the primary complete-image route must pass: +- complete-image ownership: visible text, arrows, modules, and illustrations + are present in the Image2 result rather than added by a later overlay; +- target-framework alignment against `target_figure.json`, including direct + caption evidence and sufficient target-caption terminology coverage; - exact-label OCR with no missing, misspelled, duplicate, or copied reference labels; - scientific module and relation checks with no invention or reversal; -- template alignment score of at least `0.72` with no copied reference content; -- aesthetic score of at least `0.75`; +- for every declared visible relation, a second focused topology review that + verifies visible connector endpoints and arrowheads, rejects + bypasses, and writes `topology_critic_report.json`; +- template alignment score of at least `0.72`, including visual-density match of at least `0.85`, with no copied reference content; +- aesthetic score of at least `0.80`, visual-information density of at least `0.88`, mechanism visualization of at least `0.80`, and whitespace score of at least `0.84`; +- active-region blueprint enrichment of at least `0.10` and detailed-tile ratio of at least `0.80`; - valid image resolution and blueprint aspect ratio. Scientific score must be at least `0.95`; OCR must be exact. If three candidates fail, the best candidate receives one localized Image2 edit repair. If the repair still fails, the command stops without writing `selected_image.png`. diff --git a/docs/rebuild-editable.md b/docs/rebuild-editable.md index aac01ac..7fc1d00 100644 --- a/docs/rebuild-editable.md +++ b/docs/rebuild-editable.md @@ -31,13 +31,13 @@ rfs rebuild-editable --reference input.png --out output\demo --asset-mode api -- Professional scripted rebuild: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --repair-rounds 2 --export-preview +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --asset-policy smart-api --repair-rounds 2 --export-preview ``` Compare against a known high-quality specialized rebuild: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode crop --benchmark-out output\autofigure_architecture_ai_rebuild +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode crop --asset-policy smart-api --benchmark-out output\autofigure_architecture_ai_rebuild ``` Offline professional smoke test: @@ -52,6 +52,7 @@ rfs rebuild-editable-pro --reference input.png --out output\demo_pro_placeholder --reference Input reference image. --out Output directory. --asset-mode api | crop | placeholder. Default: api. +--asset-policy legacy | smart-api. Default: smart-api. Use legacy only for old crop-fallback behavior. --asset-workers Parallel asset workers. Default: 4. --asset-retries Retries used by strict regeneration. Default: 1. --economy-mode Reuse accepted/passing assets. Enabled by default. @@ -59,7 +60,7 @@ rfs rebuild-editable-pro --reference input.png --out output\demo_pro_placeholder --regenerate-slots Comma-separated slot ids to rerun. --text-mode ocr | manual | off. Default: ocr. --layout-mode heuristic | vlm | hybrid. Default: hybrid. ---ocr-engine paddle | easyocr | off. Default: paddle. +--ocr-engine paddle | easyocr | off. Default: easyocr. Paddle remains available and falls back to EasyOCR if the local runtime fails. --ocr-lang en | ch | en_ch. Default: en_ch. --control-mode heuristic | vlm | hybrid | manual. Default: hybrid. --skip-analysis Reuse existing JSON contracts in --out. @@ -77,6 +78,8 @@ rebuild-editable-pro Use the controlled professional Figure DSL workflow. --compile-only Recompile from professional_rebuild_script.dsl.json without VLM planning or asset API calls. ``` +`smart-api` disables final reference-crop assets. Local crops are still saved and passed as API reference context, but final PPT assets are API-generated, reused from another generated slot, or skipped when the region is primarily editable text. If `--asset-mode crop --asset-policy smart-api` is used for a low-cost dry run, the final visual asset is a placeholder and the report records `crop_disabled_by_smart_api_policy_placeholder`. + ## API Environment `--asset-mode api` uses the same image-generation route as the specialized rebuild scripts: @@ -92,6 +95,7 @@ Hybrid VLM layout/control/semantic planning uses an OpenAI-compatible chat compl API_BASE API_KEY or GEMINI_API_KEY MODEL_VLM +RFS_REBUILD_DESIGN_MODEL optional, global design and panel skeleton RFS_REBUILD_LAYOUT_MODEL optional RFS_REBUILD_CONTROL_MODEL optional RFS_REBUILD_SEMANTIC_MODEL optional @@ -104,7 +108,8 @@ PowerShell example: $env:API_BASE='https://your-openai-compatible-provider/v1' $env:API_KEY='' $env:MODEL_VLM='your-vision-language-model' -$env:RFS_REBUILD_LAYOUT_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_DESIGN_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_LAYOUT_MODEL='gemini-3.5-flash' $env:RFS_REBUILD_CONTROL_MODEL=$env:MODEL_VLM $env:RFS_REBUILD_SEMANTIC_MODEL=$env:MODEL_VLM $env:RFS_PROFESSIONAL_REBUILD_MODEL=$env:MODEL_VLM @@ -112,7 +117,7 @@ $env:GEMINI_API_KEY=$env:API_KEY $env:GEMINI_GEN_IMG_URL='https://your-provider/v1beta/models/your-image-model:generateContent' ``` -If the API call fails for a slot, v1 falls back to the reference crop for that slot and records the failure in `asset_generation_report.json`. +With `--asset-policy legacy`, if the API call fails for a slot, v1 falls back to the reference crop for that slot and records the failure in `asset_generation_report.json`. With `--asset-policy smart-api`, crop fallback is disabled; API failure falls back to a placeholder and records `api_failed_placeholder_fallback`. ## Output Files @@ -122,6 +127,8 @@ The workflow writes these files: input_manifest.json reference_geometry.json reference_geometry_overlay.png +reference_panel_layout_plan.json +reference_panel_layout_overlay.png reference_text_geometry.json reference_controls.json reference_controls_overlay.png @@ -131,6 +138,9 @@ asset_generation_specs.json asset_generation_report.json asset_economy_report.json asset_ratio_fit_report.json +asset_decision_report.json +text_asset_filter_report.json +api_asset_plan.json figure_program.json composition_quality_report.json rebuild_vlm_validation_report.json diff --git a/docs/rebuild-process-judge.md b/docs/rebuild-process-judge.md new file mode 100644 index 0000000..463a450 --- /dev/null +++ b/docs/rebuild-process-judge.md @@ -0,0 +1,68 @@ +# Rebuild 过程监督 Judge + +这是「分块进化 / 过程监督」的第一版：给 `rebuild-editable` 管线加一层 append-only +运行日志，并配一个能「看过程、按阶段归因」的 Judge。它对应 `docs/paperimage.docx` +里的痛点二（「缺少从论文理解…到视觉渲染的过程证据，难以确定缺陷产生的具体环节」）。 + +当前它是 **prompt-as-judge**：不依赖训练好的模型，而是调通用 VLM 扮演评委。它产出的 +`stage_feedback` 本身就是将来微调「过程监督 Judge 模型」的标注数据。 + +## 产物（都在 rebuild 输出目录里） + +| 文件 | 给谁看 | 内容 | +|---|---|---| +| `run_log.jsonl` | 机器 | append-only 事件流（每个阶段 started/completed/failed） | +| `run_report.md` | 人 | 阶段→状态→耗时→产物的 Markdown 表 | +| `rebuild_process_judgement.json` | 机器 | Judge 的阶段归因：`stage_feedback` | +| `rebuild_repair_plan.json` | 人/机器 | v1 粗修复计划（fault 阶段 → 动作） | + +事件字段约定见 [`docs/run-log-events.md`](run-log-events.md)。 + +## 用法 + +```powershell +# 1. 跑一次重建（会自动写 run_log.jsonl + run_report.md） +rfs rebuild-editable --reference "C:\path\ref.png" --out "output\demo" ` + --asset-mode api --asset-policy smart-api --layout-mode hybrid ` + --control-mode hybrid --text-mode ocr --export-preview + +# 2. 让过程 Judge 看过程、按阶段归因（需要 API_BASE / API_KEY） +rfs rebuild-judge --out "output\demo" --repair-plan --json +``` + +`rebuild-judge` 的 `--reference` / `--preview` 可覆盖，默认从 `rebuild_result.json` 读取； +`--model` / `--timeout` / `--retries` 可覆盖 VLM 配置；`--repair-plan` 才额外生成修复计划。 + +## 设计要点 + +- **事实与视角分离**：`run_log.jsonl` 只存事实；`rfs/run_projection.py` 把同一条日志 + 投影成「给人看的报告」和「给 Judge 的轨迹」两个视图，不是两套日志。 +- **日志非权威**：快照 JSON（`figure_program.json` 等）仍是可信状态来源；删掉日志重跑 + 不应改变任何快照。 +- **阶段归因**：Judge 输出 `stage_feedback`（`{stage, phase, verdict, problem, suggested_fix}`）， + 把缺陷定位到具体环节，而不是只说「图难看」。 +- **续跑升级**：`--skip-analysis` 的续跑判断从「文件存在」升级为「日志里该阶段上次 + `completed` 且 `ok`」，旧目录无日志仍回退文件存在判断。 + +## v1 / v2 边界 + +**v1（已实现）**：fault 阶段映射到三个现有粗开关—— + +| Judge 判错在 | 动作 | +|---|---| +| 分析段（design/layout/control/text/semantic） | `reanalyze`（整段重分析） | +| `asset_generation` | `regen_assets` | +| `compile` | `recompile` | + +**v2（未实现，后续单独做）**： + +- 细粒度「只重跑某一个阶段」的开关（现在 pipeline 没有按阶段重跑能力）。 +- 自动修复闭环 `--loop N`（重跑 → 再判 → 按 fault 只修 → 再判，直到 ok 或到轮数）。 + +## 关联模块 + +- `rfs/run_log.py` —— `EventLog`（append-only 日志 + fold/last_event）。 +- `rfs/run_projection.py` —— 投影（`render_run_report` / `project_judge_trajectory`）。 +- `rfs/rebuild_process_judge.py` —— `judge_rebuild_run`（过程监督 Judge）。 +- `rfs/rebuild_judge.py` —— `run_rebuild_judge`（编排 + v1 修复计划）。 +- `rfs/coevolution/judges.py` —— 已有的「整图 Judge」（全局进化），与此处「过程 Judge」互补。 diff --git a/docs/run-log-events.md b/docs/run-log-events.md new file mode 100644 index 0000000..2370a7b --- /dev/null +++ b/docs/run-log-events.md @@ -0,0 +1,67 @@ +# run_log.jsonl 事件 Schema + +`rebuild-editable`（后续扩展到 `paper-to-editable` 全链路）在运行目录下追加写一个 +`run_log.jsonl`，作为**运行事实的 append-only 日志**。它本身不是权威数据——各阶段的 +快照 JSON（`figure_program.json`、`reference_geometry.json` 等）仍然是可信状态来源， +日志只是「索引 + 历史」。删除日志重跑，不应改变任何快照产物。 + +设计原则（本项目从 Harness 范式学习的部分）： + +1. **事实与视角分离**：日志只存事实，每个消费者写自己的投影（`fold`）来得到视图。 + 给人看的是 `run_report.md`，给 Judge 做过程监督的是压缩后的轨迹摘要——它们是同一条 + 日志上的两个投影，不是两套日志。 +2. **whole-value 规则**：携带状态的事件（`stage/completed`）必须携带该阶段产出的 + **最小完整摘要**（`summary`），不能只写「done」；全量数据在快照文件里，靠 + `artifacts` 指针关联。 +3. **可归因**：每个阶段事件带 `stage` 和 `phase`，让 Judge 能把缺陷定位到具体环节 + （对应 `docs/paperimage.docx` 痛点二的过程监督要求）。 + +## 通用字段 + +每条事件都是单行 JSON，UTF-8、`ensure_ascii=False`，追加顺序写入： + +| 字段 | 类型 | 含义 | +|---|---|---| +| `seq` | int | 本次运行内单调递增的事件序号 | +| `ts` | float | Unix 时间戳（墙钟，便于人看） | +| `type` | string | 事件类型（见下表） | + +## 事件类型 + +| type | 必需字段 | 可选字段 | 含义 | +|---|---|---|---| +| `run/started` | `workflow`, `reference`, `out_dir` | `params{}` | 一次运行开始 | +| `stage/started` | `stage` | `phase`, `mode`, 其他 meta | 某阶段开始 | +| `stage/completed` | `stage`, `ok=true`, `duration_ms` | `phase`, `artifacts[]`, `summary{}` | 某阶段成功结束 | +| `stage/failed` | `stage`, `ok=false`, `duration_ms`, `error`, `error_type` | `phase` | 某阶段失败（异常不吞，仍上抛） | +| `run/completed` | `ok=true` | `pptx`, `preview` | 运行成功结束 | +| `run/failed` | `ok=false`, `error` | — | 运行失败 | + +`summary{}` 是 whole-value 的最小完整摘要，示例： + +```json +{"seq": 12, "ts": 1719000031.1, "type": "stage/completed", "stage": "build_program", + "phase": "layout", "ok": true, "duration_ms": 18200, + "artifacts": ["reference_geometry.json", "slot_inventory.json"], + "summary": {"slots": 12, "panels": 3, "arrows": 5}} +``` + +## phase 映射（docx 五个概念环节 → rebuild 具体阶段） + +`docs/paperimage.docx` 把过程监督的环节定为： +`论文理解 → 内容抽取 → 拓扑构建 → 布局规划 → 视觉渲染`。rebuild 段的映射如下 +（后续扩展到 paper-to-image 段时补 `paper_understanding` 与完整的 `content_extraction`）： + +| phase | 环节 | rebuild 阶段 | +|---|---|---| +| `topology` | 拓扑构建 | `design_plan` | +| `layout` | 布局规划 | `build_program`, `localize_controls` | +| `content_extraction` | 内容抽取 | `slot_semantics` | +| `rendering` | 视觉渲染 | `text_layer`, `compile` | + +## 消费者（投影） + +- 人看：`run_projection.render_run_report()` → `run_report.md`。 +- Judge 过程监督：`run_projection.project_judge_trajectory()` → 压缩轨迹文本注入 Judge prompt。 +- 续跑水位线：`EventLog.last_event(path, "stage/completed", stage=...)` 或 + `run_projection.last_completed_stage()`。 diff --git a/docs/target_evolution_pipeline_plan.md b/docs/target_evolution_pipeline_plan.md new file mode 100644 index 0000000..dc67694 --- /dev/null +++ b/docs/target_evolution_pipeline_plan.md @@ -0,0 +1,429 @@ +# Paperimage 目标链路构建计划 + +> 状态：目标架构与分期实施计划。 +> 范围：论文 → 整体框架图 → 可编辑 PPTX 的科学性与美观性协同进化。 +> 本计划以 `docs/paperimage.docx` 的方法主张为准，并与当前 Paperimage 的工作流和合同文件对齐。 + +## 1. 目标与边界 + +Paperimage 的目标不是生成一张看起来合理的图片，而是稳定地产出： + +1. 科学上忠实表达论文方法的整体框架图； +2. 视觉上清晰、专业、有完整叙事的整体图； +3. 保留文本、箭头、连接器、面板、公式、分组等关键对象可编辑性的 PPTX； +4. 可追溯、可归因、可回退、可跨论文验证的进化过程。 + +系统的优化优先级固定如下： + +```text +科学事实与关键关系正确 + > 方法覆盖和叙事清晰 + > 可编辑性与结构保留 + > 全局美观性 + > 局部视觉细节 +``` + +科学硬错误不能被审美高分抵消。核心模块缺失、虚构科学模块、关键关系反向、训练/推理混淆、核心术语错误均直接触发 `gate_fail`。 + +## 2. 总体双阶段链路 + +```text +阶段 A：论文 → 整体框架图（全局进化） + +论文 PDF + → 论文证据与科学合同 + → 全图结构/叙事/风格候选 + → 整体框架图候选 + → 科学 Grounding Gate + 全图 Judge + → 选优、修复、跨论文验证 + → approved_image.png + +阶段 B：整体框架图 → 可编辑 PPTX（全局—分块协同进化） + +approved_image.png + 论文科学合同 + → 全局图理解、布局与分块候选 + → 选择分块/接口/对象职责方案 + → 分块资产重建 + 原生 PPT 对象生成 + → 全局 PPT 装配 + → 科学保留 Gate + 全局/分块/接口/PPT Judge + → 局部修复或全局重规划 + → editable_composition.pptx +``` + +阶段 A 的生成单位始终是**整图**，不要求提前生成局部块。阶段 B 才将已通过科学门禁的整图拆为可编辑重建所需的局部资产和 PPT 原生对象。 + +阶段 B 不是“先全局、后分块”的一次性流水线，而是下列交替循环： + +```text +全局规划 → 分块重建 → 全局装配 → 联合评判 + ↓ + 局部修复 ← 归因 → 全局重规划 + ↓ + 再装配、再评判 +``` + +只有上游整图存在科学错误，或经验证无法以合理对象边界拆解时，阶段 B 才向阶段 A 发出“建议重生成”的反馈；正常的 PPT 重建问题不应触发整图重画。 + +## 3. 阶段 A：论文到整体框架图的全局进化 + +### 3.1 输入与科学合同 + +首先建立可审计的论文合同，而不是直接生成图。建议固定以下产物： + +```text +paper_review.json +figure_specification.json +scientific_ground_truth.json +``` + +合同至少定义：研究问题、核心贡献、必须出现的模块、输入输出、训练/推理阶段、实体、关系、术语、允许简化项、禁止虚构项和对应论文证据。 + +每个科学节点和关系必须有稳定 ID 与证据链接： + +```json +{ + "id": "node_encoder", + "label": "Feature Encoder", + "role": "core_method_module", + "must_show": true, + "evidence": [{"page": 3, "quote": "..."}] +} +``` + +### 3.2 全图候选与全局进化变量 + +每轮输出多张整体候选图。候选之间可改变： + +- 模块层级、分组与阅读方向； +- 主流程、支路、反馈环的表达； +- 内容密度、留白、视觉隐喻和配色； +- 全局风格与视觉节奏； +- 模块边界清晰度。 + +整图生成阶段不切局部资产，但应考虑下游重建，额外优化 `decomposability_score`（可分解性）： + +- 模块区块边界是否清楚； +- 文本、箭头、面板和局部视觉是否可区分； +- 关键关系是否依赖难以分离的复杂纹理； +- 跨区连接是否可识别； +- 是否有足够留白让 PPT 中的可编辑连接器路由。 + +可分解性是图质量的一项正向标准，不是要求牺牲图的视觉表现来迎合下游。 + +### 3.3 阶段 A 评判与选择 + +图像 Judge 输入为：论文证据摘要、科学合同、候选整图及审美 Profile。输出必须分项、可引用、可执行： + +```json +{ + "coverage_score": 0.0, + "relation_correctness": 0.0, + "terminology_accuracy": 0.0, + "narrative_score": 0.0, + "aesthetic_score": 0.0, + "decomposability_score": 0.0, + "blocking_issues": [], + "evidence": [], + "preserve": [], + "repair": [] +} +``` + +独立 Grounding Gate 根据 `must_show`、`must_not_invent`、实体、关系及术语进行确定性/半确定性核验。其失败优先级高于所有软分数。 + +Online Judge 用于提出修复与选择候选；Frozen Judge 用于决定是否接受新最佳候选。两者不得在同一进化周期同步升级。 + +### 3.4 阶段 A 的停止与经验沉淀 + +整图通过条件：无科学 blocker，达到科学与审美的最低阈值，且可分解性不低于产品阈值。每条修复经验必须记录为“条件 → 行动 → 结果”，并先在本案例、再在留出论文上验证；只在跨论文不退化时升级为通用策略。 + +## 4. 阶段 B：整体图到可编辑 PPTX 的全局—分块协同进化 + +### 4.1 全局规划的职责 + +阶段 B 的全局进化不重新创作图片，而是为 PPT 重建选择全局可编辑结构： + +- 阅读顺序、面板层级与页面区域； +- 模块与视觉区域如何映射； +- 分块方案及每块边界； +- 关键文本、公式、箭头、面板、控制符应由何种 PPT 原生对象承载； +- raster 局部资产的职责边界； +- slot 的输入/输出端口； +- 连接器的路由、避让、虚线反馈环； +- 装配后的视觉还原约束。 + +全局规划输出： + +```text +reference_logic_plan.json +reference_layer_plan.json +reference_flow_graph.json +segmentation_candidates.json +segmentation_plan.json +``` + +`segmentation_plan.json` 是新增的显式合同。它不只是图像切块坐标，而是“语义—视觉—可编辑对象”的职责分配： + +```json +{ + "plan_id": "segmentation_02", + "slots": [ + { + "slot_id": "encoder_visual", + "semantic_node_ids": ["node_encoder"], + "bbox_percent": {"x": 0.1, "y": 0.3, "w": 0.2, "h": 0.3}, + "visual_asset_scope": ["encoder_illustration"], + "editable_object_ids": ["label_encoder", "edge_input_encoder", "edge_encoder_decoder"], + "input_ports": ["left_center"], + "output_ports": ["right_center"] + } + ], + "rationale": [] +} +``` + +### 4.2 分块策略是进化变量 + +同一整图可以存在多个分块候选。例如按科学模块、按视觉卡片、按面板层级、或将装饰性视觉与科学模块分离。分块策略的优化目标为： + +```text +分块质量 = +语义模块一致性 ++ 视觉区域完整性 ++ 原生对象可编辑性 ++ 连接可恢复性 ++ 局部可修复性 ++ 装配后的整体还原度 +- 过度切分 +- 不当合并 +- 文本/箭头/公式被烘焙进 asset +- 跨块科学关系丢失 +``` + +以下情况应重新分块，而非重画上游整图： + +- 一个 slot 合并了本应独立编辑的两个科学模块； +- 一个模块被拆得过碎，导致接口和风格失控； +- 标签、箭头或公式残留在 raster asset 中； +- slot 边界切断视觉主体或破坏端口； +- 分块后无法保持关键关系的可编辑连接。 + +### 4.3 分块与全局装配的协同循环 + +每次选择一个全局候选后，执行： + +```text +1. 生成/选择 segmentation plan； +2. 对每个 slot 生成或提取局部视觉资产； +3. 用 PPT 原生对象创建文字、箭头、面板、公式与分组； +4. 根据 ports 和 flow graph 路由连接器； +5. 编译 figure_program.json 与 editable_composition.pptx； +6. 渲染 preview； +7. 运行科学保留 Gate、全局 Judge、分块 Judge、接口 Judge、PPT 编辑性检查； +8. 按归因执行局部修复或全局重规划； +9. 重新装配并比较修复前后结果。 +``` + +局部最优不能替代全局回评：某 slot 单独合格，也可能在装配后造成拥挤、风格断裂、箭头交叉或主流程弱化。 + +### 4.4 阶段 B 的问题归因与修复范围 + +| 问题 | 默认修复动作 | +|---|---| +| 论文核心模块或关系在 PPT 中丢失/改变 | `reanalyze_semantics`，必要时 `resegment` | +| 分块边界不合理、过拆或合并 | `resegment` | +| 整体布局拥挤、主流程弱、面板层级混乱 | `relayout` | +| 局部图像资产错误、模糊或风格不一致 | `regen_assets(slot_ids)` | +| 标签、公式、端口或箭头错位 | `repair_interfaces(object_ids, edge_ids)` | +| 连接器路由不正确 | `reroute(edge_ids)` | +| PPT 对象缺失、顺序或导出异常 | `recompile` | +| 上游整图科学错误或不可合理分解 | `request_regenerate_image` | + +正常情况下，阶段 B 只修复当前对象或重新规划 PPT 结构；`request_regenerate_image` 是向阶段 A 的升级反馈，不是默认路径。 + +## 5. Judge 系统与训练路线 + +### 5.1 Judge 的职责分层 + +```text +训练 VLM Judge：分项评分、证据引用、错误定位、修复建议 + ↓ +Grounding Gate：实体、关系、术语、must_show、must_not_invent 的硬门禁 + ↓ +代码聚合：按固定权重得分、阈值和接受/回退决策 +``` + +Judge 不自行决定最终总分，也不能让审美分抵消科学 blocker。 + +### 5.2 两个 Judge 任务阶段 + +**图像 Judge（优先建设）** + +输入：论文证据、科学 Ground Truth、候选整图和审美 Profile。 +任务：论文方法覆盖、实体关系正确、术语、整体叙事、美观性、可分解性、A/B 偏好、全图修复建议。 + +**PPT Judge（图像 Judge 通过后扩展）** + +输入：论文科学合同、`approved_image.png`、PPT 渲染预览、`figure_program.json`、对象树/统计。 +任务：科学保留、视觉还原、文本/箭头/公式/分组可编辑率、分块合理性、接口与对象级修复建议。 + +### 5.3 训练数据和人工反馈 + +训练样本必须让标注者同时看到论文证据摘要和候选图，不能只按图片主观打分。人工标签至少拆为： + +- 科学状态：`correct / incomplete / incorrect / unverifiable`； +- blocker：缺核心模块、虚构模块、关系反向、训练推理混淆、术语错误； +- 审美：布局、可读性、信息密度、配色、风格偏好； +- 修复：保留什么、修改什么、修改后是否真正提高。 + +重点数据集包括：真实生成轨迹、人工认可图、失败候选、成对偏好、修复前后轨迹，以及可控合成错误。必须覆盖 `beautiful_but_wrong`、`correct_but_ugly`、`relation_reversal`、`missing_core_stage`、`invented_module`、`terminology_corruption` 等对抗集。 + +训练顺序保持：结构化 SFT → Pairwise 排序 → 科学硬错误加权 → 建议有效性 → 分数校准。按论文/GT ID 切分训练、验证、隐藏测试，禁止同一论文跨集合泄漏。 + +## 6. 与现有代码的映射和改造计划 + +| 目标能力 | 当前基础 | 下一步改造 | +|---|---|---| +| 论文到整图全局进化 | `rfs/coevolution/`、`rfs/coevolution/groundtruth.py` | 把 `figure_specification`/论文证据稳定接入 Ground Truth；增加可分解性评分 | +| 图像 Judge 训练 | `rfs/judge_training/`、训练计划文档 | 落地 `JudgeExample`、数据 builder、corruption generator、grounding gate、split/evaluator/registry | +| 参考图到 PPT 规划 | `rfs/planning/rebuild/design.py` | 输出多套 `segmentation_candidates` 和选中的 `segmentation_plan`；绑定 semantic nodes、ports、editable scopes | +| 分块资产重建 | `rfs/workflows/rebuild_editable.py`、`rfs/generation/rebuild_assets.py` | 从 `segmentation_plan` 派生 asset specs；支持按 slot 重生与资产验收 | +| PPT 原生编译 | `rfs/composition/rebuild/`、`figure_program.json` | 保持本地 deterministic compiler 权威；增强对象 ID 与科学节点/关系绑定 | +| 过程监督 | `rfs/run_log.py`、`rfs/run_projection.py`、`rfs/rebuild_process_judge.py` | 记录 scientific、segmentation、interface 事件；Judge 输出 node/slot/edge/object 级归因 | +| 自动修复 | `rfs/rebuild_judge.py`、`rfs/rebuild_evolution.py` | 从 coarse `reanalyze/regen_assets/recompile` 升级为 `resegment/relayout/reroute/repair_interfaces`，并保留 coarse fallback | + +现有稳定合同和编译边界不应被打破：VLM 只生成 JSON 规划、评价和修复建议；本地 Python 继续编译 PPTX；文字、箭头、连接线、面板和公式保持 PPT 可编辑对象；局部图像使用 contain/no-crop。 + +## 7. 新增合同、运行事件和核心输出 + +建议新增或标准化下列产物： + +```text +scientific_ground_truth.json +segmentation_candidates.json +segmentation_plan.json +scientific_grounding_report.json +joint_judgement.json +repair_decision.json +experience_records.jsonl +``` + +每个运行日志应能记录： + +```text +paper_contract/created +image_candidate/generated +image_candidate/judged +image_candidate/approved +segmentation/candidate_created +segmentation/selected +slot/generation_started +slot/judged +interface/routed +ppt/compiled +scientific_gate/passed | scientific_gate/failed +repair/accepted | repair/rolled_back +experience/promoted | experience/rejected +``` + +最终 `paper_to_editable_result.json` 应至少汇总科学覆盖、关系正确性、全局叙事、可分解性、分块选择、slot 修复、PPT 可编辑率、修复轮次和最终 gate 状态。 + +## 8. 分期实施与门禁 + +### P0：合同与观测性 + +- 冻结论文科学合同和统一 JudgeExample 结构； +- 在现有 run log 中记录论文合同、整图候选和阶段 B 的对象级产物； +- 为现有轨迹建立可追溯 manifest。 + +门禁：所有结论都能追溯到论文证据、候选、Judge 输出和编译产物。 + +### P1：图像 Judge 的科学 Gate + +- 实现 Grounding Gate v0：OCR 术语、必须实体、虚构实体、关键关系； +- 建设 D0 数据和人工复核的科学负样本； +- 将 Gate 接到 `coevolve-image` 的 Frozen 接受条件。 + +门禁：已知注入科学错误有高召回，且美观但错误的对抗图不能通过。 + +### P2：整图全局进化 + +- 将 `paper-to-image` / `coevolve-image` 明确为整图候选循环； +- 增加论文覆盖、叙事、可分解性评分； +- 记录全图修复经验和跨论文验证结果。 + +门禁：留出论文上的科学质量不退化，且没有明显审美模板坍缩。 + +### P3：PPT 分块合同与全局—分块协同 + +- 在 `planning/rebuild/design.py` 增加 segmentation candidate/plan； +- 在 rebuild workflow 消费该合同； +- 对每个 slot 建立语义节点、端口和可编辑职责绑定； +- 增加装配后全局回评。 + +门禁：分块不丢失科学关系，关键对象可编辑，PPT 预览还原达到阈值。 + +### P4：对象级修复进化 + +- 扩展过程 Judge 输出 `semantic_node_ids`、`slot_ids`、`edge_ids`、`object_ids`； +- 扩展 evolution action：`resegment`、`relayout`、`reroute`、`repair_interfaces`； +- 记录前后分数、blocker 变化和副作用，拒绝无效修复。 + +门禁：局部修复不引入新科学错误；全局问题不会被局部修饰掩盖。 + +### P5：训练 Judge 与 Shadow 接入 + +- 用真实进化轨迹、合成错误、人工筛选与修复结果训练 Judge； +- 先以 Shadow 模式和当前 Bootstrap Judge 并行； +- 达到隐藏集和对抗集门禁后才作为 Online Judge；旧稳定版本继续担任 Frozen Judge。 + +门禁：科学硬错误漏检受控、输出 JSON 稳定、人工审计无系统性偏差；新模型随时可回滚。 + +### P6：PPT Judge 与用户编辑反馈 + +- 在图像 Judge 稳定后，增加 PPT 对象树和渲染预览样本； +- 采集用户在 PPT 中的移动、改字、删箭头、替换资产等编辑事件； +- 将编辑前后状态及 Judge 分数变化转为可学习的修复轨迹。 + +门禁：用户偏好只能影响审美与布局策略，不得降低科学硬门禁。 + +## 9. 接受、回退与跨论文泛化规则 + +一次修复被接受，必须同时满足： + +1. 不产生新的科学 blocker； +2. 修复目标的分项指标提升； +3. 已正确的内容被 preserve； +4. 整体或局部改动的副作用在阈值内； +5. Frozen Judge 和 Grounding Gate 都同意； +6. 对候选通用经验，在留出论文验证后不退化。 + +经验状态： + +```text +candidate +→ current_case_validated +→ cross_paper_validated +→ promoted + +candidate/current_case_validated +→ overfit_or_failed +→ rollback +``` + +经验应保存为带适用条件的规则，而不是某张图的坐标或单次 prompt。例如：“在横向三阶段主流程中，反馈边优先放在主链下方并使用虚线；仅适用于存在明确反馈语义且下方有足够留白的布局”。 + +## 10. 最终完成定义 + +目标链路完成并不等于能跑通一次 demo，而是同时满足： + +- 论文事实、模块和关键关系可追溯并经科学 Gate 守护； +- 整图全局进化能够产出科学正确、清晰、美观且可分解的框架图； +- PPT 阶段能够通过全局—分块协同进化完成可编辑重建； +- 关键文本、箭头、关系、公式和面板保持为 PPT 原生可编辑对象； +- Judge 能定位全局、分块、接口、语义与编译问题并给出可执行修复范围； +- 修复操作能够被接受、回退和跨论文验证； +- 训练出的 Judge 与数据、配置、评测、模型版本及 Git commit 绑定； +- 用户编辑反馈可成为后续进化数据，但不突破科学性约束。 + +最终系统应能证明的不是“它生成了一张漂亮图”，而是：这张图忠实表达了论文，整体叙事清楚，能被可靠重建为可编辑 PPTX，且每次优化都有可追溯证据并能在新论文上验证其泛化性。 diff --git a/docs/workflow.md b/docs/workflow.md index 929790f..dddc136 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -1,48 +1,69 @@ # Summary -The workflow is designed to keep scientific content, layout, image generation, and PPT editability separated. This prevents the project from becoming a fragile script pile where coordinates, prompts, and rendered assets are mixed together. +The default workflow asks Image2 to generate one complete paper figure, including text, arrows, modules, and visual explanations, then reconstructs the approved image into editable PowerPoint layers. ## Full Workflow -1. Archive the paper and reference image into the output directory. -2. Extract the paper brief: title, method story, named modules, variables, and figure goal. -3. Analyze the reference image and create 25-50 non-arrow image slot targets. -4. Detect arrow/control candidates, write `reference_control_candidates.json`, and render `slot_overlay.png` plus `reference_control_overlay.png` for review or VLM binding. -5. Bind controls to source/target slots with `--control-localizer-mode hybrid` when API access exists, or deterministic heuristic fallback otherwise. -6. Create `arrow_style_profile.json`, `selected_arrow_routes.json`, and `arrow_quality_report.json`; this stage may soften and classify arrows, and may use orthogonal obstacle-aware fallback routing only for missing or explicitly fallback-allowed paths. In `--arrow-style-mode aesthetic`, it may apply curve connectors and halo underlays by default; bundle lane offsets require explicit route opt-in and must stay inside the recorded reference tunnel. It must preserve reference-image flow logic. -7. Create the style sheet before any image prompt is written. -8. Create `layout_plan.json`; use VLM only for normalized coordinate estimation when enabled. -9. Create `figure_program.json`; this is the only layout source for the PPT compiler. -10. Write `reference_slot_prompt_brief.json`; it records what each local reference slot does and what paper concept it carries. -11. Write `slot_prompt_plan.json`; default `--prompt-plan-mode vlm` calls the model for each slot and produces `image_prompt_core`. Use `--prompt-plan-workers` to run these per-slot VLM calls in parallel. -12. Generate multiple image candidates per slot from the model-planned prompt core. -13. Select assets by fill, margin, ratio, and cutoff metrics; never use semantic cropping. -14. Review selected assets with heuristic or VLM review. -15. Compile `editable_composition.pptx` deterministically from the figure program. -16. Export PDF/PNG for review where possible. -17. Run visual critic against the reference image and rendered output. -18. Write alignment review and critic report. -19. Run `rfs validate` before delivery. - -## Recommended Development Runs +1. Archive the paper and optional positive/negative visual references. +2. Extract and validate the evidence-grounded semantic contract. +3. Compile a full layout blueprint and Prompt whose whitelist contains every visible label and whose connector checklist contains every required relation. +4. Ask Image2 to generate the complete raster figure, including visible text and arrows. +5. Review candidates for paper faithfulness, terminology, topology, information density, and visual quality; repair failed regions with Image2. +6. Promote only a passing candidate to `selected_image.png`. +7. Reconstruct the approved complete image with layout analysis, OCR, connector localization, asset extraction, and paper-semantic binding. +8. Compile `editable_composition.pptx`, export review files with a hard timeout, and validate the result. + +## Recommended Product Runs + +Complete default product run: + +```powershell +rfs paper-to-editable --paper "C:\path\paper.pdf" --out .\output\paper_to_editable --editable-route rebuild --image-asset-mode image2 --review-mode vlm --json +``` + +The explicit `--editable-route rebuild` is optional because it is the default. + +Generate and review the complete Image2 figure without rebuilding PPT: + +```powershell +rfs paper-to-image --paper "C:\path\paper.pdf" --out .\output\paper_to_image --asset-mode image2 --candidates 3 --review-mode vlm --json +``` + +## Optional Multi-Asset Experiment + +The commands and failure policy below apply only to `make-framework`. They are +not the primary complete-image product route. + +Offline engineering check: Offline engineering check: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\offline_check --asset-mode placeholder --locator-mode heuristic --control-localizer-mode heuristic --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json -rfs validate --out D:\ResearchFigureStudio\output\offline_check --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\offline_check --asset-mode placeholder --locator-mode heuristic --control-localizer-mode heuristic --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json +rfs validate --out .\output\offline_check --json ``` +Paper-native contract-driven engineering check (recommended after a fast/paper-to-image run): + +```powershell +rfs make-framework --paper "C:\path\paper.pdf" --paper-contract-dir ".\output\paper_to_image\run" --out .\output\contract_check --slot-source paper-native --complexity-profile paper-native-dense --asset-mode placeholder --asset-workers 3 --locator-mode heuristic --control-localizer-mode off --prompt-plan-mode heuristic --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --text-extractor-mode heuristic --ocr-engine off --no-export --json +``` + +This route does not call the legacy paper planner or generate a whole reference +figure. `panel_graphs` instance IDs remain distinct, relations remain +evidence-backed, exact labels remain editable PPT text, and extra slots are +grounded visual facets of real entities rather than invented modules. + Small real image check: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\real_small --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\real_small --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json ``` Full quality run: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\full_quality --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\full_quality --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json ``` ## Parallel Asset Generation @@ -52,6 +73,7 @@ rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.pn - `--asset-retries` controls retries per candidate. Retry handles transient 429, 5xx, and timeout failures. - Candidate failures are recorded as rejected candidates. A slot only blocks the run when all of its candidates fail. - The pipeline does not silently switch to placeholder mode when Gemini fails. +- An uncached 25-slot Image2 run is expected to take minutes. A warm run can reuse the prompt plan and validated assets and should normally return in tens of seconds. ## Failure Policy diff --git a/examples/banana_game.md b/examples/banana_game.md index 3ad9844..6006013 100644 --- a/examples/banana_game.md +++ b/examples/banana_game.md @@ -4,32 +4,38 @@ This example runs the BananaGame paper through ResearchFigureStudio using the cu ## Inputs -- Paper: `C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf` -- Reference image: `D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png` +- Paper: `path/to/BananaGame.pdf` +- Reference image: `path/to/reference_architecture.png` If the reference path is unavailable, provide a new user reference image. The workflow should not generate its own full reference image unless explicitly requested. ## Offline Validation Run ```powershell -$out='D:\ResearchFigureStudio\output\banana_full_pipeline_placeholder_36' +$paper='path/to/BananaGame.pdf' +$reference='path/to/reference_architecture.png' +$out='output/banana_full_pipeline_placeholder_36' if (Test-Path $out) { Remove-Item -LiteralPath $out -Recurse -Force } -rfs make-framework --paper 'C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf' --reference 'D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png' --out $out --asset-mode placeholder --locator-mode heuristic --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json +rfs make-framework --paper $paper --reference $reference --out $out --asset-mode placeholder --locator-mode heuristic --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json rfs validate --out $out --json ``` ## Small Real Image Run ```powershell -$out='D:\ResearchFigureStudio\output\banana_real_small_25x1' -rfs make-framework --paper 'C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf' --reference 'D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png' --out $out --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json +$paper='path/to/BananaGame.pdf' +$reference='path/to/reference_architecture.png' +$out='output/banana_real_small_25x1' +rfs make-framework --paper $paper --reference $reference --out $out --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json ``` ## Full Quality Run ```powershell -$out='D:\ResearchFigureStudio\output\banana_real_36x3_vlm' -rfs make-framework --paper 'C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf' --reference 'D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png' --out $out --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json +$paper='path/to/BananaGame.pdf' +$reference='path/to/reference_architecture.png' +$out='output/banana_real_36x3_vlm' +rfs make-framework --paper $paper --reference $reference --out $out --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json ``` ## Expected Outputs diff --git a/experiments/legacy_reference_rebuilds/README.md b/experiments/legacy_reference_rebuilds/README.md new file mode 100644 index 0000000..7c63f2f --- /dev/null +++ b/experiments/legacy_reference_rebuilds/README.md @@ -0,0 +1,5 @@ +# Legacy reference-specific rebuilds + +This directory documents the repository's early image-to-PPT experiments. The old sample-specific rebuild scripts were removed from the public repository because they depended on local reference images and hard-coded machine paths. + +Use this directory only for historical context. Do not import anything from here into production workflows, plugin skills, tests, or packaging code. New reusable behavior belongs in `rfs/` with tests and path-independent inputs. diff --git a/experiments/literature_review/README.md b/experiments/literature_review/README.md new file mode 100644 index 0000000..c48bc32 --- /dev/null +++ b/experiments/literature_review/README.md @@ -0,0 +1,8 @@ +# Literature review archive + +This directory contains early research notes used to understand related diagram-generation systems. + +- Raw paper text extracts are intentionally not tracked in Git to avoid shipping large or copyrighted source material. +- Historical extraction and translation helpers were removed from the public repository because they depended on machine-specific local paths and were not production utilities. + +Nothing under this directory may be imported by `rfs/`, the Codex plugin skill, or automated production workflows. diff --git a/experiments/literature_review/reviewer/README.md b/experiments/literature_review/reviewer/README.md new file mode 100644 index 0000000..e04977b --- /dev/null +++ b/experiments/literature_review/reviewer/README.md @@ -0,0 +1,109 @@ +# 论文筛选工具（2505+） + +本地浏览 + 分类 + 备注的网页工具，用于快速筛查论文。仅用 Python 标准库，无需安装额外依赖。 + +## 启动 + +```powershell +cd experiments\literature_review\reviewer +python server.py +``` + +默认在 `http://127.0.0.1:8765/` 启动并自动打开浏览器。 + +常用参数： + +```powershell +python server.py --port 9000 # 换端口 +python server.py --no-browser # 不自动打开浏览器 +``` + +## 范围 + +只纳入 `arxiv_id >= 2505`（2025 年 5 月及以后）的论文，共 **490 篇**（其中 488 篇有本地 PDF）。 + +## 操作 + +- 左侧列表选择论文，右侧内嵌 PDF 全文 + 标题/摘要/分类信息。 +- 顶部可切换筛选：全部 / 未筛 / 科研风 / 生动活泼 / 坏，支持搜索标题、作者、ID、分类。 +- 分类： + - **好 ▾** → 下拉选择「科研风」或「生动活泼」 + - **坏** + - **清除**（撤销当前分类） +- 每篇论文底部有**标记 / 备注**文本框，自动保存。 +- 键盘快捷键： + - `1` 科研风，`2` 生动活泼，`B` 坏，`U` 清除 + - `←` / `→` 上一篇 / 下一篇 + - 在「未筛」视图里标注后会自动跳到下一篇未筛论文。 + +## 数据保存 + +分类与备注实时写入： + +``` +experiments\literature_review\papers\review_2505.json +``` + +每条格式： + +```json +{ + "arxiv_id": { + "label": "good" | "bad", + "style": "research" | "lively", + "note": "自由文本标记", + "updated_at": "…" + } +} +``` + +刷新页面或重开后进度保留，可断点续标。清空重来：删除该 JSON 文件，或调用 `POST /api/reset`。 + +## 筛选完后：分类摆放 PDF + +```powershell +cd experiments\literature_review\reviewer +python sync_classified.py # 全量重建 + 生成 CSV +python sync_classified.py --dry-run # 先预览统计，不写盘 +``` + +分类来源（合并）： +1. `review_2505.json` —— 网站上筛的 2505+ 论文（好/坏/未筛 + 风格 + 备注/优）； +2. Downloads 两个文件夹 —— 2301~2504 手动筛出的「科研风 / 生动活泼」： + - `C:\Users\Lenovo\Downloads\1.科研风\1.科研风` + - `C:\Users\Lenovo\Downloads\2.生动型\2.生动型` + +把 PDF **复制**（不移动原文）到： + +``` +experiments\literature_review\papers\classified\ + 优\ # 好论文里备注含「优」的（跨风格汇总） + 好\ + 科研风\xxxx.pdf + 生动活泼\xxxx.pdf + 坏\xxxx.pdf + 未筛\xxxx.pdf + 分类清单.csv # UTF-8 BOM，Excel 可直接打开 +``` + +可反复重跑：会自动同步（新增/更新文件、清理已改分类的过期副本）。缺本地 PDF 的论文（如 `2507.23261`、`2510.18515`）只进清单、不复制文件。 + +## 推送到 GitHub + +```powershell +cd experiments\literature_review\reviewer +python push_to_github.py # 把 classified/ 推送到 Sen-illion/papers-dataset +python push_to_github.py --dry-run # 只预览，不推送 +``` + +- 免 token，走本机 SSH 密钥（`~/.ssh/id_ed25519`，需已授权访问 GitHub）。 +- 复用仓库已有的 PDF blob，实际只上传目录结构 + 分类清单 CSV，一个 commit 搞定。 +- 筛完新的、重新 `sync_classified.py` 后，再跑一次即可增量更新。 + +## 接口 + +- `GET /api/papers` — 论文列表（含 label / style / note）与统计 +- `POST /api/classify` — `{"arxiv_id": "...", "label": "good"|"bad"|null, "style": "research"|"lively"}` +- `POST /api/note` — `{"arxiv_id": "...", "note": "..."}` +- `POST /api/reset` — 清空全部分类与备注 +- `GET /pdf/.pdf` — 内嵌 PDF diff --git a/experiments/literature_review/reviewer/organize.py b/experiments/literature_review/reviewer/organize.py new file mode 100644 index 0000000..6662103 --- /dev/null +++ b/experiments/literature_review/reviewer/organize.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""按 review_2505.json 的分类结果，把 PDF 复制到分类文件夹。 + +结构（复制，不移动原文）： + classified/ + 优/ ← 好论文里备注含“优”的（跨风格汇总，另外复制一份） + 好/ + 科研风/xxxx.pdf + 生动活泼/xxxx.pdf + 坏/xxxx.pdf + 未筛/xxxx.pdf + 分类清单.csv + +用法： + python organize.py # 实际复制 + 生成 CSV + python organize.py --dry-run # 只预览统计，不写盘 +""" +from __future__ import annotations + +import argparse +import csv +import json +import os +import shutil + +HERE = os.path.dirname(os.path.abspath(__file__)) +PAPERS_DIR = os.path.normpath(os.path.join(HERE, "..", "papers")) +PAPERS_JSON = os.path.join(PAPERS_DIR, "papers.json") +PDF_DIR = os.path.join(PAPERS_DIR, "pdf") +REVIEW_JSON = os.path.join(PAPERS_DIR, "review_2505.json") +OUT_DIR = os.path.join(PAPERS_DIR, "classified") +CSV_PATH = os.path.join(OUT_DIR, "分类清单.csv") + +ARXIV_FROM = "2505" +YOU_DIR = "优" + + +def category_of(label, style): + """返回 (顶级分类, 子文件夹或 None)。""" + if label == "good": + return ("好", "生动活泼" if style == "lively" else "科研风") + if label == "bad": + return ("坏", None) + return ("未筛", None) + + +def load_review(): + if not os.path.isfile(REVIEW_JSON): + return {} + with open(REVIEW_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + + +def copy_pdf(src, dst): + if not os.path.isfile(dst) or os.path.getsize(dst) != os.path.getsize(src): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + + +def main(): + parser = argparse.ArgumentParser(description="按筛选结果整理 PDF 到分类文件夹") + parser.add_argument("--dry-run", action="store_true", help="只预览，不实际复制/删除/写 CSV") + args = parser.parse_args() + + with open(PAPERS_JSON, "r", encoding="utf-8") as f: + all_papers = json.load(f) + review = load_review() + + papers = [p for p in all_papers if (p.get("arxiv_id") or "") >= ARXIV_FROM] + papers.sort(key=lambda p: p["arxiv_id"]) + + meta = {} # aid -> dict(top, sub, note, you, has_pdf) + rows = [] # CSV 行 + for p in papers: + aid = p["arxiv_id"] + entry = review.get(aid) + entry = entry if isinstance(entry, dict) else {} + label = entry.get("label") + style = entry.get("style") + note = (entry.get("note") or "").strip() + top, sub = category_of(label, style) + you = (label == "good") and ("优" in note) + meta[aid] = {"top": top, "sub": sub, "note": note, "you": you} + rows.append({ + "分类": top, + "风格": sub or "", + "优": "优" if you else "", + "arxiv_id": aid, + "标题": p.get("title", "") or "", + "备注": note, + "被引": p.get("cited_by_count", 0) or 0, + "arXiv": p.get("arxiv_url", "") or "", + }) + + # 统计 + stat = {} + you_count = 0 + for r in rows: + key = r["分类"] if not r["风格"] else f'{r["分类"]}/{r["风格"]}' + stat[key] = stat.get(key, 0) + 1 + if r["优"]: + you_count += 1 + stat[YOU_DIR] = you_count + + copied = 0 + you_copied = 0 + missing = [] + expected = set() # 期望存在的相对 .pdf 路径（用于清理过期文件） + + for p in papers: + aid = p["arxiv_id"] + src = os.path.join(PDF_DIR, aid + ".pdf") + m = meta[aid] + rel = os.path.join(m["top"], m["sub"]) if m["sub"] else m["top"] + rel_pdf = os.path.join(rel, aid + ".pdf") + expected.add(rel_pdf) + if m["you"]: + expected.add(os.path.join(YOU_DIR, aid + ".pdf")) + + if not os.path.isfile(src): + missing.append(aid) + continue + + if args.dry_run: + copied += 1 + if m["you"]: + you_copied += 1 + continue + + copy_pdf(src, os.path.join(OUT_DIR, rel, aid + ".pdf")) + if m["you"]: + copy_pdf(src, os.path.join(OUT_DIR, YOU_DIR, aid + ".pdf")) + you_copied += 1 + copied += 1 + + removed = 0 + if not args.dry_run and os.path.isdir(OUT_DIR): + for root, _dirs, files in os.walk(OUT_DIR): + for fn in files: + if not fn.lower().endswith(".pdf"): + continue + full = os.path.join(root, fn) + rel_pdf = os.path.relpath(full, OUT_DIR) + if rel_pdf not in expected: + os.remove(full) + removed += 1 + + if not args.dry_run: + os.makedirs(OUT_DIR, exist_ok=True) + with open(CSV_PATH, "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=["分类", "风格", "优", "arxiv_id", "标题", "备注", "被引", "arXiv"]) + writer.writeheader() + writer.writerows(rows) + + print("=== 分类整理结果" + ("（dry-run 预览）" if args.dry_run else "") + " ===") + for key in sorted(stat): + print(f" {key}: {stat[key]} 篇") + print(f" 合计: {len(papers)} 篇 · 已复制 PDF: {copied} · 其中“优”: {you_copied} · 缺 PDF: {len(missing)}" + (f" · 清理过期文件: {removed}" if removed else "")) + if missing: + print(" 缺本地 PDF 的 arxiv_id:", ", ".join(missing)) + if not args.dry_run: + print(f" 输出目录: {OUT_DIR}") + print(f" 清单文件: {CSV_PATH}") + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/reviewer/push_to_github.py b/experiments/literature_review/reviewer/push_to_github.py new file mode 100644 index 0000000..f50718f --- /dev/null +++ b/experiments/literature_review/reviewer/push_to_github.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- +"""把 classified/ 目录推送到 GitHub（免 token、免 REST API，走本机 SSH 密钥）。 + +用法（先确保已运行 sync_classified.py 生成最新 classified/）： + python push_to_github.py # 推送 + python push_to_github.py --dry-run # 只构建对象、不推送 + +原理： + - classified/ 里的 PDF 与仓库 pdf/ 字节一致，直接复用已有 blob SHA（本地计算）。 + - 根目录里固定的文件（README/_meta/index/papers.csv/papers.json/_raw）从 + raw.githubusercontent.com 取内容算 SHA；pdf/ 树用本地 pdf/ 计算。 + - head 从 SSH git-receive-pack 的 ref 广告里读取。 + 因此整个流程不调用 api.github.com，不受未认证 API 限流影响。 +依赖：本机已配置好可访问 github.com 的 SSH 密钥（~/.ssh/id_ed25519）。 +""" +import hashlib +import os +import struct +import subprocess +import sys +import time +import urllib.request +import zlib + +REPO = "Sen-illion/papers-dataset" +RAW = f"https://raw.githubusercontent.com/{REPO}/main" +LOCAL_CLASSIFIED = os.path.join("experiments", "literature_review", "papers", "classified") +LOCAL_PDF_DIR = os.path.join("experiments", "literature_review", "papers", "pdf") +SSH = r"C:\Windows\System32\OpenSSH\ssh.exe" + +# 根目录里固定的条目（不含 classified，classified 每次重建） +ROOT_FILES = ["README.md", "_meta.json", "index.md", "papers.csv", "papers.json"] +RAW_SUBDIR_FILES = ["arxiv_metadata.json", "papers_enriched.json", "pdf_progress.json"] # _raw/ + +pack_objects = [] # (type_int, payload) + + +# ---------- git 对象 ---------- +def sha1(b): + return hashlib.sha1(b).hexdigest() + + +def blob_sha_from_bytes(data): + return sha1(b"blob %d\x00" % len(data) + data) + + +def obj_sha(typ, payload): + return sha1(typ.encode() + b" " + str(len(payload)).encode() + b"\x00" + payload) + + +def add_obj(typ, payload): + pack_objects.append((typ, payload)) + + +def make_blob(payload): + add_obj(3, payload) + return obj_sha("blob", payload) + + +def make_tree(entries): + payload = _tree_payload(entries) + add_obj(2, payload) + return obj_sha("tree", payload) + + +def make_commit(tree_sha, parent_sha, message): + now = int(time.time()) + ident = "Sen-illion " + payload = (f"tree {tree_sha}\nparent {parent_sha}\n" + f"author {ident} {now} +0800\ncommitter {ident} {now} +0800\n\n{message}\n").encode("utf-8") + add_obj(1, payload) + return obj_sha("commit", payload) + + +def _tree_payload(entries): + norm = [] + for mode, name, sha in entries: + m = mode.lstrip("0") or "0" + norm.append((m, name, sha)) + + def key(e): + m, name, _ = e + nb = name.encode("utf-8") + return nb + b"/" if m == "40000" else nb + + norm.sort(key=key) + payload = b"" + for m, name, sha in norm: + payload += m.encode() + b" " + name.encode("utf-8") + b"\x00" + bytes.fromhex(sha) + return payload + + +# ---------- 目录 -> git 树 ---------- +def build_tree_for_dir(dirpath, pdf_map): + entries = [] + for name in os.listdir(dirpath): + full = os.path.join(dirpath, name) + if os.path.isdir(full): + child = build_tree_for_dir(full, pdf_map) + entries.append(("40000", name, child)) + else: + if name.lower().endswith(".pdf"): + sha = pdf_map[name] + if sha is None: + raise SystemExit(f"pdf_map 缺少: {name}") + else: + data = open(full, "rb").read() + sha = make_blob(data) + entries.append(("100644", name, sha)) + return make_tree(entries) + + +# ---------- pack ---------- +def encode_header(typ, size): + out = bytearray() + byte = (typ << 4) | (size & 0x0F) + size >>= 4 + if size: + byte |= 0x80 + out.append(byte) + while size: + byte = size & 0x7F + size >>= 7 + if size: + byte |= 0x80 + out.append(byte) + return bytes(out) + + +def build_pack(objects): + body = b"PACK" + struct.pack(">II", 2, len(objects)) + for typ, payload in objects: + body += encode_header(typ, len(payload)) + zlib.compress(payload) + body += hashlib.sha1(body).digest() + return body + + +# ---------- 远端状态（raw CDN + 本地，不用 api.github.com） ---------- +def raw_fetch(path): + url = f"{RAW}/{path}" + req = urllib.request.Request(url, headers={"User-Agent": "dsh"}) + with urllib.request.urlopen(req, timeout=60) as r: + return r.read() + + +def local_pdf_map(): + m = {} + for fn in os.listdir(LOCAL_PDF_DIR): + if fn.lower().endswith(".pdf"): + with open(os.path.join(LOCAL_PDF_DIR, fn), "rb") as f: + m[fn] = blob_sha_from_bytes(f.read()) + return m + + +def root_nonclassified_entries(pdf_map): + """返回根树里除 classified 之外的条目（mode, name, sha）。""" + entries = [] + for name in ROOT_FILES: + data = raw_fetch(name) + entries.append(("100644", name, blob_sha_from_bytes(data))) + + # _raw/ 子树（树对象已在远端，本地算出相同 sha，不加入 pack） + raw_entries = [] + for name in RAW_SUBDIR_FILES: + data = raw_fetch(f"_raw/{name}") + raw_entries.append(("100644", name, blob_sha_from_bytes(data))) + raw_tree_sha = obj_sha("tree", _tree_payload(raw_entries)) + entries.append(("40000", "_raw", raw_tree_sha)) + + # pdf/ 子树（本地计算，与远端一致） + pdf_entries = [("100644", fn, sha) for fn, sha in pdf_map.items()] + pdf_tree_sha = obj_sha("tree", _tree_payload(pdf_entries)) + entries.append(("40000", "pdf", pdf_tree_sha)) + + return entries + + +# ---------- SSH ---------- +def read_advert(p): + """读取 receive-pack 的 v0 ref 广告，返回 {refname: sha}。""" + refs = {} + while True: + b = p.stdout.read(4) + if not b: + break + ln = int(b, 16) + if ln == 0: + break + payload = p.stdout.read(ln - 4) + if b"\x00" in payload: + payload = payload.split(b"\x00", 1)[0] + parts = payload.split(b" ") + if len(parts) >= 2: + refs[parts[1].decode()] = parts[0].decode() + return refs + + +def probe_head(): + cmd = [SSH, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", + "-T", "git@github.com", "git-receive-pack", f"{REPO}.git"] + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + refs = read_advert(p) + p.stdin.close() + p.terminate() + head = refs.get("refs/heads/main") + if not head: + raise SystemExit("未读到 main 的 head") + return head + + +def push(old_head, new_commit, pack_bytes): + cmd = [SSH, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", + "-T", "git@github.com", "git-receive-pack", f"{REPO}.git"] + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + refs = read_advert(p) + if refs.get("refs/heads/main") != old_head: + print(" [警告] 远端 head 已变化，以广告为准") + old_head = refs.get("refs/heads/main") + + cmdline = f"{old_head} {new_commit} refs/heads/main\x00report-status".encode("utf-8") + p.stdin.write(b"%04x" % (len(cmdline) + 4) + cmdline) + p.stdin.write(b"0000") + p.stdin.write(pack_bytes) + p.stdin.close() + + out = p.stdout.read() + err = p.stderr.read() + p.wait() + return out, err, p.returncode + + +def parse_report(out): + lines = [] + i = 0 + while i + 4 <= len(out): + ln = int(out[i:i + 4], 16) + if ln == 0: + break + if ln > 4: + lines.append(out[i + 4:i + ln].decode("utf-8", "replace").rstrip("\n")) + i += ln + return lines + + +def main(): + print("计算本地 pdf blob map ...") + pdf_map = local_pdf_map() + print(f" pdf_map: {len(pdf_map)} 个") + + print("读取远端根目录固定条目 ...") + root_entries = root_nonclassified_entries(pdf_map) + + print("构建 classified 树 ...") + classified_sha = build_tree_for_dir(LOCAL_CLASSIFIED, pdf_map) + print(" classified tree:", classified_sha) + + root_entries.append(("40000", "classified", classified_sha)) + new_root_sha = make_tree(root_entries) + print(" new root tree:", new_root_sha) + + head = probe_head() + print(" remote head:", head) + + new_commit = make_commit(new_root_sha, head, "Add classified papers (优/好/坏/未筛)") + print(" new commit:", new_commit) + + pack_bytes = build_pack(pack_objects) + print(f"pack objects: {len(pack_objects)} · pack size: {len(pack_bytes)} bytes") + if "--dry-run" in sys.argv: + from collections import Counter + print(" [dry-run] 对象分布:", dict(Counter({1: 'commit', 2: 'tree', 3: 'blob'}[t] for t, _ in pack_objects))) + return + + out, err, rc = push(head, new_commit, pack_bytes) + print("ssh rc:", rc) + if err.strip(): + print("stderr:", err.decode("utf-8", "replace")[:500]) + print("report:") + for line in parse_report(out): + print(" ", line) + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/reviewer/server.py b/experiments/literature_review/reviewer/server.py new file mode 100644 index 0000000..d9fe729 --- /dev/null +++ b/experiments/literature_review/reviewer/server.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +"""论文筛选工具 —— 本地 Web 服务（仅用 Python 标准库，无第三方依赖）。 + +用法： + python server.py [--port 8765] [--no-browser] + +行为： + - 从 experiments/literature_review/papers/papers.json 加载元数据； + - 只纳入 arxiv_id >= "2505"（2025 年 5 月及以后）的论文； + - 通过 http://127.0.0.1: 提供浏览界面； + - 好 / 坏分类结果写入 papers/review_2505.json，可断点续标。 +""" +from __future__ import annotations + +import argparse +import json +import os +import threading +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +HERE = os.path.dirname(os.path.abspath(__file__)) +PAPERS_DIR = os.path.normpath(os.path.join(HERE, "..", "papers")) +PAPERS_JSON = os.path.join(PAPERS_DIR, "papers.json") +PDF_DIR = os.path.join(PAPERS_DIR, "pdf") +STATIC_DIR = os.path.join(HERE, "static") +REVIEW_JSON = os.path.join(PAPERS_DIR, "review_2505.json") + +ARXIV_FROM = "2505" # 只纳入 arxiv_id >= 该值 +LABELS = ("good", "bad") # 可选分类 +STYLES = ("research", "lively") # “好”的两种风格：科研风 / 生动活泼 +DEFAULT_PORT = 8765 + +_lock = threading.Lock() +_papers: list[dict] = [] +_review: dict = {} + + +def load_papers() -> list[dict]: + """加载并筛选 papers.json，按 arxiv_id 升序返回。""" + with open(PAPERS_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + + out: list[dict] = [] + for p in data: + aid = (p.get("arxiv_id") or "").strip() + if aid < ARXIV_FROM: + continue + pdf_path = (p.get("pdf_path") or "").strip() + has_pdf = bool(pdf_path) and os.path.isfile(os.path.join(PAPERS_DIR, pdf_path)) + out.append({ + "arxiv_id": aid, + "title": p.get("title", "") or "", + "authors": p.get("authors", []) or [], + "categories": p.get("categories", []) or [], + "primary_category": p.get("primary_category", "") or "", + "year": p.get("year"), + "published": p.get("published", "") or "", + "cited_by_count": p.get("cited_by_count", 0) or 0, + "arxiv_url": p.get("arxiv_url", "") or "", + "abstract": p.get("abstract", "") or "", + "has_pdf": has_pdf, + }) + out.sort(key=lambda x: x["arxiv_id"]) + return out + + +def load_review() -> dict: + if not os.path.isfile(REVIEW_JSON): + return {} + try: + with open(REVIEW_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except (json.JSONDecodeError, OSError): + return {} + + +def save_review(review: dict) -> None: + tmp = REVIEW_JSON + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(review, f, ensure_ascii=False, indent=2) + os.replace(tmp, REVIEW_JSON) + + +def _entry(review: dict, aid: str) -> dict: + """返回某篇论文的标注条目（无则返回空 dict）。""" + e = review.get(aid) + return e if isinstance(e, dict) else {} + + +def counts(review: dict) -> dict: + c = { + "total": len(_papers), + "good_research": 0, + "good_lively": 0, + "bad": 0, + "unreviewed": 0, + } + for p in _papers: + entry = _entry(review, p["arxiv_id"]) + label = entry.get("label") + if label == "good": + if entry.get("style") == "lively": + c["good_lively"] += 1 + else: + c["good_research"] += 1 + elif label == "bad": + c["bad"] += 1 + else: + c["unreviewed"] += 1 + return c + + +def papers_with_label(review: dict) -> list[dict]: + result = [] + for p in _papers: + entry = _entry(review, p["arxiv_id"]) + label = entry.get("label") if entry.get("label") in LABELS else None + if label == "good": + style = entry.get("style") if entry.get("style") in STYLES else "research" + else: + style = None + note = (entry.get("note") or "").strip() + result.append({**p, "label": label, "style": style, "note": note}) + return result + + +class Handler(BaseHTTPRequestHandler): + server_version = "PaperReviewer/1.0" + + # ---- helpers --------------------------------------------------------- + def _send(self, code: int, body: bytes, ctype: str = "application/octet-stream", + extra: dict | None = None) -> None: + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + if extra: + for k, v in extra.items(): + self.send_header(k, v) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _send_json(self, code: int, obj) -> None: + body = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self._send(code, body, "application/json; charset=utf-8") + + def _read_body(self) -> dict: + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + return {} + raw = self.rfile.read(length) + try: + return json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return {} + + # ---- routing --------------------------------------------------------- + def do_HEAD(self) -> None: # noqa: N802 + # 复用 GET 路由（_send/_serve_pdf 内部已对 HEAD 跳过响应体） + self.do_GET() + + def do_GET(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + path = parsed.path + + if path in ("/", "/index.html"): + return self._serve_static("index.html") + + if path.startswith("/static/"): + name = os.path.basename(path[len("/static/"):]) + return self._serve_static(name) + + if path == "/api/papers": + with _lock: + return self._send_json(200, { + "papers": papers_with_label(_review), + "counts": counts(_review), + }) + + if path == "/favicon.ico": + return self._send(204, b"") + + if path.startswith("/pdf/"): + return self._serve_pdf(path[len("/pdf/"):]) + + return self._send_json(404, {"error": "not found"}) + + def do_POST(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + path = parsed.path + + if path == "/api/classify": + body = self._read_body() + aid = (body.get("arxiv_id") or "").strip() + label = body.get("label") + style = body.get("style") + + if aid not in {p["arxiv_id"] for p in _papers}: + return self._send_json(404, {"error": "unknown arxiv_id", "ok": False}) + + if label not in LABELS and label is not None: + return self._send_json(400, {"error": "label must be 'good', 'bad' or null", "ok": False}) + + with _lock: + if label is None: + _review.pop(aid, None) + else: + entry = _entry(_review, aid) + entry["label"] = label + if label == "good": + entry["style"] = style if style in STYLES else "research" + else: + entry.pop("style", None) + entry["updated_at"] = _now_iso() + _review[aid] = entry + save_review(_review) + return self._send_json(200, {"ok": True, "counts": counts(_review)}) + + if path == "/api/note": + body = self._read_body() + aid = (body.get("arxiv_id") or "").strip() + note = body.get("note") + + if aid not in {p["arxiv_id"] for p in _papers}: + return self._send_json(404, {"error": "unknown arxiv_id", "ok": False}) + + note = note.strip() if isinstance(note, str) else "" + with _lock: + entry = _entry(_review, aid) + if note: + entry["note"] = note + entry["updated_at"] = _now_iso() + _review[aid] = entry + else: + entry.pop("note", None) + if entry: + _review[aid] = entry + else: + _review.pop(aid, None) + save_review(_review) + return self._send_json(200, {"ok": True}) + + if path == "/api/reset": + with _lock: + _review.clear() + save_review(_review) + return self._send_json(200, {"ok": True, "counts": counts(_review)}) + + return self._send_json(404, {"error": "not found"}) + + # ---- static / pdf ---------------------------------------------------- + def _serve_static(self, name: str) -> None: + safe = os.path.basename(name) + full = os.path.join(STATIC_DIR, safe) + if not os.path.isfile(full): + return self._send_json(404, {"error": "not found"}) + ctype = MIME.get(os.path.splitext(safe)[1].lower(), "application/octet-stream") + with open(full, "rb") as f: + return self._send(200, f.read(), ctype) + + def _serve_pdf(self, suffix: str) -> None: + aid = os.path.basename(suffix) + # 接受 "" 或 ".pdf" + if aid.lower().endswith(".pdf"): + aid = aid[:-4] + if aid not in {p["arxiv_id"] for p in _papers}: + return self._send_json(404, {"error": "unknown arxiv_id"}) + + pdf_path = os.path.join(PDF_DIR, aid + ".pdf") + if not os.path.isfile(pdf_path): + return self._send_json(404, {"error": "pdf missing"}) + + size = os.path.getsize(pdf_path) + self.send_response(200) + self.send_header("Content-Type", "application/pdf") + self.send_header("Content-Length", str(size)) + self.send_header("Content-Disposition", f'inline; filename="{aid}.pdf"') + self.end_headers() + if self.command != "HEAD": + with open(pdf_path, "rb") as f: + while True: + chunk = f.read(65536) + if not chunk: + break + self.wfile.write(chunk) + + def log_message(self, fmt, *args): # noqa: A003 + # 简洁日志，避免刷屏 + print("[%s] %s" % (self.address_string(), fmt % args)) + + +def _now_iso() -> str: + import datetime + return datetime.datetime.now().astimezone().isoformat(timespec="seconds") + + +MIME = { + ".html": "text/html; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".png": "image/png", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", +} + + +def main() -> None: + global _papers, _review + parser = argparse.ArgumentParser(description="本地论文筛选服务") + parser.add_argument("--port", type=int, default=DEFAULT_PORT) + parser.add_argument("--no-browser", action="store_true", help="不自动打开浏览器") + args = parser.parse_args() + + _papers = load_papers() + _review = load_review() + + host = "127.0.0.1" + httpd = ThreadingHTTPServer((host, args.port), Handler) + url = f"http://{host}:{args.port}/" + + print(f"论文筛选服务已启动：{url}") + print(f" 共 {len(_papers)} 篇（arxiv_id >= {ARXIV_FROM}），" + f"其中 {sum(1 for p in _papers if p['has_pdf'])} 篇有 PDF") + c = counts(_review) + print(f" 已标注：科研风 {c['good_research']} · 生动活泼 {c['good_lively']} · 坏 {c['bad']} · 未筛 {c['unreviewed']}") + print(f" 分类结果保存到：{REVIEW_JSON}") + print(" 按 Ctrl+C 退出。") + + if not args.no_browser: + try: + webbrowser.open(url) + except Exception as exc: # noqa: BLE001 + print(f" (自动打开浏览器失败: {exc})") + + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\n已退出。") + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/reviewer/static/app.js b/experiments/literature_review/reviewer/static/app.js new file mode 100644 index 0000000..00995a8 --- /dev/null +++ b/experiments/literature_review/reviewer/static/app.js @@ -0,0 +1,355 @@ +/* 论文筛选前端逻辑 */ +(function () { + "use strict"; + + const $ = (s) => document.querySelector(s); + + const state = { + papers: [], // 每篇: {arxiv_id, title, authors, categories, ..., label, style, note} + counts: { total: 0, good_research: 0, good_lively: 0, bad: 0, unreviewed: 0 }, + filter: "all", // all | unreviewed | research | lively | bad + query: "", + currentId: null, + }; + + const els = { + filters: $("#filters"), + search: $("#search"), + list: $("#list"), + listCount: $("#list-count"), + detail: $("#detail"), + statReviewed: $("#stat-reviewed"), + statTotal: $("#stat-total"), + statResearch: $("#stat-research"), + statLively: $("#stat-lively"), + statBad: $("#stat-bad"), + }; + + /* ---------- 数据 ---------- */ + async function load() { + const res = await fetch("/api/papers"); + const data = await res.json(); + state.papers = data.papers; + state.counts = data.counts; + renderStats(); + renderList(); + const first = filtered()[0]; + if (first) select(first.arxiv_id); + } + + function filtered() { + const q = state.query.trim().toLowerCase(); + return state.papers.filter((p) => { + if (state.filter === "unreviewed" && p.label != null) return false; + if (state.filter === "research" && !(p.label === "good" && p.style === "research")) return false; + if (state.filter === "lively" && !(p.label === "good" && p.style === "lively")) return false; + if (state.filter === "bad" && p.label !== "bad") return false; + if (q) { + const hay = [ + p.title, p.arxiv_id, + (p.authors || []).join(" "), + (p.categories || []).join(" "), + ].join(" ").toLowerCase(); + if (!hay.includes(q)) return false; + } + return true; + }); + } + + function byId(id) { + return state.papers.find((p) => p.arxiv_id === id) || null; + } + + /* ---------- 渲染 ---------- */ + function renderStats() { + const c = state.counts; + els.statTotal.textContent = c.total; + els.statReviewed.textContent = c.good_research + c.good_lively + c.bad; + els.statResearch.textContent = "科研风 " + c.good_research; + els.statLively.textContent = "生动 " + c.good_lively; + els.statBad.textContent = "坏 " + c.bad; + } + + function labelText(p) { + if (p.label === "good") return p.style === "lively" ? "生动活泼" : "科研风"; + if (p.label === "bad") return "坏"; + return null; + } + + function flagClass(p) { + if (p.label === "good") return p.style === "lively" ? "flag-lively" : "flag-research"; + if (p.label === "bad") return "flag-bad"; + return ""; + } + + function flagHtml(p) { + const t = labelText(p); + let html = ""; + if (t) html += '' + t + ""; + if (p.note) html += '📝'; + return html ? '
' + html + "
" : ""; + } + + function itemHtml(p) { + const cats = (p.categories || []).slice(0, 3).join(" · "); + const cls = p.arxiv_id === state.currentId ? "item is-current" : "item"; + return ( + '
' + + '
' + p.arxiv_id + "
" + + '
' + + '
' + esc(p.title) + "
" + + '
' + esc(cats || p.primary_category || "") + + (p.year ? " · " + p.year : "") + + (p.cited_by_count ? " · 被引 " + p.cited_by_count : "") + + "
" + + "
" + flagHtml(p) + + "
" + ); + } + + function renderList() { + const items = filtered(); + els.listCount.textContent = "(" + items.length + ")"; + + if (!items.length) { + els.list.innerHTML = '
没有符合条件的论文
'; + return; + } + els.list.innerHTML = items.map(itemHtml).join(""); + } + + function renderDetail(p) { + if (!p) { + els.detail.innerHTML = '
没有可显示的论文
'; + return; + } + const authors = (p.authors || []).join(", "); + const cats = (p.categories || []).map((c) => '' + esc(c) + "").join(" "); + const meta = []; + meta.push('' + esc(p.primary_category || "") + ""); + if (p.year) meta.push('' + p.year + ""); + meta.push('被引 ' + (p.cited_by_count || 0) + ""); + + const goodText = p.label === "good" + ? (p.style === "lively" ? "好 · 生动活泼" : "好 · 科研风") + : "好"; + const goodOn = p.label === "good" ? " is-on" : ""; + const badOn = p.label === "bad" ? " is-on" : ""; + const researchOn = (p.label === "good" && p.style === "research") ? " is-current" : ""; + const livelyOn = (p.label === "good" && p.style === "lively") ? " is-current" : ""; + + let pdfHtml; + if (p.has_pdf) { + pdfHtml = ''; + } else { + pdfHtml = '
此篇缺少本地 PDF（可点 arXiv 链接在线查看）
'; + } + + els.detail.innerHTML = + '
' + + '
' + + '' + + '
' + + '' + + '' + + "
" + + "
" + + '' + + '' + + (p.arxiv_url ? 'arXiv ↗' : "") + + "
" + + '

' + esc(p.title) + "

" + + '
' + esc(authors) + "
" + + '
' + meta.join("") + '' + esc(p.arxiv_id) + "
" + + '
' + pdfHtml + "
" + + (p.abstract + ? '
摘要' + esc(p.abstract) + "
" + : "") + + '
' + + '' + + '" + + '' + + "
"; + + bindNoteInput(p.arxiv_id); + } + + function select(id) { + state.currentId = id; + renderList(); + renderDetail(byId(id)); + } + + function move(delta) { + const items = filtered(); + if (!items.length) return; + let idx = items.findIndex((p) => p.arxiv_id === state.currentId); + if (idx === -1) idx = 0; + idx = Math.max(0, Math.min(items.length - 1, idx + delta)); + select(items[idx].arxiv_id); + const el = els.list.querySelector('[data-id="' + items[idx].arxiv_id + '"]'); + if (el && el.scrollIntoView) el.scrollIntoView({ block: "nearest" }); + } + + /* ---------- 分类 ---------- */ + async function classify(label, style) { + const p = byId(state.currentId); + if (!p) return; + + const oldItems = filtered(); + const oldIdx = oldItems.findIndex((x) => x.arxiv_id === state.currentId); + + const res = await fetch("/api/classify", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ arxiv_id: p.arxiv_id, label: label, style: style }), + }); + const data = await res.json(); + if (!data.ok) return; + + p.label = label; + p.style = label === "good" ? (style || "research") : null; + state.counts = data.counts; + + renderStats(); + renderList(); + + // 自动跳到下一篇（在"未筛"等会移除当前项的视图里，oldIdx 即下一条） + const items = filtered(); + let idx = oldIdx === -1 ? 0 : oldIdx; + if (state.filter === "all") idx += 1; + idx = Math.max(0, Math.min(items.length - 1, idx)); + const next = items.length ? items[idx] : null; + + if (next) { + state.currentId = next.arxiv_id; + renderList(); + renderDetail(byId(state.currentId)); + } else { + state.currentId = null; + renderList(); + renderDetail(null); + } + } + + /* ---------- 备注 ---------- */ + let noteTimer = null; + + function bindNoteInput(id) { + const ta = els.detail.querySelector(".note-input"); + if (!ta) return; + ta.addEventListener("input", () => { + clearTimeout(noteTimer); + noteTimer = setTimeout(() => saveNote(id, ta.value), 600); + }); + ta.addEventListener("blur", () => { + clearTimeout(noteTimer); + saveNote(id, ta.value); + }); + } + + async function saveNote(id, value) { + const res = await fetch("/api/note", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ arxiv_id: id, note: value }), + }); + const data = await res.json(); + if (!data.ok) return; + + const p = byId(id); + if (p) p.note = value.trim(); + + // 就地更新列表里的备注标记，不整列表重绘（避免滚动跳动） + const item = els.list.querySelector('[data-id="' + id + '"]'); + if (item && p) item.outerHTML = itemHtml(p); + + if (state.currentId === id) { + const st = els.detail.querySelector("[data-note-status]"); + if (st) { + st.textContent = "已保存"; + st.classList.add("saved"); + setTimeout(() => { if (st.isConnected) { st.textContent = ""; st.classList.remove("saved"); } }, 1500); + } + } + } + + /* ---------- 事件 ---------- */ + function closeMenus() { + els.detail.querySelectorAll(".good-wrap.open").forEach((w) => w.classList.remove("open")); + } + + els.filters.addEventListener("click", (e) => { + const btn = e.target.closest("[data-filter]"); + if (!btn) return; + state.filter = btn.dataset.filter; + els.filters.querySelectorAll(".chip").forEach((c) => c.classList.remove("is-active")); + btn.classList.add("is-active"); + renderList(); + const first = filtered()[0]; + if (first) select(first.arxiv_id); + else renderDetail(null); + }); + + els.search.addEventListener("input", (e) => { + state.query = e.target.value; + renderList(); + const first = filtered()[0]; + if (first) select(first.arxiv_id); + else renderDetail(null); + }); + + els.list.addEventListener("click", (e) => { + const item = e.target.closest("[data-id]"); + if (item) select(item.dataset.id); + }); + + els.detail.addEventListener("click", (e) => { + const btn = e.target.closest("[data-act]"); + if (btn && btn.dataset.act === "good") { + closeMenus(); + btn.closest(".good-wrap").classList.toggle("open"); + return; + } + const mi = e.target.closest("[data-style]"); + if (mi) { + closeMenus(); + classify("good", mi.dataset.style); + return; + } + if (btn) { + if (btn.dataset.act === "bad") classify("bad"); + else if (btn.dataset.act === "clear") classify(null); + } + }); + + document.addEventListener("click", (e) => { + if (!e.target.closest(".good-wrap")) closeMenus(); + }); + + document.addEventListener("keydown", (e) => { + const tag = (e.target.tagName || "").toLowerCase(); + if (tag === "input" || tag === "textarea" || tag === "select") return; + if (e.metaKey || e.ctrlKey || e.altKey) return; + + switch (e.key) { + case "1": classify("good", "research"); e.preventDefault(); break; + case "2": classify("good", "lively"); e.preventDefault(); break; + case "b": case "B": classify("bad"); e.preventDefault(); break; + case "u": case "U": classify(null); e.preventDefault(); break; + case "ArrowLeft": move(-1); e.preventDefault(); break; + case "ArrowRight": move(1); e.preventDefault(); break; + } + }); + + /* ---------- 工具 ---------- */ + function esc(s) { + return String(s == null ? "" : s) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); + } + + load(); +})(); diff --git a/experiments/literature_review/reviewer/static/index.html b/experiments/literature_review/reviewer/static/index.html new file mode 100644 index 0000000..b4df511 --- /dev/null +++ b/experiments/literature_review/reviewer/static/index.html @@ -0,0 +1,52 @@ + + + + + + 论文筛选 · 2505+ + + + +
+
+ 论文筛选 + arxiv_id ≥ 2505 +
+ +
+ + + + + +
+ + + +
+ 0/0 已筛 + 科研风 0 + 生动 0 + 坏 0 +
+
+ +
+ + +
+
← 从左侧选择一篇论文开始筛选
+
+
+ +
+ 1 科研风   2 生动活泼   B 坏   U 清除   + ←→ 上一篇/下一篇 +
+ + + + diff --git a/experiments/literature_review/reviewer/static/style.css b/experiments/literature_review/reviewer/static/style.css new file mode 100644 index 0000000..93c5317 --- /dev/null +++ b/experiments/literature_review/reviewer/static/style.css @@ -0,0 +1,266 @@ +:root { + --bg: #f6f7f9; + --panel: #ffffff; + --border: #e4e7ec; + --text: #1f2733; + --muted: #7a8598; + --accent: #2f6fed; + --research: #2563eb; + --research-bg: #e8eefc; + --lively: #d97706; + --lively-bg: #fdf1e3; + --bad: #d64545; + --bad-bg: #fdeeee; + --radius: 10px; +} + +* { box-sizing: border-box; } + +html, body { + margin: 0; + height: 100%; + font-family: -apple-system, "Segoe UI", "Microsoft YaHei", Roboto, sans-serif; + color: var(--text); + background: var(--bg); + font-size: 14px; +} + +/* ---- top bar ---- */ +.topbar { + display: flex; + align-items: center; + gap: 14px; + padding: 10px 16px; + background: var(--panel); + border-bottom: 1px solid var(--border); + flex-wrap: wrap; +} +.brand-title { font-size: 17px; font-weight: 700; } +.brand-sub { font-size: 12px; color: var(--muted); margin-left: 6px; } + +.filters { display: flex; gap: 6px; } +.chip { + border: 1px solid var(--border); + background: #fff; + color: var(--text); + border-radius: 999px; + padding: 5px 13px; + cursor: pointer; + font-size: 13px; +} +.chip:hover { border-color: var(--accent); } +.chip.is-active { background: var(--accent); border-color: var(--accent); color: #fff; } +.chip-research.is-active { background: var(--research); border-color: var(--research); } +.chip-lively.is-active { background: var(--lively); border-color: var(--lively); } +.chip-bad.is-active { background: var(--bad); border-color: var(--bad); } + +.search { + flex: 1 1 200px; + min-width: 160px; + padding: 7px 12px; + border: 1px solid var(--border); + border-radius: 999px; + outline: none; +} +.search:focus { border-color: var(--accent); } + +.stats { display: flex; gap: 12px; margin-left: auto; font-size: 13px; } +.stat { color: var(--muted); } +.stat b { color: var(--text); } +.stat-research { color: var(--research); font-weight: 600; } +.stat-lively { color: var(--lively); font-weight: 600; } +.stat-bad { color: var(--bad); font-weight: 600; } + +/* ---- layout ---- */ +.layout { + display: grid; + grid-template-columns: 380px 1fr; + gap: 0; + height: calc(100vh - 57px); +} +@media (max-width: 900px) { + .layout { grid-template-columns: 1fr; grid-template-rows: 45% 1fr; } +} + +.list-pane { + border-right: 1px solid var(--border); + background: var(--panel); + display: flex; + flex-direction: column; + min-height: 0; +} +.list-head { + padding: 10px 14px; + font-weight: 600; + border-bottom: 1px solid var(--border); + font-size: 13px; +} +.muted { color: var(--muted); font-weight: 400; } + +.list { overflow-y: auto; flex: 1; } +.item { + padding: 10px 14px; + border-bottom: 1px solid var(--border); + cursor: pointer; + display: flex; + gap: 10px; + align-items: flex-start; +} +.item:hover { background: #f0f4fb; } +.item.is-current { background: #e8f0fe; } +.item .id { font-family: ui-monospace, Consolas, monospace; font-size: 12px; color: var(--muted); white-space: nowrap; } +.item .body { flex: 1; min-width: 0; } +.item .title { font-weight: 600; line-height: 1.35; margin: 2px 0 3px; } +.item .meta { font-size: 12px; color: var(--muted); } +.flags { flex: 0 0 auto; display: flex; flex-direction: column; gap: 3px; align-items: flex-end; margin-left: auto; } +.flag { font-size: 11px; padding: 2px 8px; border-radius: 999px; white-space: nowrap; } +.flag-research { background: var(--research-bg); color: var(--research); } +.flag-lively { background: var(--lively-bg); color: var(--lively); } +.flag-bad { background: var(--bad-bg); color: var(--bad); } +.note-dot { font-size: 12px; } + +/* ---- detail ---- */ +.detail-pane { min-width: 0; overflow-y: auto; padding: 16px 20px 120px; } +.empty { color: var(--muted); text-align: center; margin-top: 80px; } + +.d-head { display: flex; gap: 16px; align-items: flex-start; margin-bottom: 14px; } +.d-title { font-size: 20px; font-weight: 700; line-height: 1.3; margin: 0; } +.d-authors { color: var(--muted); margin: 8px 0 4px; } +.d-meta { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px; } +.tag { + font-size: 12px; + background: #eef1f6; + color: #445; + padding: 2px 9px; + border-radius: 6px; +} +.tag-cat { background: #e8f0fe; color: #2f6fed; } +.tag-cited { background: #fff4e0; color: #b07a00; } + +.d-actions { + position: sticky; + top: 0; + z-index: 5; + display: flex; + gap: 10px; + align-items: center; + background: var(--bg); + padding: 10px 0; + margin-bottom: 8px; +} +.btn { + border: none; + border-radius: 8px; + padding: 9px 20px; + font-size: 14px; + font-weight: 600; + cursor: pointer; + color: #fff; +} +.btn-good { background: var(--research); } +.btn-bad { background: var(--bad); } +.btn-clear { background: #e7eaf0; color: #445; } +.btn:hover { filter: brightness(0.95); } +.btn.is-on { box-shadow: 0 0 0 3px rgba(0,0,0,.15) inset; } + +/* 好 → 子类下拉 */ +.good-wrap { position: relative; display: inline-block; } +.good-menu { + position: absolute; + top: calc(100% + 4px); + left: 0; + background: #fff; + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: 0 6px 18px rgba(0,0,0,.12); + padding: 4px; + display: none; + z-index: 10; + min-width: 130px; +} +.good-wrap.open .good-menu { display: block; } +.menu-item { + display: block; + width: 100%; + text-align: left; + border: none; + background: none; + padding: 8px 12px; + border-radius: 6px; + cursor: pointer; + font-size: 14px; +} +.menu-item:hover { background: #f0f4fb; } +.menu-item.is-current { background: var(--research-bg); color: var(--research); font-weight: 600; } + +.pdf-wrap { + border: 1px solid var(--border); + border-radius: var(--radius); + overflow: hidden; + background: #fafbfc; + height: 70vh; + margin-bottom: 16px; +} +.pdf-wrap iframe { width: 100%; height: 100%; border: none; } +.pdf-missing { + display: flex; + align-items: center; + justify-content: center; + height: 100%; + color: var(--muted); +} + +.abs { + background: var(--panel); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 14px 16px; + line-height: 1.6; + color: #37404d; + margin-bottom: 16px; +} +.abs summary { cursor: pointer; font-weight: 600; margin-bottom: 8px; } + +/* 备注 */ +.note-block { + background: var(--panel); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 12px 14px; +} +.note-label { font-weight: 600; display: block; margin-bottom: 6px; } +.note-input { + width: 100%; + min-height: 72px; + border: 1px solid var(--border); + border-radius: 8px; + padding: 8px 10px; + font-family: inherit; + font-size: 13px; + resize: vertical; + outline: none; +} +.note-input:focus { border-color: var(--accent); } +.note-status { font-size: 12px; color: var(--muted); } +.note-status.saved { color: #1a9e5c; } + +/* ---- hint ---- */ +.kb-hint { + position: fixed; + right: 14px; + bottom: 12px; + font-size: 12px; + color: var(--muted); + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + padding: 6px 10px; + box-shadow: 0 2px 8px rgba(0,0,0,.06); +} +kbd { + background: #eef1f6; + border: 1px solid #d8dee8; + border-radius: 4px; + padding: 0 5px; + font-family: ui-monospace, Consolas, monospace; +} diff --git a/experiments/literature_review/reviewer/sync_classified.py b/experiments/literature_review/reviewer/sync_classified.py new file mode 100644 index 0000000..88fbbde --- /dev/null +++ b/experiments/literature_review/reviewer/sync_classified.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +"""重建 classified/ 并生成清单 CSV。 + +分类来源（合并）： + 1. review_2505.json —— 2505+ 论文（好/坏/未筛 + 风格 + 备注/优） + 2. Downloads 三个文件夹 —— 2301~2504 手动筛出的「科研风 / 生动活泼 / 反面教材(坏)」 + +用法： + python sync_classified.py # 重建（复制 + 清理过期 + 写 CSV） + python sync_classified.py --dry-run # 只打印统计，不写盘 +""" +import argparse +import csv +import json +import os +import shutil + +HERE = os.path.dirname(os.path.abspath(__file__)) +PAPERS_DIR = os.path.normpath(os.path.join(HERE, "..", "papers")) +PAPERS_JSON = os.path.join(PAPERS_DIR, "papers.json") +PDF_DIR = os.path.join(PAPERS_DIR, "pdf") +REVIEW_JSON = os.path.join(PAPERS_DIR, "review_2505.json") +CLASSIFIED = os.path.join(PAPERS_DIR, "classified") +CSV_PATH = os.path.join(CLASSIFIED, "分类清单.csv") + +DOWN = { + "research": r"C:\Users\Lenovo\Downloads\1.科研风\1.科研风", + "lively": r"C:\Users\Lenovo\Downloads\2.生动型\2.生动型", + "bad": r"C:\Users\Lenovo\Downloads\3.反面教材", +} +YOU_DIR = "优" + + +def copy_pdf(src, dst): + if not os.path.isfile(dst) or os.path.getsize(dst) != os.path.getsize(src): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + papers = {p["arxiv_id"]: p for p in json.load(open(PAPERS_JSON, encoding="utf-8"))} + review = json.load(open(REVIEW_JSON, encoding="utf-8")) + + # aid -> (top, sub, note, you) + cmap = {} + + # 1) 2505+ 从 review + for aid, p in papers.items(): + if aid < "2505": + continue + e = review.get(aid) or {} + label = e.get("label") + style = e.get("style") + note = (e.get("note") or "").strip() + if label == "good": + top, sub = "好", ("生动活泼" if style == "lively" else "科研风") + elif label == "bad": + top, sub = "坏", "" + else: + top, sub = "未筛", "" + you = (label == "good" and "优" in note) + cmap[aid] = (top, sub, note, you) + + # 2) Downloads（2301~2504） + down_ids = {cat: set() for cat in DOWN} + for cat, d in DOWN.items(): + if not os.path.isdir(d): + print(f"[警告] 目录不存在：{d}") + continue + for f in os.listdir(d): + if f.lower().endswith(".pdf"): + down_ids[cat].add(f[:-4]) + for aid in down_ids["research"]: + cmap[aid] = ("好", "科研风", "", False) + for aid in down_ids["lively"]: + cmap[aid] = ("好", "生动活泼", "", False) + for aid in down_ids["bad"]: + cmap[aid] = ("坏", "", "", False) + + # 复制 + 清理 + expected = set() + copied = you_copied = 0 + missing = [] + for aid, (top, sub, note, you) in cmap.items(): + rel = os.path.join(top, sub) if sub else top + src = os.path.join(PDF_DIR, aid + ".pdf") + rel_pdf = os.path.join(rel, aid + ".pdf") + if not os.path.isfile(src): + missing.append(aid) + continue + expected.add(rel_pdf) + if you: + expected.add(os.path.join(YOU_DIR, aid + ".pdf")) + if not args.dry_run: + copy_pdf(src, os.path.join(CLASSIFIED, rel, aid + ".pdf")) + if you: + copy_pdf(src, os.path.join(CLASSIFIED, YOU_DIR, aid + ".pdf")) + you_copied += 1 + copied += 1 + + removed = 0 + if not args.dry_run and os.path.isdir(CLASSIFIED): + for root, _dirs, files in os.walk(CLASSIFIED): + for fn in files: + if not fn.lower().endswith(".pdf"): + continue + full = os.path.join(root, fn) + if os.path.relpath(full, CLASSIFIED) not in expected: + os.remove(full) + removed += 1 + + # CSV + rows = [] + for aid in sorted(cmap): + top, sub, note, you = cmap[aid] + p = papers.get(aid, {}) + rows.append([top, sub, "优" if you else "", aid, + p.get("title", "") or "", note, + p.get("cited_by_count", 0) or 0, p.get("arxiv_url", "") or ""]) + if not args.dry_run: + with open(CSV_PATH, "w", newline="", encoding="utf-8-sig") as f: + w = csv.writer(f) + w.writerow(["分类", "风格", "优", "arxiv_id", "标题", "备注", "被引", "arXiv"]) + w.writerows(rows) + + # 统计 + stat = {} + for aid, (top, sub, note, you) in cmap.items(): + key = f"{top}/{sub}" if sub else top + stat[key] = stat.get(key, 0) + 1 + stat[YOU_DIR] = sum(1 for _, _, _, you in cmap.values() if you) + + print("=== 分类重建" + ("（dry-run）" if args.dry_run else "") + " ===") + for k in sorted(stat): + print(f" {k}: {stat[k]}") + print(f" 合计: {len(cmap)} 篇 · 有 PDF: {copied} · 其中优: {you_copied} · 缺 PDF: {len(missing)}" + (f" · 清理过期: {removed}" if removed else "")) + if missing: + print(" 缺 PDF:", ", ".join(sorted(missing))) + if not args.dry_run: + print(f" CSV: {CSV_PATH} ({len(rows)} 行)") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index a78f4df..bbc58df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,16 +8,22 @@ version = "0.1.0" description = "PPTX-first research figure generation pipeline guided by papers and reference images." requires-python = ">=3.10" dependencies = [ + "numpy", "pillow", "python-pptx", "PyMuPDF", + "pypdf", + "pdfplumber", + "flask", "requests", "opencv-python-headless", + "wordninja", "pywin32; platform_system == 'Windows'" ] [project.optional-dependencies] ocr = [ + "rapidocr-onnxruntime", "paddleocr", "easyocr" ] @@ -27,4 +33,4 @@ rfs = "rfs.cli:main" [tool.setuptools.packages.find] where = ["."] -include = ["rfs*"] +include = ["rfs*", "webapp*"] diff --git a/reference-figures/README.md b/reference-figures/README.md new file mode 100644 index 0000000..0132143 --- /dev/null +++ b/reference-figures/README.md @@ -0,0 +1,50 @@ +# Reference Figures + +Put local paper framework figures here for the paper-to-image reference learning workflow. + +Suggested use: + +- `excellent/`: strong framework figures that Paperimage should learn from. +- `negative/`: weak or unwanted styles that Paperimage should avoid. +- `user/`: reserved for future user-provided style learning. + +Do not commit real paper figures, screenshots, PDFs, or private data. The workflow will write structured analysis records to an output directory; those records are safer to review and curate than the original images. + +Stage 1 command: + +```powershell +rfs analyze-reference-figures ` + --references reference-figures ` + --out output/reference_learning_stage1 ` + --mode heuristic ` + --json +``` + +Stage 2 API-backed library build: + +```powershell +Get-Content .env | ForEach-Object { + if ($_ -match '^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$') { + [Environment]::SetEnvironmentVariable($matches[1], $matches[2].Trim("'"""), "Process") + } +} + +rfs analyze-reference-figures ` + --references reference-figures ` + --out output/reference_learning_stage2 ` + --mode vlm ` + --model gpt-5.6-sol ` + --json +``` + +Stage 3 exemplar selection for a paper run: + +```powershell +rfs select-reference-exemplars ` + --paper-review output/paper_to_image/paper_review.json ` + --library output/reference_learning_stage2/paper_to_image_reference_library.jsonl ` + --out output/paper_to_image/reference_exemplars ` + --selected-template output/paper_to_image/selected_template.json ` + --top-k 5 ` + --json +``` diff --git a/reference-figures/excellent/.gitkeep b/reference-figures/excellent/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference-figures/excellent/.gitkeep @@ -0,0 +1 @@ + diff --git a/reference-figures/negative/.gitkeep b/reference-figures/negative/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference-figures/negative/.gitkeep @@ -0,0 +1 @@ + diff --git a/reference-figures/user/.gitkeep b/reference-figures/user/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference-figures/user/.gitkeep @@ -0,0 +1 @@ + diff --git a/rfs/analysis/__init__.py b/rfs/analysis/__init__.py new file mode 100644 index 0000000..f181c12 --- /dev/null +++ b/rfs/analysis/__init__.py @@ -0,0 +1,16 @@ +"""Input and visual analysis entry points. + +Implementations remain in their existing modules while interfaces stabilize. +""" + +from ..paper_to_image.analyzer import parse_paper +from .rebuild import estimate_background, extract_reference_text, group_text_regions, group_text_regions_heuristic, plan_reference_layout + +__all__ = [ + "estimate_background", + "extract_reference_text", + "group_text_regions", + "group_text_regions_heuristic", + "parse_paper", + "plan_reference_layout", +] diff --git a/rfs/analysis/rebuild/__init__.py b/rfs/analysis/rebuild/__init__.py new file mode 100644 index 0000000..79efd3e --- /dev/null +++ b/rfs/analysis/rebuild/__init__.py @@ -0,0 +1,16 @@ +"""Canonical analysis APIs for reference-image editable rebuilds.""" + +from .controls import localize_reference_controls +from .layout import dominant_palette, estimate_background, plan_reference_layout +from .reference_text import extract_reference_text +from ...text_grouping import group_text_regions, group_text_regions_heuristic + +__all__ = [ + "estimate_background", + "dominant_palette", + "extract_reference_text", + "group_text_regions", + "group_text_regions_heuristic", + "localize_reference_controls", + "plan_reference_layout", +] diff --git a/rfs/analysis/rebuild/controls.py b/rfs/analysis/rebuild/controls.py new file mode 100644 index 0000000..fae3bd2 --- /dev/null +++ b/rfs/analysis/rebuild/controls.py @@ -0,0 +1,832 @@ +from __future__ import annotations + +import math +import inspect +from pathlib import Path +from typing import Callable + +from PIL import Image, ImageDraw + +from ...utils import read_json, write_json +from ...cv_compat import hough_line_coordinates + +VALID_RENDER_STYLES = { + "filled_block_arrow", + "line_connector", + "elbow_connector", + "branch_line_connector", + "dashed_loop_connector", +} +VALID_ROUTE_INTENTS = {"straight", "orthogonal", "branch", "loop"} +VALID_VISUAL_WEIGHTS = {"chunky", "normal", "thin"} +VALID_PREFERRED_AXES = {"horizontal", "vertical", "horizontal_first", "vertical_first"} +VALID_LINE_PATTERNS = {"solid", "dash", "dashed"} + + +def _center(box: dict) -> tuple[float, float]: + return float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2 + + +def _nearest_slot(point: tuple[float, float], slots: list[dict]) -> dict | None: + if not slots: + return None + px, py = point + return min(slots, key=lambda slot: (_center(slot["bbox_percent"])[0] - px) ** 2 + (_center(slot["bbox_percent"])[1] - py) ** 2) + + +def _distance_to_bbox(point: tuple[float, float], obj: dict | None) -> float: + if not obj or not isinstance(obj.get("bbox_percent"), dict): + return 1.0 + box = obj["bbox_percent"] + px, py = point + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + dx = max(x - px, 0.0, px - (x + w)) + dy = max(y - py, 0.0, py - (y + h)) + return math.hypot(dx, dy) + + +def _coerce_choice(value, allowed: set[str], default: str = "") -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "block_arrow": "filled_block_arrow", + "chunky_arrow": "filled_block_arrow", + "thick_arrow": "filled_block_arrow", + "straight_arrow": "line_connector", + "elbow": "elbow_connector", + "orthogonal_connector": "elbow_connector", + "branch_connector": "branch_line_connector", + "dashed_loop": "dashed_loop_connector", + "loop_connector": "dashed_loop_connector", + "dashed": "dash", + } + text = aliases.get(text, text) + return text if text in allowed else default + + +def _slot_by_id(slots: list[dict]) -> dict[str, dict]: + return {str(slot.get("id")): slot for slot in slots if isinstance(slot, dict) and slot.get("id")} + + +def _path_bbox(points: list[list[float]]) -> dict[str, float]: + xs = [float(point[0]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + ys = [float(point[1]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + if not xs or not ys: + return {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0} + x0 = min(xs) + y0 = min(ys) + return {"x": x0, "y": y0, "w": max(xs) - x0, "h": max(ys) - y0} + + +def _bbox_iou(a: dict, b: dict) -> float: + ax0, ay0 = float(a.get("x", 0.0)), float(a.get("y", 0.0)) + ax1, ay1 = ax0 + float(a.get("w", 0.0)), ay0 + float(a.get("h", 0.0)) + bx0, by0 = float(b.get("x", 0.0)), float(b.get("y", 0.0)) + bx1, by1 = bx0 + float(b.get("w", 0.0)), by0 + float(b.get("h", 0.0)) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + inter = ix * iy + union = float(a.get("w", 0.0)) * float(a.get("h", 0.0)) + float(b.get("w", 0.0)) * float(b.get("h", 0.0)) - inter + return inter / max(union, 0.000001) + + +def _point_distance(a: list[float], b: list[float]) -> float: + return math.hypot(float(a[0]) - float(b[0]), float(a[1]) - float(b[1])) + + +def _path_distance(a: list[list[float]], b: list[list[float]]) -> float: + if len(a) < 2 or len(b) < 2: + return 1.0 + return (_point_distance(a[0], b[0]) + _point_distance(a[-1], b[-1])) / 2 + + +def _control_signature(control: dict) -> tuple[str, str]: + return str(control.get("source_id") or control.get("source") or ""), str(control.get("target_id") or control.get("target") or "") + + +def _controls_are_duplicate(a: dict, b: dict) -> bool: + ap = a.get("path_percent") or [] + bp = b.get("path_percent") or [] + same_binding = _control_signature(a) == _control_signature(b) + if same_binding and _path_distance(ap, bp) <= 0.08: + return True + return _bbox_iou(_path_bbox(ap), _path_bbox(bp)) >= 0.55 and _path_distance(ap, bp) <= 0.12 + + +def _controls_match_for_disagreement(heuristic: dict, vlm: dict) -> bool: + if _controls_are_duplicate(heuristic, vlm): + return True + hp = heuristic.get("path_percent") or [] + vp = vlm.get("path_percent") or [] + return _control_signature(heuristic) == _control_signature(vlm) or _path_distance(hp, vp) <= 0.08 or _bbox_iou(_path_bbox(hp), _path_bbox(vp)) >= 0.35 + + +def _bindable_objects(out_dir: Path, slots: list[dict]) -> list[dict]: + bindable = [dict(slot) for slot in slots if isinstance(slot, dict) and isinstance(slot.get("bbox_percent"), dict)] + seen = {str(item.get("id")) for item in bindable} + try: + geometry = read_json(out_dir / "reference_geometry.json") + except Exception: + geometry = {} + for card in geometry.get("cards", []) if isinstance(geometry, dict) else []: + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + card_id = str(card.get("id") or "") + if not card_id or card_id in seen: + continue + enriched = dict(card) + enriched.setdefault("display_label", card.get("title") or card_id) + enriched.setdefault("paper_concept", card.get("title") or card_id) + enriched.setdefault("semantic_role", "reference_card") + enriched.setdefault("asset_type", "editable_card") + bindable.append(enriched) + seen.add(card_id) + return bindable + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _center(sbox) + tx, ty = _center(tbox) + if abs(tx - sx) >= abs(ty - sy): + start = (float(sbox["x"]) + (float(sbox["w"]) if tx >= sx else 0.0), sy) + end = (float(tbox["x"]) + (0.0 if tx >= sx else float(tbox["w"])), ty) + else: + start = (sx, float(sbox["y"]) + (float(sbox["h"]) if ty >= sy else 0.0)) + end = (tx, float(tbox["y"]) + (0.0 if ty >= sy else float(tbox["h"]))) + return [[round(start[0], 4), round(start[1], 4)], [round(end[0], 4), round(end[1], 4)]] + + +def _sequence_controls(slots: list[dict], palette: list[str]) -> list[dict]: + ordered = sorted(slots, key=lambda slot: (float(slot["bbox_percent"]["y"]), float(slot["bbox_percent"]["x"]))) + arrows = [] + for idx, (src, dst) in enumerate(zip(ordered[:-1], ordered[1:]), start=1): + sbox = src["bbox_percent"] + dbox = dst["bbox_percent"] + sx = float(sbox["x"]) + float(sbox["w"]) + sy = float(sbox["y"]) + float(sbox["h"]) / 2 + tx = float(dbox["x"]) + ty = float(dbox["y"]) + float(dbox["h"]) / 2 + dx = tx - sx + dy = ty - sy + arrows.append({ + "id": f"arrow_{idx:02d}", + "source": src["id"], + "target": dst["id"], + "source_id": src["id"], + "target_id": dst["id"], + "control_kind": "straight_arrow", + "path_percent": [[round(sx, 4), round(sy, 4)], [round(tx, 4), round(ty, 4)]], + "arrowhead_direction": round(math.degrees(math.atan2(dy, dx)), 2), + "stroke_color": palette[2] if len(palette) > 2 else "#4A6FA5", + "stroke_width_pt": 1.5, + "dash_style": "solid", + "line_pattern": "solid", + "confidence": 0.35, + "detected_by": "sequence_fallback", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + }) + return arrows + + +def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str]) -> list[dict]: + try: + import cv2 + except Exception: + return [] + image = cv2.imread(str(reference_path)) + if image is None: + return [] + height, width = image.shape[:2] + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + edges = cv2.Canny(gray, 50, 150, apertureSize=3) + lines = cv2.HoughLinesP(edges, 1, math.pi / 180, threshold=40, minLineLength=max(24, min(width, height) // 14), maxLineGap=12) + if lines is None: + return [] + candidates = [] + for line in lines[:80]: + coordinates = hough_line_coordinates(line) + if coordinates is None: + values = line.reshape(-1) if hasattr(line, "reshape") else line + if len(values) < 4: + continue + coordinates = [int(v) for v in values[:4]] + x1, y1, x2, y2 = coordinates + length = math.hypot(x2 - x1, y2 - y1) + if length < min(width, height) * 0.06: + continue + p1 = (x1 / width, y1 / height) + p2 = (x2 / width, y2 / height) + source = _nearest_slot(p1, slots) + target = _nearest_slot(p2, slots) + if not source or not target or source["id"] == target["id"]: + continue + if _distance_to_bbox(p1, source) > 0.075 or _distance_to_bbox(p2, target) > 0.075: + continue + direction = math.degrees(math.atan2(p2[1] - p1[1], p2[0] - p1[0])) + candidates.append({ + "line": [p1, p2], + "length": length, + "source": source, + "target": target, + "direction": direction, + }) + candidates.sort(key=lambda item: item["length"], reverse=True) + arrows = [] + seen = set() + for candidate in candidates: + source = candidate["source"]["id"] + target = candidate["target"]["id"] + key = (source, target) + if key in seen: + continue + seen.add(key) + p1, p2 = candidate["line"] + arrows.append({ + "id": f"arrow_{len(arrows) + 1:02d}", + "source": source, + "target": target, + "source_id": source, + "target_id": target, + "control_kind": "straight_arrow", + "path_percent": [[round(p1[0], 4), round(p1[1], 4)], [round(p2[0], 4), round(p2[1], 4)]], + "arrowhead_direction": round(candidate["direction"], 2), + "stroke_color": palette[2] if len(palette) > 2 else "#4A6FA5", + "stroke_width_pt": 1.4, + "dash_style": "solid", + "line_pattern": "solid", + "confidence": 0.55, + "detected_by": "cv_hough_line", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + }) + if len(arrows) >= max(1, min(16, len(slots) + 2)): + break + return arrows + + +def _normalize_controls(raw: list[dict], slots: list[dict], palette: list[str]) -> list[dict]: + slot_ids = {str(slot.get("id")) for slot in slots} + objects = _slot_by_id(slots) + normalized = [] + for idx, item in enumerate(raw or [], start=1): + if not isinstance(item, dict): + continue + source = str(item.get("source_id") or item.get("source") or "") + target = str(item.get("target_id") or item.get("target") or "") + path = item.get("path_percent") + if source not in slot_ids or target not in slot_ids: + continue + points = [] + if isinstance(path, list): + for point in path: + if not isinstance(point, (list, tuple)) or len(point) < 2: + continue + points.append([round(max(0.0, min(1.0, float(point[0]))), 4), round(max(0.0, min(1.0, float(point[1]))), 4)]) + has_reference_path = len(points) >= 2 + if len(points) < 2: + points = _edge_path(objects.get(source), objects.get(target)) + if len(points) < 2: + continue + dx = points[-1][0] - points[0][0] + dy = points[-1][1] - points[0][1] + dash_style = str(item.get("dash_style") or item.get("line_pattern") or "solid") + line_pattern = _coerce_choice(dash_style, VALID_LINE_PATTERNS, "solid") + render_style = _coerce_choice(item.get("render_style"), VALID_RENDER_STYLES, "") + route_intent = _coerce_choice(item.get("route_intent"), VALID_ROUTE_INTENTS, "") + visual_weight = _coerce_choice(item.get("visual_weight"), VALID_VISUAL_WEIGHTS, "") + preferred_axis = _coerce_choice(item.get("preferred_axis"), VALID_PREFERRED_AXES, "") + explicit_route_policy = str(item.get("route_policy") or "").strip() + route_policy = explicit_route_policy or ("preserve_reference_path" if has_reference_path else "bbox_route_from_hint") + route_style = str(item.get("route_style") or ("dashed_loop" if str(item.get("control_kind") or "").lower() == "dashed_loop" else "")) + if line_pattern in {"dash", "dashed"} and len(points) >= 3 and not route_style: + route_style = "dashed_spline_like" + record = { + "id": str(item.get("id") or f"arrow_{idx:02d}"), + "source": source, + "target": target, + "source_id": source, + "target_id": target, + "target_ids": [str(target_id) for target_id in item.get("target_ids", []) if str(target_id) in slot_ids] if isinstance(item.get("target_ids"), list) else [], + "control_kind": str(item.get("control_kind") or "straight_arrow"), + "path_percent": points, + "arrowhead_direction": float(item.get("arrowhead_direction") if item.get("arrowhead_direction") is not None else math.degrees(math.atan2(dy, dx))), + "stroke_color": str(item.get("stroke_color") or (palette[2] if len(palette) > 2 else "#4A6FA5")), + "stroke_width_pt": float(item.get("stroke_width_pt") or 1.5), + "dash_style": dash_style, + "line_pattern": "dash" if line_pattern in {"dash", "dashed"} else "solid", + "route_style": route_style, + "render_style": render_style, + "route_intent": route_intent, + "visual_weight": visual_weight, + "preferred_axis": preferred_axis, + "bend_side": str(item.get("bend_side") or ""), + "fill_color": str(item.get("fill_color") or ""), + "outline_color": str(item.get("outline_color") or ""), + "outline_width_pt": item.get("outline_width_pt"), + "block_arrow_thickness_percent": item.get("block_arrow_thickness_percent"), + "line_cap": str(item.get("line_cap") or ""), + "arrowhead_size": str(item.get("arrowhead_size") or ""), + "binding_source": str(item.get("binding_source") or ("heuristic" if str(item.get("detected_by") or "").startswith(("cv_", "sequence_")) else "vlm")), + "route_policy": route_policy, + "confidence": float(item.get("confidence") or 0.8), + "detected_by": str(item.get("detected_by") or "vlm"), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + } + normalized.append({key: value for key, value in record.items() if value not in ("", [], None)}) + return normalized + + +def _disagreement_brief(item: dict | None) -> dict | None: + if not isinstance(item, dict): + return None + return { + key: item.get(key) + for key in ( + "id", + "source_id", + "target_id", + "control_kind", + "render_style", + "route_intent", + "visual_weight", + "path_percent", + "stroke_color", + "stroke_width_pt", + "dash_style", + "line_pattern", + "confidence", + "detected_by", + ) + if key in item + } + + +def _control_disagreements(heuristic: list[dict], vlm: list[dict]) -> dict: + matched_heuristic: set[int] = set() + matched_vlm: set[int] = set() + disagreements: list[dict] = [] + duplicates: list[dict] = [] + for v_idx, vlm_item in enumerate(vlm): + best_idx = None + best_score = -1.0 + for h_idx, heuristic_item in enumerate(heuristic): + if h_idx in matched_heuristic: + continue + if not _controls_match_for_disagreement(heuristic_item, vlm_item): + continue + score = _bbox_iou(_path_bbox(heuristic_item.get("path_percent") or []), _path_bbox(vlm_item.get("path_percent") or [])) + if _control_signature(heuristic_item) == _control_signature(vlm_item): + score += 1.0 + score -= _path_distance(heuristic_item.get("path_percent") or [], vlm_item.get("path_percent") or []) + if score > best_score: + best_idx = h_idx + best_score = score + if best_idx is None: + continue + heuristic_item = heuristic[best_idx] + matched_heuristic.add(best_idx) + matched_vlm.add(v_idx) + same_style = ( + str(heuristic_item.get("render_style") or "") == str(vlm_item.get("render_style") or "") + or not heuristic_item.get("render_style") + or not vlm_item.get("render_style") + ) + same_route = ( + str(heuristic_item.get("route_intent") or "") == str(vlm_item.get("route_intent") or "") + or not heuristic_item.get("route_intent") + or not vlm_item.get("route_intent") + ) + if _controls_are_duplicate(heuristic_item, vlm_item) and same_style and same_route: + duplicates.append({ + "id": f"duplicate_{len(duplicates)+1:02d}", + "kind": "duplicate", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + else: + disagreements.append({ + "id": f"control_mismatch_{len(disagreements)+1:02d}", + "kind": "mismatch", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + for idx, item in enumerate(heuristic): + if idx not in matched_heuristic: + disagreements.append({ + "id": f"heuristic_only_{len(disagreements)+1:02d}", + "kind": "heuristic_only", + "heuristic": _disagreement_brief(item), + "vlm": None, + "default_decision": "keep_heuristic", + }) + for idx, item in enumerate(vlm): + if idx not in matched_vlm: + disagreements.append({ + "id": f"vlm_only_{len(disagreements)+1:02d}", + "kind": "vlm_only", + "heuristic": None, + "vlm": _disagreement_brief(item), + "default_decision": "keep_vlm", + }) + return { + "summary": "Control disagreements between heuristic/CV and VLM arrow detection.", + "heuristic_count": len(heuristic), + "vlm_count": len(vlm), + "duplicate_count": len(duplicates), + "disagreement_count": len(disagreements), + "duplicates": duplicates, + "disagreements": disagreements, + } + + +def _call_adjudicator(adapter: Callable, reference: Path, bindable: list[dict], disagreements: dict, flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, disagreements, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, disagreements) + + +def _decision_by_id(raw: dict | list | None) -> dict[str, dict]: + decisions = raw.get("decisions", []) if isinstance(raw, dict) else raw + result = {} + for item in decisions or []: + if not isinstance(item, dict): + continue + disagreement_id = str(item.get("disagreement_id") or item.get("id") or "") + if disagreement_id: + result[disagreement_id] = item + return result + + +def _append_unique(arrows: list[dict], candidate: dict) -> bool: + if any(_controls_are_duplicate(existing, candidate) for existing in arrows): + return False + candidate_id = str(candidate.get("id") or f"arrow_{len(arrows)+1:02d}") + used_ids = {str(item.get("id") or "") for item in arrows} + if candidate_id in used_ids: + stem = candidate_id + suffix = 2 + while f"{stem}_{suffix}" in used_ids: + suffix += 1 + candidate = dict(candidate) + candidate["id"] = f"{stem}_{suffix}" + arrows.append(candidate) + return True + + +def _merge_with_adjudication( + heuristic: list[dict], + vlm: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + adjudicator: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict, dict]: + disagreements = _control_disagreements(heuristic, vlm) + write_json(out / "reference_control_disagreements.json", disagreements) + if mode == "off": + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(vlm), + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return vlm, disagreements, adjudication_doc + raw_adjudication: dict | list | None = None + decisions: dict[str, dict] = {} + status = "skipped" + if mode == "vlm" and adjudicator and disagreements["disagreement_count"]: + try: + raw_adjudication = _call_adjudicator(adjudicator, reference, bindable, disagreements, flow_graph) + decisions = _decision_by_id(raw_adjudication) + status = "used" + except Exception as exc: + status = "fallback" + warnings.append(f"control_adjudication_failed:{exc}") + elif mode == "vlm" and disagreements["disagreement_count"]: + status = "unavailable" + merged: list[dict] = [] + for duplicate in disagreements["duplicates"]: + vlm_item = duplicate.get("vlm") + normalized = _normalize_controls([vlm_item], bindable, palette) + if normalized: + _append_unique(merged, normalized[0]) + for item in disagreements["disagreements"]: + decision = decisions.get(str(item.get("id") or ""), {}) + action = str(decision.get("decision") or item.get("default_decision") or "").lower() + selected: list[dict] = [] + if action == "keep_heuristic": + if item.get("heuristic"): + selected = [item["heuristic"]] + elif action == "keep_both": + selected = [candidate for candidate in [item.get("vlm"), item.get("heuristic")] if candidate] + elif action == "drop_both": + selected = [] + elif action == "merge_or_adjust": + selected = [decision.get("merged_control") or decision.get("control") or item.get("vlm") or item.get("heuristic")] + else: + selected = [item.get("vlm") or item.get("heuristic")] + normalized = _normalize_controls(selected, bindable, palette) + if len(normalized) < len([candidate for candidate in selected if candidate]): + warnings.append(f"invalid_control_adjudication_item_dropped:{item.get('id')}") + for candidate in normalized: + _append_unique(merged, candidate) + if not merged: + merged = vlm or heuristic + adjudication_doc = { + "summary": "Control disagreement adjudication report.", + "mode": mode, + "status": status, + "decision_count": len(decisions), + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(merged), + "raw_adjudication": raw_adjudication, + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return merged, disagreements, adjudication_doc + + +def _call_coverage_audit(adapter: Callable, reference: Path, bindable: list[dict], arrows: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, arrows, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, arrows) + + +def _apply_coverage_audit( + arrows: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + coverage_audit_adapter: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict]: + audit_doc: dict = { + "summary": "Control coverage audit report.", + "mode": mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } + if mode != "vlm": + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + if not coverage_audit_adapter: + audit_doc["status"] = "unavailable" + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + try: + raw = _call_coverage_audit(coverage_audit_adapter, reference, bindable, arrows, flow_graph) + raw_items = raw.get("missing_controls") if isinstance(raw, dict) else raw + if raw_items is None and isinstance(raw, dict): + raw_items = raw.get("arrows", []) + normalized = _normalize_controls(raw_items or [], bindable, palette) + appended = 0 + for candidate in normalized: + candidate = dict(candidate) + candidate["detected_by"] = "coverage_audit_vlm" + candidate["binding_source"] = "coverage_audit_vlm" + if _append_unique(arrows, candidate): + appended += 1 + audit_doc.update({ + "status": "used", + "raw_audit": raw, + "missing_control_count": len(raw_items or []), + "valid_missing_control_count": len(normalized), + "appended_control_count": appended, + }) + invalid_count = max(0, len(raw_items or []) - len(normalized)) + if invalid_count: + warnings.append(f"{invalid_count}_invalid_coverage_audit_control(s)_dropped") + except Exception as exc: + audit_doc["status"] = "fallback" + audit_doc["fallback_reason"] = f"control_coverage_audit_failed:{exc}" + warnings.append(str(audit_doc["fallback_reason"])) + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + + +def _draw_overlay(reference_path: Path, out_path: Path, arrows: list[dict]) -> None: + with Image.open(reference_path).convert("RGB") as image: + draw = ImageDraw.Draw(image) + width, height = image.size + for arrow in arrows: + points = arrow.get("path_percent") or [] + if len(points) < 2: + continue + xy = [(float(p[0]) * width, float(p[1]) * height) for p in points] + color = str(arrow.get("stroke_color") or "#D94141") + draw.line(xy, fill=color, width=4) + end = xy[-1] + prev = xy[-2] + angle = math.atan2(end[1] - prev[1], end[0] - prev[0]) + head = 12 + left = (end[0] - head * math.cos(angle - 0.45), end[1] - head * math.sin(angle - 0.45)) + right = (end[0] - head * math.cos(angle + 0.45), end[1] - head * math.sin(angle + 0.45)) + draw.polygon([end, left, right], fill=color) + draw.text((end[0] + 3, end[1] + 3), str(arrow.get("id") or ""), fill=color) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.save(out_path) + + +def _flow_edges(flow_graph: dict | None) -> list[dict]: + if not isinstance(flow_graph, dict): + return [] + return [item for item in flow_graph.get("edges", []) or [] if isinstance(item, dict)] + + +def _flow_graph_stats(flow_graph: dict | None, objects: list[dict]) -> dict: + object_ids = {str(item.get("id") or "") for item in objects if isinstance(item, dict)} + edges = _flow_edges(flow_graph) + invalid = [] + for edge in edges: + source = str(edge.get("source_id") or edge.get("source") or "") + target = str(edge.get("target_id") or edge.get("target") or "") + if source not in object_ids or target not in object_ids: + invalid.append(str(edge.get("id") or f"{source}->{target}")) + return { + "flow_graph_used": bool(edges), + "flow_graph_edge_count": len(edges), + "invalid_flow_edge_count": len(invalid), + "invalid_flow_edge_ids": invalid, + } + + +def _call_control_adapter(adapter: Callable, reference: Path, bindable: list[dict], heuristic: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, heuristic, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, heuristic) + + +def localize_reference_controls( + reference_path: str | Path, + slots: list[dict], + palette: list[str], + out_dir: str | Path, + mode: str = "hybrid", + control_adapter: Callable[[str | Path, list[dict], list[dict]], dict | list[dict]] | None = None, + flow_graph: dict | None = None, + control_adjudication_mode: str = "off", + control_adjudicator: Callable | None = None, + control_coverage_audit_mode: str = "off", + coverage_audit_adapter: Callable | None = None, +) -> dict: + reference = Path(reference_path) + out = Path(out_dir) + warnings = [] + bindable = _bindable_objects(out, slots) + flow_stats = _flow_graph_stats(flow_graph, bindable) + if flow_stats["invalid_flow_edge_count"]: + warnings.append(f"{flow_stats['invalid_flow_edge_count']}_invalid_flow_graph_edge(s)") + cv_heuristic = _cv_line_controls(reference, bindable, palette) + heuristic = cv_heuristic or _sequence_controls(slots, palette) + _draw_overlay(reference, out / "reference_controls_candidates_overlay.png", heuristic) + arrows = heuristic + vlm_status = "not_requested" + normalized: list[dict] = [] + raw: dict | list | None = None + disagreements = { + "summary": "Control disagreement detection skipped.", + "heuristic_count": len(heuristic), + "vlm_count": 0, + "duplicate_count": 0, + "disagreement_count": 0, + "duplicates": [], + "disagreements": [], + } + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": control_adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": 0, + "input_duplicate_count": 0, + "merged_arrow_count": len(arrows), + } + audit_doc = { + "summary": "Control coverage audit skipped.", + "mode": control_coverage_audit_mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } + if mode in {"vlm", "hybrid"}: + if control_adapter: + try: + raw = _call_control_adapter(control_adapter, reference, bindable, heuristic, flow_graph) + raw_arrows = raw.get("arrows", []) if isinstance(raw, dict) else raw + normalized = _normalize_controls(raw_arrows, bindable, palette) + if normalized: + vlm_status = "used" + if mode == "hybrid": + arrows, disagreements, adjudication_doc = _merge_with_adjudication( + heuristic, + normalized, + bindable, + palette, + reference, + out, + flow_graph, + control_adjudication_mode, + control_adjudicator, + warnings, + ) + else: + arrows = normalized + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + invalid_count = max(0, len(raw_arrows or []) - len(normalized)) + if invalid_count: + warnings.append(f"{invalid_count}_invalid_vlm_control_arrow(s)_dropped") + else: + vlm_status = "fallback" + warnings.append("control_adapter_returned_no_valid_arrows") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + except Exception as exc: + vlm_status = "fallback" + warnings.append(f"vlm_control_failed:{exc}") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + else: + vlm_status = "unavailable_fallback_to_heuristic" + if mode == "vlm": + warnings.append("control_mode_vlm_requested_without_adapter") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + else: + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + if mode == "manual": + arrows = [] + vlm_status = "manual_empty" + _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) + if mode in {"vlm", "hybrid"}: + arrows, audit_doc = _apply_coverage_audit( + arrows, + bindable, + palette, + reference, + out, + flow_graph, + control_coverage_audit_mode, + coverage_audit_adapter, + warnings, + ) + _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) + return { + "summary": "Reference control localization report.", + "mode": mode, + "status": "ok", + "vlm_status": vlm_status, + "vlm_model": str(raw.get("_vlm_model")) if mode in {"vlm", "hybrid"} and isinstance(raw, dict) and raw.get("_vlm_model") else None, + "bindable_object_count": len(bindable), + "heuristic_arrow_count": len(heuristic), + "cv_heuristic_arrow_count": len(cv_heuristic), + "vlm_arrow_count": len(normalized), + "merged_arrow_count": len(arrows), + "control_adjudication_mode": control_adjudication_mode, + "control_adjudication_status": adjudication_doc.get("status"), + "control_disagreement_count": disagreements.get("disagreement_count", 0), + "control_duplicate_count": disagreements.get("duplicate_count", 0), + "control_coverage_audit_mode": control_coverage_audit_mode, + "control_coverage_audit_status": audit_doc.get("status"), + "control_coverage_appended_count": audit_doc.get("appended_control_count", 0), + "warnings": warnings, + "arrows": arrows, + **flow_stats, + } diff --git a/rfs/analysis/rebuild/layout.py b/rfs/analysis/rebuild/layout.py new file mode 100644 index 0000000..b4dd4cd --- /dev/null +++ b/rfs/analysis/rebuild/layout.py @@ -0,0 +1,1017 @@ +from __future__ import annotations + +import inspect +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Callable + +from PIL import Image, ImageDraw + +from ...utils import write_json + + +PANELWISE_LAYOUT_MAX_PANELS = 8 +PANELWISE_LAYOUT_MAX_WORKERS = 3 +PANELWISE_MIN_PANEL_AREA = 0.01 + + +def clamp_bbox(bbox: dict) -> dict[str, float]: + x = max(0.0, min(0.995, float(bbox.get("x", 0.0)))) + y = max(0.0, min(0.995, float(bbox.get("y", 0.0)))) + w = max(0.001, min(float(bbox.get("w", 0.001)), 1.0 - x)) + h = max(0.001, min(float(bbox.get("h", 0.001)), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: + ratio = width_px / max(height_px, 1) + width = max(10.0, min(15.6, 7.5 * ratio)) + return round(width, 3), round(width / ratio, 3) + + +def rgb_hex(rgb: tuple[int, int, int]) -> str: + return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" + + +def estimate_background(image: Image.Image) -> str: + rgb = image.convert("RGB") + w, h = rgb.size + samples = [] + edge = max(1, min(w, h) // 12) + for box in [(0, 0, w, edge), (0, h - edge, w, h), (0, 0, edge, h), (w - edge, 0, w, h)]: + crop = rgb.crop(box).resize((1, 1)) + samples.append(crop.getpixel((0, 0))) + avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) + return rgb_hex(avg) + + +def dominant_palette(image: Image.Image, count: int = 6) -> list[str]: + small = image.convert("RGB").resize((96, 96)) + colors = small.quantize(colors=count).convert("RGB").getcolors(96 * 96) or [] + colors = sorted(colors, key=lambda item: item[0], reverse=True) + return [rgb_hex(rgb) for _n, rgb in colors[:count]] + + +def _bbox_iou(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + inter = ix * iy + union = float(a["w"]) * float(a["h"]) + float(b["w"]) * float(b["h"]) - inter + return inter / max(union, 0.000001) + + +def _bbox_center(bbox: dict) -> tuple[float, float]: + return float(bbox["x"]) + float(bbox["w"]) / 2.0, float(bbox["y"]) + float(bbox["h"]) / 2.0 + + +def _bbox_center_inside(inner: dict, outer: dict) -> bool: + cx, cy = _bbox_center(inner) + return ( + float(outer["x"]) <= cx <= float(outer["x"]) + float(outer["w"]) + and float(outer["y"]) <= cy <= float(outer["y"]) + float(outer["h"]) + ) + + +def _global_to_local_bbox(bbox: dict, panel_bbox: dict) -> dict[str, float]: + panel = clamp_bbox(panel_bbox) + item = clamp_bbox(bbox) + return clamp_bbox({ + "x": (float(item["x"]) - float(panel["x"])) / max(float(panel["w"]), 0.000001), + "y": (float(item["y"]) - float(panel["y"])) / max(float(panel["h"]), 0.000001), + "w": float(item["w"]) / max(float(panel["w"]), 0.000001), + "h": float(item["h"]) / max(float(panel["h"]), 0.000001), + }) + + +def _local_to_global_bbox(bbox: dict, panel_bbox: dict) -> dict[str, float]: + panel = clamp_bbox(panel_bbox) + local = clamp_bbox(bbox) + return clamp_bbox({ + "x": float(panel["x"]) + float(local["x"]) * float(panel["w"]), + "y": float(panel["y"]) + float(local["y"]) * float(panel["h"]), + "w": float(local["w"]) * float(panel["w"]), + "h": float(local["h"]) * float(panel["h"]), + }) + + +def _safe_panel_id(value: str) -> str: + cleaned = re.sub(r"[^a-zA-Z0-9_]+", "_", str(value or "panel").strip().lower()).strip("_") + return cleaned or "panel" + + +def _is_same_frame_box(box: dict, frame: dict) -> bool: + fb = frame.get("bbox_percent") if isinstance(frame, dict) else None + if not isinstance(fb, dict): + return False + if _bbox_iou(box, fb) > 0.55: + return True + tolerance = 0.012 + return ( + abs(float(box["x"]) - float(fb["x"])) <= tolerance + and abs(float(box["y"]) - float(fb["y"])) <= tolerance + and abs(float(box["w"]) - float(fb["w"])) <= tolerance * 2 + and abs(float(box["h"]) - float(fb["h"])) <= tolerance * 2 + ) + + +def _hex_from_rgb_array(values) -> str: + try: + import numpy as np + except Exception: + return "#59AFCB" + if values is None or len(values) == 0: + return "#59AFCB" + rgb = np.median(values, axis=0).astype(int) + return rgb_hex((int(rgb[0]), int(rgb[1]), int(rgb[2]))) + + +def _dash_style_from_edge_runs(edge_crop, thickness: int) -> str: + try: + import numpy as np + except Exception: + return "solid" + if edge_crop.size == 0: + return "solid" + h, w = edge_crop.shape[:2] + bands = [ + edge_crop[:thickness, :].max(axis=0), + edge_crop[max(0, h - thickness):, :].max(axis=0), + edge_crop[:, :thickness].max(axis=1), + edge_crop[:, max(0, w - thickness):].max(axis=1), + ] + dashed_votes = 0 + for band in bands: + values = np.asarray(band) > 0 + if values.size < 16: + continue + # Close tiny gaps caused by anti-aliasing, then count meaningful on/off runs. + closed = np.convolve(values.astype(int), np.ones(3, dtype=int), mode="same") > 0 + runs = [] + current = bool(closed[0]) + length = 1 + for value in closed[1:]: + if bool(value) == current: + length += 1 + else: + runs.append((current, length)) + current = bool(value) + length = 1 + runs.append((current, length)) + on_runs = [length for is_on, length in runs if is_on and length >= 3] + off_runs = [length for is_on, length in runs if not is_on and length >= 4] + on_ratio = float(closed.mean()) + if len(on_runs) >= 3 and len(off_runs) >= 2 and 0.10 <= on_ratio <= 0.82: + dashed_votes += 1 + return "dash" if dashed_votes >= 2 else "solid" + + +def _find_frame_components(reference_path: Path, canvas_w: int, canvas_h: int) -> list[dict]: + try: + import cv2 + import numpy as np + except Exception: + return [] + image = cv2.imread(str(reference_path)) + if image is None: + return [] + rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + blur = cv2.GaussianBlur(gray, (3, 3), 0) + edges = cv2.Canny(blur, 35, 110) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) + candidate_edges = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel, iterations=1) + candidate_edges = cv2.dilate(candidate_edges, kernel, iterations=1) + contours, hierarchy = cv2.findContours(candidate_edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + if hierarchy is None: + return [] + image_area = canvas_w * canvas_h + candidates = [] + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + area = w * h + if area < image_area * 0.006 or area > image_area * 0.82: + continue + if w < 36 or h < 28: + continue + aspect = w / max(h, 1) + if aspect < 0.25 or aspect > 5.8: + continue + perimeter = cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, 0.025 * perimeter, True) + if len(approx) < 4 or len(approx) > 18: + continue + + thickness = max(3, min(14, int(min(w, h) * 0.06))) + crop_edges = edges[y:y + h, x:x + w] + crop_rgb = rgb[y:y + h, x:x + w] + if crop_edges.size == 0 or crop_rgb.size == 0: + continue + + ring_mask = np.zeros((h, w), dtype=bool) + ring_mask[:thickness, :] = True + ring_mask[max(0, h - thickness):, :] = True + ring_mask[:, :thickness] = True + ring_mask[:, max(0, w - thickness):] = True + center_mask = ~ring_mask + if not center_mask.any(): + continue + ring_edge_density = float((crop_edges[ring_mask] > 0).mean()) + center_edge_density = float((crop_edges[center_mask] > 0).mean()) + center_pixels = crop_rgb[center_mask] + ring_pixels = crop_rgb[ring_mask] + center_median = np.median(center_pixels, axis=0) + color_distance = np.linalg.norm(ring_pixels.astype(float) - center_median.astype(float), axis=1) + stroke_pixels = ring_pixels[color_distance > 18] + stroke_coverage = float(len(stroke_pixels) / max(len(ring_pixels), 1)) + if ring_edge_density < 0.045 and stroke_coverage < 0.12: + continue + if center_edge_density > max(0.30, ring_edge_density * 3.2): + continue + + bbox = clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}) + candidates.append({ + "id": f"card_{len(candidates)+1:02d}", + "semantic_role": "outer_group_boundary" if area > image_area * 0.16 else "subcard_frame", + "bbox_percent": bbox, + "shape_kind": "rounded_rect" if len(approx) > 4 else "rect", + "fill_color": _hex_from_rgb_array(center_pixels), + "fill_transparency": 1.0, + "stroke_color": _hex_from_rgb_array(stroke_pixels if len(stroke_pixels) else ring_pixels), + "color_status": "sampled_reference_pixels", + "color_source": "cv_frame_ring_pixels", + "stroke_width_pt": 1.5, + "dash_style": _dash_style_from_edge_runs(crop_edges, thickness), + "corner_radius": 0.08 if len(approx) > 4 else 0.0, + "editable_in": "pptx", + "z_index": 12, + "confidence": round(min(0.9, 0.45 + ring_edge_density + stroke_coverage), 3), + "detected_by": "cv_hollow_frame", + }) + + candidates.sort(key=lambda item: (float(item["bbox_percent"]["y"]), float(item["bbox_percent"]["x"]))) + kept: list[dict] = [] + for item in candidates: + if any(_bbox_iou(item["bbox_percent"], other["bbox_percent"]) > 0.72 for other in kept): + continue + item["id"] = f"card_{len(kept)+1:02d}" + kept.append(item) + if len(kept) >= 24: + break + return kept + + +def _find_visual_components(reference_path: Path, canvas_w: int, canvas_h: int, excluded_frames: list[dict] | None = None) -> list[dict]: + try: + import cv2 + except Exception: + return [] + image = cv2.imread(str(reference_path)) + if image is None: + return [] + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + blur = cv2.GaussianBlur(gray, (5, 5), 0) + edges = cv2.Canny(blur, 45, 120) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) + dilated = cv2.dilate(edges, kernel, iterations=2) + contours, _hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + boxes = [] + image_area = canvas_w * canvas_h + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + area = w * h + if area < image_area * 0.008 or area > image_area * 0.35: + continue + if w < 24 or h < 24: + continue + bbox = clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}) + if any(_is_same_frame_box(bbox, frame) for frame in excluded_frames or []): + continue + boxes.append((x, y, w, h)) + boxes.sort(key=lambda item: (item[1], item[0])) + kept: list[tuple[int, int, int, int]] = [] + for box in boxes: + x, y, w, h = box + duplicate = False + for kx, ky, kw, kh in kept: + ix = max(0, min(x + w, kx + kw) - max(x, kx)) + iy = max(0, min(y + h, ky + kh) - max(y, ky)) + if ix * iy / max(w * h, 1) > 0.55: + duplicate = True + break + if not duplicate: + kept.append(box) + if len(kept) >= 16: + break + slots = [] + for idx, (x, y, w, h) in enumerate(kept, start=1): + slots.append({ + "id": f"slot_{idx:02d}", + "asset_id": f"slot_{idx:02d}", + "paper_concept": f"visual element {idx}", + "display_label": "", + "bbox_percent": clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}), + "composition_type": "full_frame_icon", + "show_slot_caption": False, + "z_index": 20 + idx, + "confidence": 0.55, + "detected_by": "cv_contour", + }) + return slots + + +def _fallback_slots(count: int = 6) -> list[dict]: + boxes = [ + (0.08, 0.26, 0.16, 0.22), + (0.31, 0.26, 0.16, 0.22), + (0.54, 0.26, 0.16, 0.22), + (0.77, 0.26, 0.16, 0.22), + (0.25, 0.58, 0.18, 0.22), + (0.57, 0.58, 0.18, 0.22), + ][:count] + return [{ + "id": f"slot_{idx:02d}", + "asset_id": f"slot_{idx:02d}", + "paper_concept": f"visual element {idx}", + "display_label": "", + "bbox_percent": clamp_bbox({"x": x, "y": y, "w": w, "h": h}), + "composition_type": "full_frame_icon", + "show_slot_caption": False, + "z_index": 20 + idx, + "confidence": 0.25, + "detected_by": "fallback_grid", + } for idx, (x, y, w, h) in enumerate(boxes, start=1)] + + +def _normalize_items(items: list, prefix: str, canvas_id: str = "reference_canvas") -> list[dict]: + normalized = [] + for idx, item in enumerate(items or [], start=1): + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + item_id = str(item.get("id") or f"{prefix}_{idx:02d}") + record = dict(item) + record["id"] = item_id + if prefix == "slot": + record.setdefault("asset_id", item_id) + record.setdefault("paper_concept", item.get("prompt_subject") or item.get("label") or f"visual element {idx}") + record.setdefault("composition_type", "full_frame_icon") + record.setdefault("show_slot_caption", False) + record.setdefault("z_index", 20 + idx) + record.setdefault("panel_id", str(item.get("panel_id") or canvas_id)) + else: + record.setdefault("title", item.get("label") or item_id) + record.setdefault("editable_in", "pptx") + if prefix == "card": + record.setdefault("semantic_role", item.get("semantic_role") or "subcard_frame") + record.setdefault("shape_kind", item.get("shape_kind") or "rounded_rect") + stroke_color = str(item.get("stroke_color") or "").strip() + if stroke_color: + record["stroke_color"] = stroke_color + if str(item.get("detected_by") or "") == "cv_hollow_frame": + record.setdefault("color_status", "sampled_reference_pixels") + record.setdefault("color_source", "cv_frame_ring_pixels") + else: + record.setdefault("color_status", "vlm_reference_estimated") + record.setdefault("color_source", "vlm") + else: + record["stroke_color"] = "#59AFCB" + record.setdefault("color_status", "fallback_missing_vlm_color") + record.setdefault("color_source", "local_default") + record.setdefault("stroke_width_pt", item.get("stroke_width_pt") or 1.5) + dash_style = str(item.get("dash_style") or "solid").lower() + record["dash_style"] = "dash" if dash_style == "dashed" else dash_style + record.setdefault("fill_color", item.get("fill_color") or "#FFFFFF") + record.setdefault("fill_transparency", item.get("fill_transparency", 1.0)) + record.setdefault("corner_radius", item.get("corner_radius", 0.08)) + record.setdefault("z_index", item.get("z_index", 12)) + raw_bbox = dict(record["bbox_percent"]) + record["bbox_percent"] = clamp_bbox(record["bbox_percent"]) + try: + raw_normalized = {key: round(float(raw_bbox[key]), 4) for key in ("x", "y", "w", "h")} + except Exception: + raw_normalized = {} + record["bbox_was_clamped"] = record["bbox_percent"] != raw_normalized + record.setdefault("confidence", 0.8) + record.setdefault("detected_by", "vlm") + normalized.append(record) + return normalized + + +def _layout_item_brief(item: dict | None) -> dict | None: + if not isinstance(item, dict): + return None + return { + key: item.get(key) + for key in ( + "id", + "asset_id", + "title", + "paper_concept", + "prompt_subject", + "semantic_role", + "shape_kind", + "bbox_percent", + "panel_id", + "confidence", + "detected_by", + ) + if key in item + } + + +def _layout_disagreements(base: dict, vlm: dict) -> dict: + disagreements: list[dict] = [] + duplicates: list[dict] = [] + for collection in ("panels", "cards", "slots", "legend_regions"): + heuristic_items = [item for item in base.get(collection, []) or [] if isinstance(item, dict)] + vlm_items = [item for item in vlm.get(collection, []) or [] if isinstance(item, dict)] + matched_h: set[int] = set() + matched_v: set[int] = set() + for v_idx, vlm_item in enumerate(vlm_items): + vbox = vlm_item.get("bbox_percent") if isinstance(vlm_item.get("bbox_percent"), dict) else None + best_idx = None + best_iou = 0.0 + for h_idx, heuristic_item in enumerate(heuristic_items): + if h_idx in matched_h: + continue + hbox = heuristic_item.get("bbox_percent") if isinstance(heuristic_item.get("bbox_percent"), dict) else None + if not hbox or not vbox: + continue + iou = _bbox_iou(hbox, vbox) + if iou > best_iou: + best_idx = h_idx + best_iou = iou + if best_idx is None or best_iou < 0.25: + continue + matched_h.add(best_idx) + matched_v.add(v_idx) + kind = "duplicate" if best_iou >= 0.72 else "bbox_mismatch" + record = { + "id": f"{collection}_{kind}_{len(disagreements)+len(duplicates)+1:02d}", + "collection": collection, + "kind": kind, + "heuristic": _layout_item_brief(heuristic_items[best_idx]), + "vlm": _layout_item_brief(vlm_item), + "iou": round(best_iou, 4), + "default_decision": "keep_vlm", + } + if kind == "duplicate": + duplicates.append(record) + else: + disagreements.append(record) + for h_idx, item in enumerate(heuristic_items): + if h_idx not in matched_h: + disagreements.append({ + "id": f"{collection}_heuristic_only_{len(disagreements)+1:02d}", + "collection": collection, + "kind": "heuristic_only", + "heuristic": _layout_item_brief(item), + "vlm": None, + "default_decision": "drop_heuristic", + }) + for v_idx, item in enumerate(vlm_items): + if v_idx not in matched_v: + disagreements.append({ + "id": f"{collection}_vlm_only_{len(disagreements)+1:02d}", + "collection": collection, + "kind": "vlm_only", + "heuristic": None, + "vlm": _layout_item_brief(item), + "default_decision": "keep_vlm", + }) + return { + "summary": "Layout disagreements between heuristic/CV and VLM object detection.", + "disagreement_count": len(disagreements), + "duplicate_count": len(duplicates), + "disagreements": disagreements, + "duplicates": duplicates, + } + + +def _call_layout_adjudicator(adapter: Callable, reference: Path, base: dict, vlm: dict, disagreements: dict): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, base, vlm, disagreements) + except (TypeError, ValueError): + pass + return adapter(reference, disagreements) + + +def _layout_decision_by_id(raw: dict | list | None) -> dict[str, dict]: + decisions = raw.get("decisions", []) if isinstance(raw, dict) else raw + result = {} + for item in decisions or []: + if not isinstance(item, dict): + continue + disagreement_id = str(item.get("disagreement_id") or item.get("id") or "") + if disagreement_id: + result[disagreement_id] = item + return result + + +def _dedupe_layout_items(items: list[dict]) -> list[dict]: + kept: list[dict] = [] + used_ids: set[str] = set() + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + if any(_bbox_iou(item["bbox_percent"], existing["bbox_percent"]) > 0.72 for existing in kept): + continue + record = dict(item) + item_id = str(record.get("id") or f"item_{len(kept)+1:02d}") + if item_id in used_ids: + stem = item_id + suffix = 2 + while f"{stem}_{suffix}" in used_ids: + suffix += 1 + item_id = f"{stem}_{suffix}" + record["id"] = item_id + used_ids.add(item_id) + kept.append(record) + return kept + + +def _merge_vlm_layout( + base: dict, + vlm: dict, + reference_path: str | Path | None = None, + out_dir: str | Path | None = None, + adjudicator: Callable | None = None, + adjudication_mode: str = "off", +) -> dict: + merged = dict(base) + panels = _normalize_items(vlm.get("panels", []), "panel") + cards = _normalize_items(vlm.get("cards", []), "card") + slots = _normalize_items(vlm.get("slots", []), "slot") + legends = _normalize_items(vlm.get("legend_regions", []), "legend") + vlm_normalized = {"panels": panels, "cards": cards, "slots": slots, "legend_regions": legends} + disagreements = _layout_disagreements(base, vlm_normalized) + adjudication_doc = { + "summary": "Layout disagreement adjudication report.", + "mode": adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + } + if out_dir: + out = Path(out_dir) + write_json(out / "reference_layout_disagreements.json", disagreements) + decisions: dict[str, dict] = {} + if adjudication_mode == "vlm" and adjudicator and disagreements["disagreement_count"] and reference_path: + try: + raw_adjudication = _call_layout_adjudicator(adjudicator, Path(reference_path), base, vlm_normalized, disagreements) + decisions = _layout_decision_by_id(raw_adjudication) + adjudication_doc.update({"status": "used", "decision_count": len(decisions), "raw_adjudication": raw_adjudication}) + except Exception as exc: + adjudication_doc.update({"status": "fallback", "fallback_reason": f"layout_adjudication_failed:{exc}"}) + merged.setdefault("warnings", []).append(str(adjudication_doc["fallback_reason"])) + elif adjudication_mode == "vlm" and disagreements["disagreement_count"]: + adjudication_doc["status"] = "unavailable" + if adjudication_mode == "vlm" and decisions: + selected: dict[str, list[dict]] = {"panels": [], "cards": [], "slots": [], "legend_regions": []} + for duplicate in disagreements["duplicates"]: + collection = str(duplicate.get("collection")) + item = duplicate.get("vlm") or duplicate.get("heuristic") + if collection in selected and item: + selected[collection].append(item) + for item in disagreements["disagreements"]: + collection = str(item.get("collection")) + if collection not in selected: + continue + decision = decisions.get(str(item.get("id") or ""), {}) + action = str(decision.get("decision") or item.get("default_decision") or "").lower() + if action == "keep_heuristic": + candidates = [item.get("heuristic")] + elif action == "keep_both": + candidates = [item.get("vlm"), item.get("heuristic")] + elif action == "drop_both": + candidates = [] + elif action == "merge_or_adjust": + candidates = [decision.get("merged_item") or decision.get("item") or item.get("vlm") or item.get("heuristic")] + elif action == "keep_vlm": + candidates = [item.get("vlm")] + else: + candidates = [item.get("vlm") or item.get("heuristic")] + selected[collection].extend([candidate for candidate in candidates if isinstance(candidate, dict)]) + panels = _normalize_items(selected["panels"], "panel") + cards = _normalize_items(selected["cards"], "card") + slots = _normalize_items(selected["slots"], "slot") + legends = _normalize_items(selected["legend_regions"], "legend") + if panels: + merged["panels"] = _dedupe_layout_items(panels) + if slots: + merged["slots"] = _dedupe_layout_items(slots) + if cards: + merged["cards"] = _dedupe_layout_items(cards) + merged["legend_regions"] = _dedupe_layout_items(legends) + merged["confidence"] = float(vlm.get("confidence") or 0.75) + merged["vlm_status"] = "used" + if vlm.get("_vlm_model"): + merged["vlm_model"] = str(vlm.get("_vlm_model")) + merged["layout_adjudication_mode"] = adjudication_mode + merged["layout_adjudication_status"] = adjudication_doc.get("status") + merged["layout_disagreement_count"] = disagreements.get("disagreement_count", 0) + merged["layout_duplicate_count"] = disagreements.get("duplicate_count", 0) + merged.setdefault("warnings", []) + if out_dir: + write_json(Path(out_dir) / "reference_layout_adjudication.json", adjudication_doc) + return merged + + +def _design_layers(design_plan: dict | None) -> list[dict]: + if not isinstance(design_plan, dict): + return [] + if isinstance(design_plan.get("layers"), list): + return [item for item in design_plan["layers"] if isinstance(item, dict)] + layer_plan = design_plan.get("layer_plan") if isinstance(design_plan.get("layer_plan"), dict) else {} + return [item for item in layer_plan.get("layers", []) or [] if isinstance(item, dict)] + + +def _apply_design_seed(layout: dict, design_plan: dict | None) -> dict: + layers = _design_layers(design_plan) + if not layers: + layout["design_seed_status"] = "not_available" + return layout + panels = _normalize_items([item for item in layers if str(item.get("kind") or "") == "panel"], "panel") + cards = _normalize_items([item for item in layers if str(item.get("kind") or "") == "card"], "card") + slots = _normalize_items([item for item in layers if str(item.get("kind") or "") == "visual_slot"], "slot") + legends = _normalize_items([item for item in layers if str(item.get("kind") or "") == "legend"], "legend") + ignored_count = sum(1 for item in layers if str(item.get("kind") or "") in {"background", "text", "connector", "ignore"}) + if panels: + layout["panels"] = panels + if cards: + layout["cards"] = cards + if slots: + layout["slots"] = slots + if legends: + layout["legend_regions"] = legends + layout["design_seed_status"] = "used" if panels or cards or slots or legends else "no_layout_layers" + layout["design_seed_layer_count"] = len(layers) + layout["design_seed_ignored_layer_count"] = ignored_count + layout["design_seed_slot_count"] = len(slots) + layout["design_seed_card_count"] = len(cards) + layout["design_seed_panel_count"] = len(panels) + layout.setdefault("warnings", []) + if ignored_count: + layout["warnings"].append(f"{ignored_count}_non_layout_design_layer(s)_ignored") + return layout + + +def _draw_overlay(reference_path: Path, out_path: Path, layout: dict) -> None: + with Image.open(reference_path).convert("RGB") as image: + draw = ImageDraw.Draw(image) + w, h = image.size + for collection, color, width in [("panels", "#2D6FB7", 4), ("cards", "#8A5CF6", 3), ("slots", "#E17721", 3), ("legend_regions", "#4B9B52", 3)]: + for item in layout.get(collection, []): + box = item.get("bbox_percent") + if not isinstance(box, dict): + continue + x0 = int(float(box["x"]) * w) + y0 = int(float(box["y"]) * h) + x1 = int((float(box["x"]) + float(box["w"])) * w) + y1 = int((float(box["y"]) + float(box["h"])) * h) + draw.rectangle((x0, y0, x1, y1), outline=color, width=width) + draw.text((x0 + 3, y0 + 3), str(item.get("id") or ""), fill=color) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.save(out_path) + + +def _real_layout_panels(layout: dict) -> list[dict]: + panels = [] + for panel in layout.get("panels", []) or []: + if not isinstance(panel, dict) or not isinstance(panel.get("bbox_percent"), dict): + continue + if str(panel.get("id") or "") == "reference_canvas" and str(panel.get("detected_by") or "") == "default_canvas": + continue + bbox = clamp_bbox(panel["bbox_percent"]) + if float(bbox["w"]) * float(bbox["h"]) < PANELWISE_MIN_PANEL_AREA: + continue + record = dict(panel) + record["bbox_percent"] = bbox + panels.append(record) + return panels + + +def _select_panelwise_panels(layout: dict, max_panels: int = PANELWISE_LAYOUT_MAX_PANELS) -> tuple[list[dict], list[dict]]: + panels = _real_layout_panels(layout) + panels.sort(key=lambda item: ( + float(item["bbox_percent"]["y"]), + float(item["bbox_percent"]["x"]), + -float(item["bbox_percent"]["w"]) * float(item["bbox_percent"]["h"]), + )) + if len(panels) <= max_panels: + return panels, [] + ranked = sorted(panels, key=lambda item: float(item["bbox_percent"]["w"]) * float(item["bbox_percent"]["h"]), reverse=True) + selected_ids = {str(item.get("id")) for item in ranked[:max_panels]} + selected = [item for item in panels if str(item.get("id")) in selected_ids] + skipped = [item for item in panels if str(item.get("id")) not in selected_ids] + return selected, skipped + + +def _items_inside_panel(layout: dict, collection: str, panel: dict, *, local: bool = False) -> list[dict]: + panel_bbox = panel.get("bbox_percent") + if not isinstance(panel_bbox, dict): + return [] + result = [] + for item in layout.get(collection, []) or []: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + bbox = clamp_bbox(item["bbox_percent"]) + if str(item.get("panel_id") or "") == str(panel.get("id") or "") or _bbox_center_inside(bbox, panel_bbox): + record = dict(item) + record["panel_id"] = str(panel.get("id") or record.get("panel_id") or "reference_canvas") + record["bbox_percent"] = _global_to_local_bbox(bbox, panel_bbox) if local else bbox + result.append(record) + return result + + +def _panel_seed_layout(layout: dict, panel: dict) -> dict: + return { + "panel": _layout_item_brief(panel), + "cards": _items_inside_panel(layout, "cards", panel, local=True), + "slots": _items_inside_panel(layout, "slots", panel, local=True), + "legend_regions": _items_inside_panel(layout, "legend_regions", panel, local=True), + } + + +def _panel_fallback_layout(layout: dict, panel: dict) -> dict[str, list[dict]]: + return { + "cards": _items_inside_panel(layout, "cards", panel), + "slots": _items_inside_panel(layout, "slots", panel), + "legend_regions": _items_inside_panel(layout, "legend_regions", panel), + } + + +def _panel_crop_path(out: Path, panel: dict, index: int) -> Path: + panel_id = _safe_panel_id(str(panel.get("id") or f"panel_{index:02d}")) + return out / "panel_layout_crops" / f"{index:02d}_{panel_id}.png" + + +def _save_panel_crop(reference: Path, out: Path, panel: dict, index: int) -> Path: + crop_path = _panel_crop_path(out, panel, index) + with Image.open(reference).convert("RGB") as image: + w, h = image.size + bbox = clamp_bbox(panel["bbox_percent"]) + left = max(0, min(w - 1, int(float(bbox["x"]) * w))) + top = max(0, min(h - 1, int(float(bbox["y"]) * h))) + right = max(left + 1, min(w, int((float(bbox["x"]) + float(bbox["w"])) * w))) + bottom = max(top + 1, min(h, int((float(bbox["y"]) + float(bbox["h"])) * h))) + crop_path.parent.mkdir(parents=True, exist_ok=True) + image.crop((left, top, right, bottom)).save(crop_path) + return crop_path + + +def _panelwise_item_id(panel: dict, item: dict, prefix: str, index: int) -> str: + panel_id = _safe_panel_id(str(panel.get("id") or "panel")) + raw_id = _safe_panel_id(str(item.get("id") or f"{prefix}_{index:02d}")) + if raw_id.startswith(f"{panel_id}_"): + return raw_id + return f"{panel_id}_{raw_id}" + + +def _normalize_panelwise_items(items: list, prefix: str, panel: dict) -> list[dict]: + normalized = [] + panel_id = str(panel.get("id") or "reference_canvas") + for idx, item in enumerate(items or [], start=1): + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + record = dict(item) + original_id = str(record.get("id") or f"{prefix}_{idx:02d}") + record["id"] = _panelwise_item_id(panel, record, prefix, idx) + record["bbox_percent"] = _local_to_global_bbox(record["bbox_percent"], panel["bbox_percent"]) + record["panel_id"] = panel_id + record["detected_by"] = str(record.get("detected_by") or "panel_layout_vlm") + if prefix == "slot": + if not record.get("asset_id") or str(record.get("asset_id")) == original_id: + record["asset_id"] = record["id"] + record.setdefault("paper_concept", record.get("prompt_subject") or record.get("label") or record["id"]) + record.setdefault("composition_type", "full_frame_icon") + record.setdefault("show_slot_caption", False) + normalized.append(record) + return _normalize_items(normalized, prefix) + + +def _panel_count_record(panel: dict, status: str, crop_path: Path | None, objects: dict[str, list[dict]], fallback_reason: str | None = None) -> dict: + record = { + "id": panel.get("id"), + "title": panel.get("title"), + "bbox_percent": panel.get("bbox_percent"), + "crop_path": str(crop_path) if crop_path else None, + "status": status, + "card_count": len(objects.get("cards", [])), + "slot_count": len(objects.get("slots", [])), + "legend_region_count": len(objects.get("legend_regions", [])), + } + if fallback_reason: + record["fallback_reason"] = fallback_reason + return record + + +def _write_panel_layout_artifacts(reference: Path, out: Path, layout: dict, plan_doc: dict | None = None) -> None: + doc = plan_doc or { + "summary": "Panel-wise layout planning skipped.", + "status": "skipped", + "panel_count": 0, + "processed_panel_count": 0, + "fallback_panel_count": 0, + "panels": [], + } + write_json(out / "reference_panel_layout_plan.json", doc) + _draw_overlay(reference, out / "reference_panel_layout_overlay.png", layout) + + +def _outside_panelwise_panels(layout: dict, panels: list[dict], collection: str) -> list[dict]: + result = [] + panel_boxes = [panel["bbox_percent"] for panel in panels if isinstance(panel.get("bbox_percent"), dict)] + for item in layout.get(collection, []) or []: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + bbox = clamp_bbox(item["bbox_percent"]) + if any(_bbox_center_inside(bbox, panel_box) for panel_box in panel_boxes): + continue + result.append(dict(item)) + return result + + +def _run_panelwise_layout( + reference: Path, + out: Path, + layout: dict, + panel_layout_adapter: Callable[[str | Path, dict, dict], dict] | None, + *, + max_panels: int = PANELWISE_LAYOUT_MAX_PANELS, + max_workers: int = PANELWISE_LAYOUT_MAX_WORKERS, +) -> dict | None: + selected, skipped = _select_panelwise_panels(layout, max_panels=max_panels) + if not selected and not skipped: + _write_panel_layout_artifacts(reference, out, layout) + return None + + all_panels = selected + skipped + panel_entries: list[dict] = [] + panel_objects: dict[str, list[dict]] = {"cards": [], "slots": [], "legend_regions": []} + + def process_panel(index_panel: tuple[int, dict]) -> tuple[int, dict, dict[str, list[dict]], str, str | None, Path | None, str | None]: + index, panel = index_panel + crop_path = _save_panel_crop(reference, out, panel, index) + if not panel_layout_adapter: + fallback = _panel_fallback_layout(layout, panel) + return index, panel, fallback, "fallback", "panel_layout_adapter_unavailable", crop_path, None + try: + seed = _panel_seed_layout(layout, panel) + raw = panel_layout_adapter(crop_path, panel, seed) + objects = { + "cards": _normalize_panelwise_items(raw.get("cards", []) if isinstance(raw, dict) else [], "card", panel), + "slots": _normalize_panelwise_items(raw.get("slots", []) if isinstance(raw, dict) else [], "slot", panel), + "legend_regions": _normalize_panelwise_items(raw.get("legend_regions", []) if isinstance(raw, dict) else [], "legend", panel), + } + model = str(raw.get("_vlm_model")) if isinstance(raw, dict) and raw.get("_vlm_model") else None + return index, panel, objects, "used", None, crop_path, model + except Exception as exc: + fallback = _panel_fallback_layout(layout, panel) + return index, panel, fallback, "fallback", f"panel_layout_failed:{exc}", crop_path, None + + results = [] + worker_count = max(1, min(max_workers, len(selected))) + if selected: + with ThreadPoolExecutor(max_workers=worker_count) as executor: + futures = [executor.submit(process_panel, item) for item in enumerate(selected, start=1)] + for future in as_completed(futures): + results.append(future.result()) + results.sort(key=lambda item: item[0]) + + models = [] + for _index, panel, objects, status, fallback_reason, crop_path, model in results: + for collection in ("cards", "slots", "legend_regions"): + panel_objects[collection].extend(objects.get(collection, [])) + panel_entries.append(_panel_count_record(panel, status, crop_path, objects, fallback_reason)) + if model and model not in models: + models.append(model) + + for panel in skipped: + fallback = _panel_fallback_layout(layout, panel) + for collection in ("cards", "slots", "legend_regions"): + panel_objects[collection].extend(fallback.get(collection, [])) + panel_entries.append(_panel_count_record(panel, "fallback", None, fallback, "panel_limit_exceeded")) + + merged = dict(layout) + merged["panels"] = layout.get("panels", []) + for collection in ("cards", "slots", "legend_regions"): + merged[collection] = _dedupe_layout_items(_outside_panelwise_panels(layout, all_panels, collection) + panel_objects[collection]) + merged["panel_layout_status"] = "used" if panel_layout_adapter else "fallback" + merged["panel_layout_processed_count"] = len(selected) + merged["panel_layout_fallback_count"] = sum(1 for item in panel_entries if item.get("status") == "fallback") + merged["layout_detection_strategy"] = "panelwise" + if models: + merged["vlm_model"] = ",".join(models) + plan_doc = { + "summary": "Panel-wise layout planning report.", + "status": merged["panel_layout_status"], + "max_panels": max_panels, + "max_workers": max_workers, + "panel_count": len(all_panels), + "processed_panel_count": len(selected), + "fallback_panel_count": merged["panel_layout_fallback_count"], + "panels": panel_entries, + } + _write_panel_layout_artifacts(reference, out, merged, plan_doc) + return merged + + +def plan_reference_layout( + reference_path: str | Path, + out_dir: str | Path, + mode: str = "hybrid", + vlm_adapter: Callable[[str | Path, dict], dict] | None = None, + panel_layout_adapter: Callable[[str | Path, dict, dict], dict] | None = None, + design_plan: dict | None = None, + layout_adjudication_mode: str = "off", + layout_adjudicator: Callable | None = None, +) -> dict: + reference = Path(reference_path) + out = Path(out_dir) + with Image.open(reference).convert("RGB") as image: + canvas_w, canvas_h = image.size + width_in, height_in = canvas_inches(canvas_w, canvas_h) + background = estimate_background(image) + palette = dominant_palette(image) + cards = _find_frame_components(reference, canvas_w, canvas_h) + slots = _find_visual_components(reference, canvas_w, canvas_h, excluded_frames=cards) or _fallback_slots() + panel = { + "id": "reference_canvas", + "title": "Editable Rebuild", + "bbox_percent": clamp_bbox({"x": 0.025, "y": 0.065, "w": 0.95, "h": 0.84}), + "editable_in": "pptx", + "confidence": 0.5, + "detected_by": "default_canvas", + } + for slot in slots: + slot["panel_id"] = panel["id"] + layout = { + "summary": "Reference geometry inferred by editable rebuild layout planner.", + "layout_mode": mode, + "status": "pass", + "canvas": {"width_px": canvas_w, "height_px": canvas_h, "width_in": width_in, "height_in": height_in, "background": background}, + "panels": [panel], + "cards": cards, + "slots": slots, + "legend_regions": [], + "palette": palette, + "confidence": 0.55 if slots else 0.25, + "vlm_status": "not_requested", + "warnings": [], + } + layout = _apply_design_seed(layout, design_plan) + write_json(out / "reference_layout_disagreements.json", { + "summary": "Layout disagreement detection skipped.", + "disagreement_count": 0, + "duplicate_count": 0, + "disagreements": [], + "duplicates": [], + }) + write_json(out / "reference_layout_adjudication.json", { + "summary": "Layout disagreement adjudication skipped.", + "mode": layout_adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": 0, + "input_duplicate_count": 0, + }) + _write_panel_layout_artifacts(reference, out, layout) + if mode in {"vlm", "hybrid"}: + panelwise_layout = _run_panelwise_layout(reference, out, layout, panel_layout_adapter) if panel_layout_adapter else None + if panelwise_layout is not None: + layout = panelwise_layout + layout["confidence"] = max(float(layout.get("confidence") or 0.0), 0.75) + layout["vlm_status"] = "used" + elif vlm_adapter: + try: + vlm_layout = vlm_adapter(reference, layout) + if isinstance(vlm_layout, dict): + layout = _merge_vlm_layout( + layout, + vlm_layout, + reference_path=reference, + out_dir=out, + adjudicator=layout_adjudicator, + adjudication_mode=layout_adjudication_mode, + ) + except Exception as exc: + layout["warnings"].append(f"vlm_layout_failed:{exc}") + layout["vlm_status"] = "fallback" + else: + layout["vlm_status"] = "unavailable_fallback_to_heuristic" + if mode == "vlm": + layout["warnings"].append("layout_mode_vlm_requested_without_adapter") + layout.setdefault("layout_adjudication_mode", layout_adjudication_mode) + layout.setdefault("layout_adjudication_status", "skipped") + layout.setdefault("layout_disagreement_count", 0) + layout.setdefault("layout_duplicate_count", 0) + _draw_overlay(reference, out / "reference_geometry_overlay.png", layout) + return layout diff --git a/rfs/analysis/rebuild/reference_text.py b/rfs/analysis/rebuild/reference_text.py new file mode 100644 index 0000000..990a122 --- /dev/null +++ b/rfs/analysis/rebuild/reference_text.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +import math +import os +import threading +from pathlib import Path +from typing import Any, Callable + +from PIL import Image + + +OCR_LOW_CONFIDENCE = 0.65 +_EASYOCR_READERS: dict[tuple[str, ...], Any] = {} +_RAPIDOCR_LOCAL = threading.local() + + +def _positive_ocr_setting(explicit: int | None, env_name: str, default: int) -> int: + try: + return max(1, int(explicit if explicit is not None else os.getenv(env_name) or default)) + except (TypeError, ValueError): + return max(1, int(default)) + + +def _rapidocr_detector_limit(explicit: int | None = None) -> int: + """Return a bounded detector resize limit suitable for RapidOCR.""" + value = _positive_ocr_setting(explicit, "RFS_RAPIDOCR_DET_LIMIT", 512) + return max(256, min(2048, value)) + + +def _round4(value: float) -> float: + return round(float(value), 4) + + +def _clamp_bbox(bbox: dict[str, float]) -> dict[str, float]: + x = max(0.0, min(0.995, float(bbox["x"]))) + y = max(0.0, min(0.995, float(bbox["y"]))) + w = max(0.001, min(float(bbox["w"]), 1.0 - x)) + h = max(0.001, min(float(bbox["h"]), 1.0 - y)) + return {"x": _round4(x), "y": _round4(y), "w": _round4(w), "h": _round4(h)} + + +def _bbox_center(bbox: dict[str, float]) -> dict[str, float]: + return {"x": _round4(float(bbox["x"]) + float(bbox["w"]) / 2), "y": _round4(float(bbox["y"]) + float(bbox["h"]) / 2)} + + +def _luminance(rgb: tuple[int, int, int]) -> float: + return 0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2] + + +def _hex(rgb: tuple[int, int, int]) -> str: + return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" + + +def _contrast(a: tuple[int, int, int], b: tuple[int, int, int]) -> float: + return abs(_luminance(a) - _luminance(b)) + + +def _sample_text_color(image: Image.Image, bbox: dict[str, float], fallback: str = "#263747") -> tuple[str, str]: + try: + rgb_image = image.convert("RGB") + iw, ih = rgb_image.size + pad_x = max(1, int(float(bbox["w"]) * iw * 0.04)) + pad_y = max(1, int(float(bbox["h"]) * ih * 0.10)) + x0 = max(0, min(iw - 1, int(float(bbox["x"]) * iw) - pad_x)) + y0 = max(0, min(ih - 1, int(float(bbox["y"]) * ih) - pad_y)) + x1 = max(x0 + 1, min(iw, int((float(bbox["x"]) + float(bbox["w"])) * iw) + pad_x)) + y1 = max(y0 + 1, min(ih, int((float(bbox["y"]) + float(bbox["h"])) * ih) + pad_y)) + crop = rgb_image.crop((x0, y0, x1, y1)) + pixels = list(crop.get_flattened_data() if hasattr(crop, "get_flattened_data") else crop.getdata()) + except Exception: + return fallback, "sample_failed" + if not pixels: + return fallback, "empty_crop" + pixels = sorted(pixels, key=_luminance) + dark = pixels[: max(1, len(pixels) // 5)] + light = pixels[-max(1, len(pixels) // 5):] + dark_avg = tuple(int(sum(px[i] for px in dark) / len(dark)) for i in range(3)) + light_avg = tuple(int(sum(px[i] for px in light) / len(light)) for i in range(3)) + if _contrast(dark_avg, light_avg) < 32: + return fallback, "low_contrast_fallback" + text_rgb = dark_avg if _luminance(dark_avg) < _luminance(light_avg) else light_avg + return _hex(text_rgb), "sampled_foreground" + + +def _bbox_overlap(a: dict[str, float], b: dict[str, float]) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + area = ix * iy + return area / max(float(a["w"]) * float(a["h"]), 0.000001) + + +def _nearest_target(bbox: dict[str, float], program: dict) -> tuple[str, str]: + candidates: list[tuple[float, str, str]] = [] + for panel in program.get("panels", []): + if isinstance(panel, dict) and isinstance(panel.get("bbox_percent"), dict): + overlap = _bbox_overlap(bbox, panel["bbox_percent"]) + if overlap > 0: + candidates.append((overlap, "panel_title", str(panel.get("id") or ""))) + for slot in program.get("slots", []): + if isinstance(slot, dict) and isinstance(slot.get("bbox_percent"), dict): + overlap = _bbox_overlap(bbox, slot["bbox_percent"]) + if overlap > 0: + candidates.append((overlap, "slot_caption", str(slot.get("id") or ""))) + if candidates: + _overlap, role, target_id = max(candidates, key=lambda item: item[0]) + return role, target_id + return "free_text", "canvas" + + +def _font_family_guess(text: str) -> str: + if any(ord(char) > 127 for char in text): + return "Microsoft YaHei" + return "Arial" + + +def _normalize_quad(quad: Any) -> list[list[float]] | None: + if hasattr(quad, "tolist"): + quad = quad.tolist() + if not isinstance(quad, (list, tuple)) or len(quad) < 4: + return None + points: list[list[float]] = [] + for point in quad[:4]: + if not isinstance(point, (list, tuple)) or len(point) < 2: + return None + points.append([float(point[0]), float(point[1])]) + return points + + +def _flatten_paddle_result(result: Any) -> list[Any]: + if not isinstance(result, list): + return [] + if result and all(isinstance(item, list) and len(item) >= 2 and _normalize_quad(item[0]) for item in result): + return result + flattened: list[Any] = [] + for page in result: + if isinstance(page, list): + flattened.extend(_flatten_paddle_result(page)) + return flattened + + +def _mapping_value(mapping: dict[str, Any], keys: tuple[str, ...]) -> Any: + for key in keys: + if key in mapping: + return mapping[key] + return None + + +def _parse_paddle_mapping(mapping: dict[str, Any]) -> list[dict[str, Any]]: + payload = mapping.get("res") if isinstance(mapping.get("res"), dict) else mapping + texts = _mapping_value(payload, ("rec_texts", "texts", "text")) + quads = _mapping_value(payload, ("rec_polys", "dt_polys", "det_polys", "polys", "boxes")) + scores = _mapping_value(payload, ("rec_scores", "scores", "confidences")) + if hasattr(texts, "tolist"): + texts = texts.tolist() + if hasattr(quads, "tolist"): + quads = quads.tolist() + if hasattr(scores, "tolist"): + scores = scores.tolist() + if isinstance(texts, str): + texts = [texts] + if not isinstance(texts, list) or not isinstance(quads, list): + return [] + records = [] + score_values = scores if isinstance(scores, list) else [] + for index, text in enumerate(texts): + if index >= len(quads): + break + quad = _normalize_quad(quads[index]) + raw_text = str(text or "").strip() + if not quad or not raw_text: + continue + confidence = 0.0 + if index < len(score_values): + try: + confidence = float(score_values[index]) + except Exception: + confidence = 0.0 + records.append({"text": raw_text, "confidence": confidence, "quad": quad}) + return records + + +def _parse_ocr_result(result: Any) -> list[dict[str, Any]]: + records = [] + if isinstance(result, dict): + return _parse_paddle_mapping(result) + if isinstance(result, list): + for page in result: + value = page + if hasattr(value, "json"): + value = value.json() if callable(value.json) else value.json + if isinstance(value, dict): + records.extend(_parse_paddle_mapping(value)) + if records: + return records + for item in _flatten_paddle_result(result): + if not isinstance(item, (list, tuple)) or len(item) < 2: + continue + quad = _normalize_quad(item[0]) + payload = item[1] + if not quad: + continue + text = "" + confidence = 0.0 + if isinstance(payload, (list, tuple)) and payload: + text = str(payload[0] or "").strip() + if len(payload) > 1: + try: + confidence = float(payload[1]) + except Exception: + confidence = 0.0 + else: + text = str(payload or "").strip() + if text: + records.append({"text": text, "confidence": confidence, "quad": quad}) + return records + + +def run_paddle_ocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: + from paddleocr import PaddleOCR # type: ignore + + lang_map = {"en_ch": "ch", "ch": "ch", "en": "en"} + mapped_lang = lang_map.get(str(lang), "ch") + init_options = [ + {"use_angle_cls": True, "lang": mapped_lang, "show_log": False}, + {"use_angle_cls": True, "lang": mapped_lang}, + {"use_textline_orientation": True, "lang": mapped_lang}, + {"lang": mapped_lang}, + ] + last_error: Exception | None = None + for options in init_options: + try: + ocr = PaddleOCR(**options) + break + except Exception as exc: + last_error = exc + else: + assert last_error is not None + raise last_error + try: + result = ocr.ocr(str(image_path), cls=True) + except Exception: + result = ocr.ocr(str(image_path)) + return _parse_ocr_result(result) + + +def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: + import easyocr # type: ignore + + lang_map = {"en_ch": ["ch_sim", "en"], "ch": ["ch_sim"], "en": ["en"]} + languages = tuple(lang_map.get(str(lang), ["ch_sim", "en"])) + reader = _EASYOCR_READERS.get(languages) + if reader is None: + allow_download = str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"} + try: + reader = easyocr.Reader(list(languages), gpu=False, download_enabled=allow_download, verbose=False) + except Exception as exc: + hint = " Set RFS_OCR_ALLOW_DOWNLOAD=1 for an explicit one-time model download." if not allow_download else "" + raise RuntimeError(f"EasyOCR model is not ready for languages {list(languages)}.{hint} Original error: {exc}") from exc + _EASYOCR_READERS[languages] = reader + records = [] + for quad, text, confidence in reader.readtext(str(image_path)): + points = _normalize_quad(quad) + if points and str(text).strip(): + records.append({"text": str(text).strip(), "confidence": float(confidence), "quad": points}) + return records + + +def run_rapidocr_detailed( + image_path: str | Path, + lang: str, + *, + threads: int | None = None, + batch_size: int | None = None, + detector_limit: int | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + del lang + thread_count = _positive_ocr_setting(threads, "RFS_RAPIDOCR_THREADS", 1) + recognition_batch = _positive_ocr_setting(batch_size, "RFS_RAPIDOCR_BATCH", 6) + detector_side_limit = _rapidocr_detector_limit(detector_limit) + engines = getattr(_RAPIDOCR_LOCAL, "engines", None) + if engines is None: + engines = {} + _RAPIDOCR_LOCAL.engines = engines + engine_key = (thread_count, recognition_batch, detector_side_limit) + if engine_key not in engines: + from rapidocr_onnxruntime import RapidOCR # type: ignore + + engines[engine_key] = RapidOCR( + intra_op_num_threads=thread_count, + inter_op_num_threads=1, + det_limit_side_len=detector_side_limit, + rec_batch_num=recognition_batch, + ) + result, timings = engines[engine_key](str(image_path)) + records = [] + for item in result or []: + if not isinstance(item, (list, tuple)) or len(item) < 3: + continue + quad = _normalize_quad(item[0]) + text = str(item[1] or "").strip() + if quad and text: + records.append({"text": text, "confidence": float(item[2] or 0.0), "quad": quad}) + timing_values = [float(value or 0.0) for value in (timings or [])] + while len(timing_values) < 3: + timing_values.append(0.0) + diagnostics = { + "detector_limit": detector_side_limit, + "thread_count": thread_count, + "recognition_batch": recognition_batch, + "detection_seconds": round(timing_values[0], 4), + "classification_seconds": round(timing_values[1], 4), + "recognition_seconds": round(timing_values[2], 4), + "inference_seconds": round(sum(timing_values[:3]), 4), + "region_count": len(records), + } + return records, diagnostics + + +def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = None, batch_size: int | None = None, detector_limit: int | None = None) -> list[dict[str, Any]]: + records, _diagnostics = run_rapidocr_detailed( + image_path, + lang, + threads=threads, + batch_size=batch_size, + detector_limit=detector_limit, + ) + return records + + +def _record_to_region(record: dict[str, Any], image: Image.Image, program: dict, index: int, canvas_height_in: float, engine: str) -> dict: + width, height = image.size + xs = [point[0] for point in record["quad"]] + ys = [point[1] for point in record["quad"]] + bbox = _clamp_bbox({ + "x": min(xs) / max(width, 1), + "y": min(ys) / max(height, 1), + "w": (max(xs) - min(xs)) / max(width, 1), + "h": (max(ys) - min(ys)) / max(height, 1), + }) + text = str(record.get("text") or "").strip() + confidence = float(record.get("confidence") or 0.0) + color_hex, color_status = _sample_text_color(image, bbox) + role, target_id = _nearest_target(bbox, program) + font_size = max(1.0, float(bbox["h"]) * canvas_height_in * 72 * 0.72) + return { + "id": f"ref_text_ocr_{index:03d}", + "text": text, + "raw_text": text, + "role": role, + "target_id": target_id, + "bbox_percent": bbox, + "line_bbox_percent": bbox, + "word_bbox_percent": [bbox], + "center_percent": _bbox_center(bbox), + "width_percent": _round4(bbox["w"]), + "height_percent": _round4(bbox["h"]), + "estimated_font_ratio": _round4(float(bbox["h"]) * 0.72), + "font_size_pt": round(font_size, 2), + "font_family_guess": _font_family_guess(text), + "font_weight_guess": "bold" if role == "panel_title" and float(bbox["h"]) > 0.025 else "regular", + "color_hex": color_hex, + "color_sampling_status": color_status, + "confidence": round(confidence, 4), + "ocr_engine": engine, + "source": "reference_ocr_text_region", + "editable_in": "pptx", + } + + +def extract_reference_text( + reference_path: str | Path, + program: dict, + *, + mode: str = "ocr", + engine: str = "easyocr", + lang: str = "en_ch", + ocr_adapter: Callable[[str | Path, str], list[dict[str, Any]]] | None = None, +) -> tuple[list[dict], dict]: + requested_mode = str(mode or "ocr").lower() + requested_engine = str(engine or "easyocr").lower() + report = { + "summary": "OCR text extraction quality report.", + "mode": requested_mode, + "ocr_engine": requested_engine, + "ocr_engine_effective": requested_engine, + "ocr_lang": lang, + "status": "not_run", + "text_region_count": 0, + "low_confidence_count": 0, + "warnings": [], + "fallback_reason": None, + } + if requested_mode != "ocr" or requested_engine == "off": + report.update({"status": "fallback", "fallback_reason": "ocr_disabled"}) + return [], report + effective_engine = requested_engine + try: + if ocr_adapter: + raw_records = ocr_adapter(reference_path, lang) + elif requested_engine == "rapidocr": + raw_records = run_rapidocr(reference_path, lang) + elif requested_engine == "easyocr": + raw_records = run_easyocr(reference_path, lang) + else: + try: + raw_records = run_paddle_ocr(reference_path, lang) + except Exception as paddle_exc: + try: + raw_records = run_easyocr(reference_path, lang) + effective_engine = "easyocr" + report["ocr_engine_effective"] = effective_engine + report["warnings"].append(f"paddle_unavailable_used_easyocr:{paddle_exc}") + report["fallback_reason"] = f"paddle_unavailable_used_easyocr:{paddle_exc}" + except Exception as easy_exc: + raise RuntimeError(f"paddle_failed:{paddle_exc}; easyocr_failed:{easy_exc}") from easy_exc + except Exception as exc: + report.update({"status": "fallback", "fallback_reason": f"ocr_unavailable:{exc}"}) + report["warnings"].append(str(exc)) + return [], report + try: + with Image.open(reference_path) as image: + rgb_image = image.convert("RGB") + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + canvas_height_in = float(canvas.get("height_in") or 7.5) + regions = [_record_to_region(record, rgb_image, program, index, canvas_height_in, effective_engine) for index, record in enumerate(raw_records, start=1)] + except Exception as exc: + report.update({"status": "fallback", "fallback_reason": f"ocr_postprocess_failed:{exc}"}) + report["warnings"].append(str(exc)) + return [], report + regions = [region for region in regions if str(region.get("text", "")).strip()] + low_conf = [region for region in regions if float(region.get("confidence") or 0.0) < OCR_LOW_CONFIDENCE] + overlap_warnings = [] + for region in regions: + if region.get("role") == "slot_caption": + overlap_warnings.append(region["id"]) + if not regions: + report.update({"status": "fallback", "fallback_reason": "ocr_returned_no_text", "text_region_count": 0}) + return [], report + report.update({ + "status": "pass", + "ocr_engine_effective": effective_engine, + "text_region_count": len(regions), + "low_confidence_count": len(low_conf), + "low_confidence_text_ids": [region["id"] for region in low_conf], + "slot_overlap_text_ids": overlap_warnings, + "warnings": report["warnings"] + ([f"{len(low_conf)} low-confidence OCR text region(s)"] if low_conf else []), + }) + return regions, report diff --git a/rfs/arrow_router.py b/rfs/arrow_router.py index 99aef2b..9ac35d3 100644 --- a/rfs/arrow_router.py +++ b/rfs/arrow_router.py @@ -1,693 +1,9 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from .utils import write_json - -ROUTER_VERSION = "reference-constrained-orthogonal-v1" -AESTHETIC_ROUTER_VERSION = "reference-tunnel-aesthetic-v2" -DEFAULT_REFERENCE_TUNNEL_PERCENT = 0.024 - - -def _bbox_center(bbox: dict[str, Any]) -> list[float]: - return [ - round(float(bbox["x"]) + float(bbox["w"]) / 2, 4), - round(float(bbox["y"]) + float(bbox["h"]) / 2, 4), - ] - - -def _anchor_point(obj: dict | None, anchor: str) -> list[float] | None: - if not obj or not isinstance(obj.get("bbox_percent"), dict): - return None - bbox = obj["bbox_percent"] - x = float(bbox["x"]) - y = float(bbox["y"]) - w = float(bbox["w"]) - h = float(bbox["h"]) - points = { - "left_mid": [x, y + h / 2], - "right_mid": [x + w, y + h / 2], - "top_mid": [x + w / 2, y], - "bottom_mid": [x + w / 2, y + h], - "center": [x + w / 2, y + h / 2], - } - point = points.get(anchor) or points["center"] - return [round(point[0], 4), round(point[1], 4)] - - -def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: - if not source or not target: - return [] - sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None - tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None - if not sbox or not tbox: - return [] - sx, sy = _bbox_center(sbox) - tx, ty = _bbox_center(tbox) - dx = tx - sx - dy = ty - sy - if abs(dx) >= abs(dy): - s_anchor = "right_mid" if dx >= 0 else "left_mid" - t_anchor = "left_mid" if dx >= 0 else "right_mid" - else: - s_anchor = "bottom_mid" if dy >= 0 else "top_mid" - t_anchor = "top_mid" if dy >= 0 else "bottom_mid" - s = _anchor_point(source, s_anchor) - t = _anchor_point(target, t_anchor) - return [s, t] if s and t else [] - - -def _clean_path(points: list[list[float] | None]) -> list[list[float]]: - cleaned: list[list[float]] = [] - for point in points: - if not point or len(point) < 2: - continue - x = min(1.0, max(0.0, float(point[0]))) - y = min(1.0, max(0.0, float(point[1]))) - rounded = [round(x, 4), round(y, 4)] - if not cleaned or abs(cleaned[-1][0] - rounded[0]) > 0.001 or abs(cleaned[-1][1] - rounded[1]) > 0.001: - cleaned.append(rounded) - if len(cleaned) <= 2: - return cleaned - simplified = [cleaned[0]] - for index in range(1, len(cleaned) - 1): - prev = simplified[-1] - cur = cleaned[index] - nxt = cleaned[index + 1] - same_x = abs(prev[0] - cur[0]) < 0.001 and abs(cur[0] - nxt[0]) < 0.001 - same_y = abs(prev[1] - cur[1]) < 0.001 and abs(cur[1] - nxt[1]) < 0.001 - if not (same_x or same_y): - simplified.append(cur) - simplified.append(cleaned[-1]) - return simplified - - -def _segments(points: list[list[float]]) -> list[tuple[list[float], list[float]]]: - return [(points[i], points[i + 1]) for i in range(max(0, len(points) - 1))] - - -def _path_length(points: list[list[float]]) -> float: - total = 0.0 - for a, b in _segments(points): - total += ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 - return round(total, 4) - - -def _path_normal(points: list[list[float]]) -> list[float]: - if len(points) < 2: - return [0.0, 0.0] - dx = float(points[-1][0]) - float(points[0][0]) - dy = float(points[-1][1]) - float(points[0][1]) - length = (dx * dx + dy * dy) ** 0.5 - if length < 0.0001: - return [0.0, 0.0] - return [-dy / length, dx / length] - - -def _point_distance(a: list[float], b: list[float]) -> float: - return ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 - - -def _point_segment_distance(point: list[float], a: list[float], b: list[float]) -> float: - px, py = float(point[0]), float(point[1]) - ax, ay = float(a[0]), float(a[1]) - bx, by = float(b[0]), float(b[1]) - dx = bx - ax - dy = by - ay - denom = dx * dx + dy * dy - if denom < 0.0000001: - return _point_distance(point, a) - t = max(0.0, min(1.0, ((px - ax) * dx + (py - ay) * dy) / denom)) - proj = [ax + t * dx, ay + t * dy] - return _point_distance(point, proj) - - -def _max_distance_to_polyline(points: list[list[float]], reference: list[list[float]]) -> float: - segments = _segments(reference) - if not segments: - return 0.0 - distances = [] - for point in points: - distances.append(min(_point_segment_distance(point, a, b) for a, b in segments)) - return round(max(distances, default=0.0), 4) - - -def _lane_offset(lane_index: int, lane_count: int, tunnel_percent: float) -> float: - if lane_count <= 1: - return 0.0 - centered = float(lane_index) - (float(lane_count) - 1.0) / 2.0 - step = min(0.006, max(0.002, tunnel_percent / max(lane_count, 1))) - return max(-tunnel_percent * 0.82, min(tunnel_percent * 0.82, centered * step)) - - -def _shift_point(point: list[float], normal: list[float], offset: float) -> list[float]: - return [ - round(min(1.0, max(0.0, float(point[0]) + normal[0] * offset)), 4), - round(min(1.0, max(0.0, float(point[1]) + normal[1] * offset)), 4), - ] - - -def _aesthetic_tunnel_path( - points: list[list[float]], - lane_index: int, - lane_count: int, - role: str, - tunnel_percent: float = DEFAULT_REFERENCE_TUNNEL_PERCENT, -) -> tuple[list[list[float]], dict[str, Any]]: - original = _clean_path(points) - if len(original) < 2: - return original, { - "routing_algorithm": AESTHETIC_ROUTER_VERSION, - "route_generation_status": "aesthetic_no_path", - "reference_tunnel_percent": tunnel_percent, - "reference_path_delta_max": 0.0, - "reference_tunnel_preserved": True, - } - normal = _path_normal(original) - offset = _lane_offset(lane_index, lane_count, tunnel_percent) - adjusted = [list(point) for point in original] - if abs(offset) > 0.0001 and len(original) == 2: - mid = [ - (float(original[0][0]) + float(original[1][0])) / 2.0, - (float(original[0][1]) + float(original[1][1])) / 2.0, - ] - adjusted = [original[0], _shift_point(mid, normal, offset), original[-1]] - elif role in {"main_flow", "module_flow"}: - # Single-lane arrows keep the exact reference geometry; the PPT renderer - # still softens them through curve connectors, halo, and line caps. - adjusted = original - adjusted = _clean_path(adjusted) - delta = _max_distance_to_polyline(adjusted, original) - return adjusted, { - "routing_algorithm": AESTHETIC_ROUTER_VERSION, - "route_generation_status": "aesthetic_tunnel_adjusted" if adjusted != original else "aesthetic_style_only", - "reference_tunnel_percent": tunnel_percent, - "reference_path_delta_max": delta, - "reference_tunnel_preserved": delta <= tunnel_percent, - "lane_offset_percent": round(offset, 4), - } - - -def _orientation(a: list[float], b: list[float], c: list[float]) -> float: - return (float(b[1]) - float(a[1])) * (float(c[0]) - float(b[0])) - (float(b[0]) - float(a[0])) * (float(c[1]) - float(b[1])) - - -def _share_endpoint(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: - pairs = ((a1, b1), (a1, b2), (a2, b1), (a2, b2)) - return any(abs(p[0] - q[0]) < 0.002 and abs(p[1] - q[1]) < 0.002 for p, q in pairs) +"""Compatibility alias for :mod:`rfs.composition.rebuild.arrows`.""" +from __future__ import annotations -def _segments_cross(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: - if _share_endpoint(a1, a2, b1, b2): - return False - # Fast reject by segment bounding boxes. - if max(min(a1[0], a2[0]), min(b1[0], b2[0])) > min(max(a1[0], a2[0]), max(b1[0], b2[0])): - return False - if max(min(a1[1], a2[1]), min(b1[1], b2[1])) > min(max(a1[1], a2[1]), max(b1[1], b2[1])): - return False - o1 = _orientation(a1, a2, b1) - o2 = _orientation(a1, a2, b2) - o3 = _orientation(b1, b2, a1) - o4 = _orientation(b1, b2, a2) - return o1 * o2 < 0 and o3 * o4 < 0 - - -def _point_inside_bbox(point: list[float], bbox: dict[str, Any], pad: float = 0.0) -> bool: - x = float(bbox["x"]) - pad - y = float(bbox["y"]) - pad - w = float(bbox["w"]) + pad * 2 - h = float(bbox["h"]) + pad * 2 - return x <= float(point[0]) <= x + w and y <= float(point[1]) <= y + h - - -def _segment_bbox_overlap(a: list[float], b: list[float], bbox: dict[str, Any], pad: float = 0.004) -> bool: - if _point_inside_bbox(a, bbox, pad=pad) or _point_inside_bbox(b, bbox, pad=pad): - return True - x0 = float(bbox["x"]) - pad - y0 = float(bbox["y"]) - pad - x1 = float(bbox["x"]) + float(bbox["w"]) + pad - y1 = float(bbox["y"]) + float(bbox["h"]) + pad - edges = [ - ([x0, y0], [x1, y0]), - ([x1, y0], [x1, y1]), - ([x1, y1], [x0, y1]), - ([x0, y1], [x0, y0]), - ] - return any(_segments_cross(a, b, e0, e1) for e0, e1 in edges) - - -def _candidate_lanes(source: dict, target: dict, obstacles: dict[str, dict], axis: str, limit: int = 18) -> list[float]: - sbox = source["bbox_percent"] - tbox = target["bbox_percent"] - if axis == "x": - base = [ - float(sbox["x"]), - float(sbox["x"]) + float(sbox["w"]), - float(sbox["x"]) + float(sbox["w"]) / 2, - float(tbox["x"]), - float(tbox["x"]) + float(tbox["w"]), - float(tbox["x"]) + float(tbox["w"]) / 2, - ] - for obj in obstacles.values(): - bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None - if not bbox: - continue - base.extend([float(bbox["x"]) - 0.014, float(bbox["x"]) + float(bbox["w"]) + 0.014]) - else: - base = [ - float(sbox["y"]), - float(sbox["y"]) + float(sbox["h"]), - float(sbox["y"]) + float(sbox["h"]) / 2, - float(tbox["y"]), - float(tbox["y"]) + float(tbox["h"]), - float(tbox["y"]) + float(tbox["h"]) / 2, - ] - for obj in obstacles.values(): - bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None - if not bbox: - continue - base.extend([float(bbox["y"]) - 0.014, float(bbox["y"]) + float(bbox["h"]) + 0.014]) - midpoint = (sum(base[:6]) / 6) if len(base) >= 6 else 0.5 - lanes = sorted({round(min(0.985, max(0.015, value)), 4) for value in base}, key=lambda value: abs(value - midpoint)) - return sorted(lanes[:limit]) - - -def _crossing_count(points: list[list[float]], existing_segments: list[tuple[list[float], list[float]]]) -> int: - count = 0 - for a, b in _segments(points): - for c, d in existing_segments: - if _segments_cross(a, b, c, d): - count += 1 - return count - - -def _obstacle_overlaps(points: list[list[float]], obstacles: dict[str, dict]) -> list[str]: - overlaps = [] - for obj_id, obj in obstacles.items(): - bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None - if not bbox: - continue - if any(_segment_bbox_overlap(a, b, bbox) for a, b in _segments(points)): - overlaps.append(obj_id) - return overlaps - - -def _score_candidate(points: list[list[float]], obstacles: dict[str, dict], existing_segments: list[tuple[list[float], list[float]]]) -> tuple[float, dict[str, Any]]: - overlaps = _obstacle_overlaps(points, obstacles) - crossings = _crossing_count(points, existing_segments) - bends = max(0, len(points) - 2) - length = _path_length(points) - near_edge_penalty = sum(1 for x, y in points if x < 0.012 or x > 0.988 or y < 0.012 or y > 0.988) - score = len(overlaps) * 1000 + crossings * 80 + bends * 8 + length * 16 + near_edge_penalty * 10 - return score, { - "candidate_score": round(score, 4), - "obstacle_overlap_count": len(overlaps), - "obstacle_overlap_ids": overlaps[:12], - "crossing_count": crossings, - "bend_count": bends, - "path_length": length, - } - - -def _best_fallback_path( - source: dict | None, - target: dict | None, - obstacles: dict[str, dict], - existing_segments: list[tuple[list[float], list[float]]], -) -> tuple[list[list[float]], dict[str, Any]]: - if not source or not target: - return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_source_or_target"} - direct = _edge_path(source, target) - if len(direct) < 2: - return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_anchor_points"} - - sx, sy = direct[0] - tx, ty = direct[-1] - x_lanes = _candidate_lanes(source, target, obstacles, "x") - y_lanes = _candidate_lanes(source, target, obstacles, "y") - candidates = [_clean_path(direct)] - candidates.extend(_clean_path([direct[0], [x, sy], [x, ty], direct[-1]]) for x in x_lanes) - candidates.extend(_clean_path([direct[0], [sx, y], [tx, y], direct[-1]]) for y in y_lanes) - # Two-lane orthogonal paths can move around large central obstacles without - # changing the source-target semantics from the reference-derived program. - for x in x_lanes[:12]: - for y in y_lanes[:12]: - candidates.append(_clean_path([direct[0], [x, sy], [x, y], [tx, y], direct[-1]])) - candidates.append(_clean_path([direct[0], [sx, y], [x, y], [x, ty], direct[-1]])) - - best_path: list[list[float]] = direct - best_meta: dict[str, Any] = {} - best_score = float("inf") - seen: set[tuple[tuple[float, float], ...]] = set() - for candidate in candidates: - if len(candidate) < 2: - continue - key = tuple((point[0], point[1]) for point in candidate) - if key in seen: - continue - seen.add(key) - score, meta = _score_candidate(candidate, obstacles, existing_segments) - if score < best_score: - best_score = score - best_path = candidate - best_meta = meta - best_meta.update({ - "routing_algorithm": ROUTER_VERSION, - "route_generation_status": "fallback_route_selected", - "candidate_count": len(seen), - }) - return best_path, best_meta - - -def _infer_role(arrow: dict, out_counts: dict[str, int], in_counts: dict[str, int], panel_ids: set[str]) -> str: - kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() - source = str(arrow.get("source_id") or arrow.get("source") or "") - target = str(arrow.get("target_id") or arrow.get("target") or "") - if "loop" in kind or "dashed" in kind: - return "feedback_loop" - # Branch/convergence semantics should win over panel membership. Some RFS - # programs use panel-like IDs for central modules, and treating those first - # collapses every connector into the same main-flow style. - if out_counts.get(source, 0) > 1: - return "branch" - if in_counts.get(target, 0) > 1: - return "convergence" - if source in panel_ids or target in panel_ids: - return "main_flow" - return "module_flow" - - -def _style_for_role(role: str, arrow: dict, mode: str = "reference") -> dict[str, Any]: - kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() - if str(mode).lower() == "aesthetic": - if role == "main_flow": - return {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "line_cap": "round", "halo_width_pt": 4.4, "halo_color": "#FFFFFF"} - if role == "branch": - return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} - if role == "convergence": - return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} - if role == "feedback_loop": - return {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} - if "elbow" in kind: - return {"route_style": "rounded_elbow", "stroke_width_pt": 1.6, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} - return {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} - if role == "main_flow": - return {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med", "line_cap": "round"} - if role == "branch": - return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} - if role == "convergence": - return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} - if role == "feedback_loop": - return {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash"} - if "elbow" in kind: - return {"route_style": "rounded_elbow", "stroke_width_pt": 1.65, "arrowhead_size": "sm", "line_cap": "round"} - return {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round"} - - -def _reference_locked(arrow: dict) -> bool: - path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - if len(path) < 2: - return False - source = str(arrow.get("binding_source") or arrow.get("detected_by") or "").lower() - if any(term in source for term in ("reference", "opencv", "vlm", "explicit", "candidate", "layout")): - return True - # In reference-primary mode, existing normalized paths are treated as a - # reference contract unless explicitly marked as fallback. - return str(arrow.get("route_policy", "")).lower() != "fallback_reroute_allowed" - - -def _route_metrics(arrows: list[dict], objects: dict[str, dict]) -> tuple[dict[str, dict], int]: - metrics: dict[str, dict] = {} - all_segments: list[tuple[str, list[float], list[float]]] = [] - for arrow in arrows: - points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - sid = str(arrow.get("source_id") or arrow.get("source") or "") - tid = str(arrow.get("target_id") or arrow.get("target") or "") - obstacles = [] - for obj_id, obj in objects.items(): - if obj_id in {sid, tid} or not isinstance(obj.get("bbox_percent"), dict): - continue - if any(_segment_bbox_overlap(a, b, obj["bbox_percent"]) for a, b in _segments(points)): - obstacles.append(obj_id) - metrics[str(arrow.get("id"))] = { - "path_length": _path_length(points), - "bend_count": max(0, len(points) - 2), - "obstacle_overlap_count": len(obstacles), - "obstacle_overlap_ids": obstacles[:12], - "point_count": len(points), - } - for a, b in _segments(points): - all_segments.append((str(arrow.get("id")), a, b)) - - crossing_count = 0 - crossings_by_arrow = {str(arrow.get("id")): 0 for arrow in arrows} - for idx, (aid, a1, a2) in enumerate(all_segments): - for bid, b1, b2 in all_segments[idx + 1:]: - if aid == bid: - continue - if _segments_cross(a1, a2, b1, b2): - crossing_count += 1 - crossings_by_arrow[aid] = crossings_by_arrow.get(aid, 0) + 1 - crossings_by_arrow[bid] = crossings_by_arrow.get(bid, 0) + 1 - for aid, count in crossings_by_arrow.items(): - if aid in metrics: - metrics[aid]["crossing_count"] = count - return metrics, crossing_count - - -def style_and_route_arrows( - program: dict, - out_dir: str | Path, - mode: str = "reference", -) -> dict: - """Add reference-preserving arrow aesthetics and QA artifacts. - - The reference image remains the hard constraint. Existing reference-derived - routes are not freely rerouted; this stage mostly adds style, bundling - metadata, and diagnostics. Only missing/fallback paths are synthesized. - """ - out = Path(out_dir) - if str(mode).lower() == "off": - write_json(out / "arrow_style_profile.json", {"summary": "Arrow styling skipped.", "mode": "off"}) - write_json(out / "selected_arrow_routes.json", {"summary": "Arrow routing skipped.", "mode": "off", "routes": []}) - write_json(out / "arrow_quality_report.json", {"summary": "Arrow quality skipped.", "mode": "off", "status": "skipped"}) - return program - - panels = program.get("panels", []) if isinstance(program.get("panels"), list) else [] - slots = program.get("slots", []) if isinstance(program.get("slots"), list) else [] - objects = {str(item.get("id")): item for item in panels + slots if isinstance(item, dict) and item.get("id")} - slot_objects = {str(item.get("id")): item for item in slots if isinstance(item, dict) and item.get("id")} - panel_ids = {str(panel.get("id")) for panel in panels if isinstance(panel, dict)} - arrows = [dict(item) for item in program.get("arrows", []) if isinstance(item, dict)] - out_counts: dict[str, int] = {} - in_counts: dict[str, int] = {} - for arrow in arrows: - source = str(arrow.get("source_id") or arrow.get("source") or "") - target = str(arrow.get("target_id") or arrow.get("target") or "") - out_counts[source] = out_counts.get(source, 0) + 1 - in_counts[target] = in_counts.get(target, 0) + 1 - - reference_segments: list[tuple[list[float], list[float]]] = [] - for original in arrows: - original_points = original.get("path_percent") if isinstance(original.get("path_percent"), list) else [] - if _reference_locked(original) and len(original_points) >= 2: - reference_segments.extend(_segments(_clean_path(original_points))) - - grouped: dict[str, list[dict]] = {} - enriched = [] - committed_fallback_segments: list[tuple[list[float], list[float]]] = [] - for index, arrow in enumerate(arrows, start=1): - source = str(arrow.get("source_id") or arrow.get("source") or "") - target = str(arrow.get("target_id") or arrow.get("target") or "") - arrow["source_id"] = source - arrow["target_id"] = target - arrow["source"] = source - arrow["target"] = target - points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - locked = _reference_locked(arrow) - route_meta: dict[str, Any] = {"routing_algorithm": "preserve_reference_path", "route_generation_status": "reference_locked"} - fallback_allowed = str(arrow.get("route_policy", "")).lower() == "fallback_reroute_allowed" - if (len(points) < 2 or (fallback_allowed and not locked)) and mode in {"reference", "aesthetic"}: - obstacles = {obj_id: obj for obj_id, obj in slot_objects.items() if obj_id not in {source, target}} - points, route_meta = _best_fallback_path( - objects.get(source), - objects.get(target), - obstacles, - reference_segments + committed_fallback_segments, - ) - locked = False - else: - points = _clean_path(points) - arrow["path_percent"] = [[round(float(p[0]), 4), round(float(p[1]), 4)] for p in points if isinstance(p, list) and len(p) >= 2] - role = _infer_role(arrow, out_counts, in_counts, panel_ids) - style = _style_for_role(role, arrow, mode=mode) - recompute_generated_style = str(arrow.get("aesthetic_policy", "")).startswith("reference_first") - for key, value in style.items(): - if recompute_generated_style or key not in arrow or not str(arrow.get(key, "")).strip(): - arrow[key] = value - if recompute_generated_style or not str(arrow.get("semantic_role", "")).strip(): - arrow["semantic_role"] = role - arrow.setdefault("aesthetic_policy", "reference_first_soft_editable_ppt_connector") - arrow.setdefault("reference_locked", locked) - arrow.setdefault("reference_path_preserved", locked) - arrow.setdefault("route_policy", "preserve_reference_path" if locked else "synthesize_missing_path_only") - arrow.setdefault("routing_algorithm", route_meta.get("routing_algorithm", ROUTER_VERSION)) - arrow.setdefault("route_generation_status", route_meta.get("route_generation_status", "fallback_route_selected" if not locked else "reference_locked")) - if "candidate_count" in route_meta: - arrow.setdefault("candidate_count", route_meta["candidate_count"]) - if "candidate_score" in route_meta: - arrow.setdefault("candidate_score", route_meta["candidate_score"]) - arrow.setdefault("corner_radius_percent", 0.018 if arrow.get("route_style") in {"rounded_elbow", "bundled_elbow"} else 0.0) - if role == "branch": - bundle_id = f"from_{source}" - elif role == "convergence": - bundle_id = f"to_{target}" - elif role == "feedback_loop": - bundle_id = f"loop_{source}_{target}" - else: - bundle_id = f"flow_{index:02d}" - arrow.setdefault("bundle_id", bundle_id) - grouped.setdefault(bundle_id, []).append(arrow) - enriched.append(arrow) - if not locked and len(arrow["path_percent"]) >= 2: - committed_fallback_segments.extend(_segments(arrow["path_percent"])) - - for bundle_id, items in grouped.items(): - items.sort(key=lambda item: (item.get("target_id", ""), item.get("id", ""))) - for lane_index, arrow in enumerate(items): - arrow["lane_index"] = lane_index - arrow["lane_count"] = len(items) - - if str(mode).lower() == "aesthetic": - tunnel_percent = DEFAULT_REFERENCE_TUNNEL_PERCENT - for arrow in enriched: - original_path = _clean_path(arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else []) - arrow["reference_original_path_percent"] = original_path - if arrow.get("reference_locked") and len(original_path) >= 2: - offset_allowed = bool(arrow.get("aesthetic_offset_allowed")) or str(arrow.get("route_policy", "")).lower() == "aesthetic_tunnel_allowed" - if offset_allowed: - adjusted_path, aesthetic_meta = _aesthetic_tunnel_path( - original_path, - int(arrow.get("lane_index") or 0), - int(arrow.get("lane_count") or 1), - str(arrow.get("semantic_role") or ""), - tunnel_percent=tunnel_percent, - ) - arrow["path_percent"] = adjusted_path - arrow["routing_algorithm"] = aesthetic_meta["routing_algorithm"] - arrow["route_generation_status"] = aesthetic_meta["route_generation_status"] - arrow["reference_tunnel_percent"] = aesthetic_meta["reference_tunnel_percent"] - arrow["reference_path_delta_max"] = aesthetic_meta["reference_path_delta_max"] - arrow["reference_tunnel_preserved"] = aesthetic_meta["reference_tunnel_preserved"] - arrow["lane_offset_percent"] = aesthetic_meta["lane_offset_percent"] - arrow["reference_path_preserved"] = bool(aesthetic_meta["reference_tunnel_preserved"]) - else: - arrow["routing_algorithm"] = AESTHETIC_ROUTER_VERSION - arrow["route_generation_status"] = "aesthetic_style_only" - arrow["reference_tunnel_percent"] = tunnel_percent - arrow["reference_path_delta_max"] = 0.0 - arrow["reference_tunnel_preserved"] = True - arrow["lane_offset_percent"] = 0.0 - arrow["reference_path_preserved"] = True +import sys as _sys - metrics, total_crossings = _route_metrics(enriched, slot_objects) - routes = [] - for arrow in enriched: - aid = str(arrow.get("id")) - m = metrics.get(aid, {}) - clutter_score = ( - int(m.get("crossing_count", 0)) * 12 - + int(m.get("bend_count", 0)) * 3 - + int(m.get("obstacle_overlap_count", 0)) * 18 - ) - arrow["aesthetic_score"] = max(0, 100 - clutter_score) - routes.append({ - "id": aid, - "source_id": arrow.get("source_id"), - "target_id": arrow.get("target_id"), - "semantic_role": arrow.get("semantic_role"), - "route_style": arrow.get("route_style"), - "bundle_id": arrow.get("bundle_id"), - "lane_index": arrow.get("lane_index"), - "lane_count": arrow.get("lane_count"), - "reference_locked": arrow.get("reference_locked"), - "reference_path_preserved": arrow.get("reference_path_preserved"), - "path_percent": arrow.get("path_percent"), - "style_token_id": arrow.get("style_token_id"), - "stroke_width_pt": arrow.get("stroke_width_pt"), - "arrowhead_size": arrow.get("arrowhead_size"), - "line_cap": arrow.get("line_cap"), - "line_pattern": arrow.get("line_pattern", "solid"), - "routing_algorithm": arrow.get("routing_algorithm"), - "route_generation_status": arrow.get("route_generation_status"), - "reference_original_path_percent": arrow.get("reference_original_path_percent"), - "reference_tunnel_percent": arrow.get("reference_tunnel_percent"), - "reference_path_delta_max": arrow.get("reference_path_delta_max"), - "reference_tunnel_preserved": arrow.get("reference_tunnel_preserved"), - "lane_offset_percent": arrow.get("lane_offset_percent"), - "halo_width_pt": arrow.get("halo_width_pt"), - "halo_color": arrow.get("halo_color"), - "candidate_count": arrow.get("candidate_count"), - "candidate_score": arrow.get("candidate_score"), - "metrics": m, - "aesthetic_score": arrow["aesthetic_score"], - }) +from .composition.rebuild import arrows as _implementation - arrow_style_profile = { - "summary": "Reference-first arrow styling profile for editable PPT connectors.", - "mode": mode, - "reference_priority": "reference_image_hard_constraint", - "routing_principle": "preserve reference-derived source-target logic and path geometry; only synthesize missing fallback paths", - "routing_algorithm": ROUTER_VERSION, - "aesthetic_routing_algorithm": AESTHETIC_ROUTER_VERSION, - "reference_tunnel_percent": DEFAULT_REFERENCE_TUNNEL_PERCENT, - "fallback_routing_policy": "only arrows with missing paths or route_policy=fallback_reroute_allowed may use obstacle-aware orthogonal routing", - "aesthetic_mode_policy": "mode=aesthetic may adjust reference-locked paths only inside the reference tunnel and must keep reference_tunnel_preserved=true", - "style_rules": { - "main_flow": {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med"}, - "branch": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, - "convergence": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, - "feedback_loop": {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm"}, - "module_flow": {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm"}, - }, - "aesthetic_style_rules": { - "main_flow": {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "halo_width_pt": 4.4}, - "branch": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, - "convergence": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, - "feedback_loop": {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "halo_width_pt": 3.0}, - "module_flow": {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "halo_width_pt": 3.0}, - }, - "ppt_editability": "all arrows render as PPT connector shapes, not raster assets", - "rounded_line_policy": "use round line caps and editable connector segments; reference-locked paths are not geometrically rewritten", - } - selected_routes = { - "summary": "Selected reference-preserving arrow routes and style assignments.", - "mode": mode, - "route_count": len(routes), - "routes": routes, - } - tunnel_violations = [route["id"] for route in routes if route.get("reference_tunnel_preserved") is False] - quality_report = { - "summary": "Arrow routing and styling quality report.", - "mode": mode, - "status": "pass" if all(route["aesthetic_score"] >= 35 for route in routes) and not tunnel_violations else "needs_review", - "arrow_count": len(routes), - "total_crossing_count": total_crossings, - "total_obstacle_overlap_count": sum(int(route["metrics"].get("obstacle_overlap_count", 0)) for route in routes), - "fallback_route_count": sum(1 for route in routes if route.get("route_generation_status") == "fallback_route_selected"), - "aesthetic_tunnel_adjusted_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_tunnel_adjusted"), - "aesthetic_style_only_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_style_only"), - "reference_tunnel_violations": tunnel_violations, - "average_aesthetic_score": round(sum(float(route["aesthetic_score"]) for route in routes) / max(len(routes), 1), 2), - "reference_path_overrides": [route["id"] for route in routes if route.get("reference_locked") and not route.get("reference_path_preserved")], - "routes": routes, - } - program["arrows"] = enriched - program["control_shapes"] = [dict(item) for item in enriched] - program.setdefault("style", {})["arrow_style_profile_path"] = "arrow_style_profile.json" - write_json(out / "arrow_style_profile.json", arrow_style_profile) - write_json(out / "selected_arrow_routes.json", selected_routes) - write_json(out / "arrow_quality_report.json", quality_report) - write_json(out / "figure_program.json", program) - return program +_sys.modules[__name__] = _implementation diff --git a/rfs/asset_generator.py b/rfs/asset_generator.py index 51abb66..fe28292 100644 --- a/rfs/asset_generator.py +++ b/rfs/asset_generator.py @@ -1,1012 +1,9 @@ -from __future__ import annotations +"""Compatibility alias for :mod:`rfs.generation.assets`.""" -import base64 -from concurrent.futures import ThreadPoolExecutor, as_completed -import json -import math -import os -import shutil -import time -from pathlib import Path -from statistics import median -from typing import Iterable +from __future__ import annotations -import requests -from PIL import Image, ImageDraw, ImageFont +import sys as _sys -from .utils import ensure_dir, write_json, write_text +from .generation import assets as _implementation -IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff"} - -CANDIDATE_DIRECTIONS = [ - "dense mechanism-first scientific mini scene with foreground, secondary objects, and micro-detail", - "reference-crop reconstruction with layered local objects and edge-to-edge support detail", - "compact paper-figure mini panel with concrete objects, internal mechanism, and background cues", - "structured process card with visible before-and-after relation plus local reference texture", - "high-density method card focused on the named operation, not a standalone pictogram", -] - -NEGATIVE_PROMPT = ( - "Avoid: large empty margin, tiny centered object, sparse white background, isolated small icon on blank canvas, " - "cropped subject, cut-off edges, fake formulas, fake axes, fake metrics, readable paragraphs, paper title text, " - "watermarks, logos, screenshots, low-resolution blur, poster style, decorative clutter without scientific meaning, " - "extra white presentation tile, white matting, large white border, generic blue-green dashboard styling, " - "simple centered icon, single object on clean blank background, standalone pictogram." -) - - -def _parse_ratio(ratio: str) -> tuple[float, float]: - try: - left, right = ratio.split(":", 1) - return max(0.001, float(left)), max(0.001, float(right)) - except Exception: - return 1.0, 1.0 - - -def _ratio_value(ratio: str) -> float: - rw, rh = _parse_ratio(ratio) - return rw / max(rh, 1) - - -def _target_size(ratio: str, max_side: int = 768, min_side: int = 96) -> tuple[int, int]: - rw, rh = _parse_ratio(ratio) - if rw >= rh: - return max_side, max(min_side, int(max_side * rh / rw)) - return max(min_side, int(max_side * rw / rh)), max_side - - -def _as_list_text(value) -> str: - if isinstance(value, list): - items = [str(item).strip() for item in value if str(item).strip()] - return "; ".join(items) - return str(value or "").strip() - - -def _fallback_visual_metaphor(slot: dict) -> str: - concept = str(slot.get("paper_concept", "scientific method component")) - panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "") - low = f"{concept} {panel}".lower() - if "sparse" in low or "nonzero" in low: - return "draw a matrix grid with many pale empty cells and a few highlighted trainable nonzero cells" - if "empty" in low or "zero" in low: - return "draw a mostly empty matrix grid with muted zero cells and clear blank positions" - if "frozen" in low or "pre-trained" in low or "pretrained" in low: - return "draw a large locked frozen weight matrix slab, with icy muted cells and no trainable highlights" - if "lora" in low: - return "draw two slim low-rank matrix factors forming a compact adapter update beside a larger frozen weight matrix" - if "ift" in low or "fourier" in low or "frequency" in low: - return "draw a frequency-domain sparse grid transforming into a smooth spatial-domain matrix through wave arcs" - if "split" in low or "submat" in low: - return "draw one matrix split into multiple smaller aligned submatrices with visible separation cuts" - if "concat" in low: - return "draw multiple transformed matrix blocks stacked vertically into one taller concatenated matrix" - if "adaptor" in low or "adapter" in low: - return "draw a frozen shared projection block receiving matrix features and emitting a compact update" - if "multiplication" in low: - return "draw two matrix blocks combining through a multiplication junction into a compact output block" - if "budget" in low or "parameter" in low: - return "draw a tiny trainable budget highlighted against a large muted frozen parameter field" - if "benchmark" in low or "evaluation" in low or "glue" in low: - return "draw a benchmark suite board with multiple task tiles and abstract bars without readable numbers" - return f"draw a concrete paper-specific visual object for {concept}, showing its role inside {panel}" - - -def build_slot_prompt(slot: dict, style: dict, candidate_index: int = 1) -> str: - palette = ", ".join(style.get("palette", [])[:5]) - direction = CANDIDATE_DIRECTIONS[(candidate_index - 1) % len(CANDIDATE_DIRECTIONS)] - composition = slot.get("composition_type", "full_frame_icon") - visual_metaphor = str(slot.get("visual_metaphor") or "").strip() or _fallback_visual_metaphor(slot) - slot_function = str(slot.get("slot_function") or "").strip() - image_prompt_core = str(slot.get("image_prompt_core") or "").strip() - reference_content_priority = str(slot.get("reference_content_priority") or "reference_primary_visual_object").strip() - reference_visual_object = str(slot.get("reference_visual_object") or "").strip() - reference_visual_elements = _as_list_text(slot.get("reference_visual_elements")) - reference_color_palette = str(slot.get("reference_color_palette") or "").strip() - paper_label_mapping = str(slot.get("paper_label_mapping") or "").strip() - must_show = _as_list_text(slot.get("must_show")) or "the concrete scientific object, its internal structure, and its input-output relation" - avoid_showing = _as_list_text(slot.get("avoid_showing")) or "generic sci-fi dashboard; unrelated robot or brain; fake formulas; fake numeric chart text" - panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "paper method") - reference_slot_role = str(slot.get("reference_slot_role") or "slot in the user-provided reference layout").strip() - reference_shape_language = str(slot.get("reference_shape_language") or slot.get("target_canvas_ratio") or "").strip() - reference_local_style = str(slot.get("reference_local_style") or "match the reference figure's local card/icon discipline").strip() - reference_prompt_hint = str(slot.get("reference_prompt_hint") or "use the reference image as local layout/style guidance, not as scientific content").strip() - reference_crop_path = str(slot.get("reference_crop_path") or "").strip() - reference_style_profile_path = str(slot.get("reference_style_profile_path") or "reference_style_profile.json").strip() - local_color_token_ids = ", ".join(str(item) for item in slot.get("local_color_token_ids", []) if str(item).strip()) - complexity_kind = str(slot.get("complexity_kind") or "pipeline_module").strip() - required_visual_complexity = str(slot.get("required_visual_complexity") or "dense").strip() - foreground_subject = str(slot.get("foreground_subject") or reference_visual_object or visual_metaphor).strip() - secondary_objects = _as_list_text(slot.get("secondary_objects")) or "2-5 supporting local objects from the reference crop" - micro_details = _as_list_text(slot.get("micro_details")) or "small non-critical marks, separators, texture, and internal details" - background_fill_elements = _as_list_text(slot.get("background_fill_elements")) or "edge-to-edge local reference-colored support detail" - scientific_mechanism_detail = str(slot.get("scientific_mechanism_detail") or "show the paper concept as a visible mechanism, not a generic icon").strip() - forbidden_simplification = _as_list_text(slot.get("forbidden_simplification")) or "simple icon; centered icon; clean blank background; single object on white canvas" - is_simple_allowed = complexity_kind == "legend_icon" - - if composition == "scene_thumbnail": - type_clause = ( - "Make it a dense small mechanism thumbnail: one concrete foreground scientific object, multiple visible process cues, secondary context objects, and enough environment to show what changes. Use edge-to-edge supporting scene detail so the crop is visually full." - ) - elif composition == "full_bleed_card": - type_clause = ( - "Make it a compact dense mechanism card: the central object fills the card, with internal subparts, small supporting objects, and micro process cues that clarify this exact operation. The card surface should extend almost to the image edge, with no separate white mat." - ) - elif composition == "symbol_cutout": - type_clause = ( - "Make it a large complete symbolic operation object plus local reference-colored support detail; the operation symbol or object should be visually distinct from other slots. If it is a line icon, thicken the strokes, enlarge it to nearly touch the safe area, and add compact supporting shapes so it is not sparse." - ) - else: - type_clause = ( - "Make it a full-frame scientific mechanism object with a large distinct subject, mechanism-specific internal details, and supporting local objects. If the subject is a waveform, face, pose skeleton, document, database, badge, or legend icon, make it bold, oversized, and backed by local reference-colored detail rather than a tiny object on a blank white field." - ) - - exact_ratio = float(slot.get("aspect_ratio_decimal") or _ratio_value(slot.get("target_canvas_ratio", "1.000:1.000"))) - center = slot.get("center_percent") if isinstance(slot.get("center_percent"), dict) else {} - center_text = f"x={float(center.get('x', 0.5)):.3f}, y={float(center.get('y', 0.5)):.3f}" - width_percent = float(slot.get("width_percent", slot.get("bbox_percent", {}).get("w", 0))) - height_percent = float(slot.get("height_percent", slot.get("bbox_percent", {}).get("h", 0))) - - return ( - f"Create candidate {candidate_index} for one slot-level image block in a publication-quality AI/ML research framework figure. " - f"This block belongs to macro panel: {panel}. Slot id: {slot['id']}. Paper concept: {slot['paper_concept']}. " - f"Slot function: {slot_function or 'visualize this paper concept as one small editable figure asset'}. " - f"Model-planned image prompt core: {image_prompt_core or visual_metaphor}. " - f"Reference content priority: {reference_content_priority}. " - f"Reference visual object to recreate: {reference_visual_object or visual_metaphor}. " - f"Reference visual elements to preserve: {reference_visual_elements or must_show}. " - f"Reference local color palette to preserve: {reference_color_palette or 'inherit colors from the local reference slot, not one global palette'}. " - f"Reference style profile source: {reference_style_profile_path}. Local reference crop used by the prompt planner: {reference_crop_path or 'missing crop path - preserve described local crop anyway'}. " - f"Local reference color token ids to inherit: {local_color_token_ids or 'none recorded; use local reference colors described in prompt plan'}. " - f"Paper label mapping: {paper_label_mapping or 'paper terminology is added later as editable PPT text'}. " - f"Main visual metaphor: {visual_metaphor}. " - f"Must visibly show: {must_show}. " - f"Reference-local slot role: {reference_slot_role}. " - f"Reference shape language: {reference_shape_language}. " - f"Reference-local style: {reference_local_style}. " - f"Reference prompt hint: {reference_prompt_hint}. " - f"Slot visual complexity contract: complexity_profile={slot.get('complexity_profile', 'reference-dense')}; complexity_kind={complexity_kind}; required_visual_complexity={required_visual_complexity}. " - f"Foreground subject must be large: {foreground_subject}. " - f"Secondary objects to include: {secondary_objects}. " - f"Micro details to include: {micro_details}. " - f"Background fill elements: {background_fill_elements}. " - f"Scientific mechanism detail: {scientific_mechanism_detail}. " - f"Forbidden simplifications: {forbidden_simplification}. " - "Use the local reference slot object first; do not invent a different abstract paper-internal object when the reference shows a concrete object. " - "The goal is to redraw/recreate the reference slot's object as a clean high-resolution asset, then adapt only minor details to the paper concept. " - f"Reference geometry is mandatory: canvas aspect ratio = exact {exact_ratio:.3f}:1, target_canvas_ratio = {slot['target_canvas_ratio']}, " - f"slot center in full figure = {center_text}, slot width_percent = {width_percent:.3f}, slot height_percent = {height_percent:.3f}. " - "Generate this asset for the exact reference slot shape, not a generic preset ratio such as 1:1, 4:3, 3:4, 16:9, or 9:16. " - "No semantic cropping: the whole subject must be visible, no object extending outside the frame, no cut-off hands/edges/bottom. " - "Safe area means important subject remains complete inside the frame, but supporting background details should extend close to edges. " - "Useful visual content should fill 90-97% of the canvas; minimum 85%; maximum empty margin is strictly below 10% on every edge. " - "Use a full-frame composition, large subject, maximal detail density, rich scientific visual texture, minimal blank canvas, no tiny centered object. " - + ("" if is_simple_allowed else "This must be a dense mini scientific scene/card with 2-5 layered objects, edge-to-edge supporting detail, and not a standalone pictogram. ") - + "The subject plus its local reference-colored support surface should occupy almost the entire frame; avoid isolated cutout icons floating on white or transparent-looking empty space. " - "For thin symbols such as pose skeletons, waveforms, clocks, microphones, documents, badges, and legend icons, use thick readable strokes, enlarged scale, and compact supporting shapes to raise visual fill while keeping the subject complete. " - f"Candidate visual direction: {direction}. {type_clause} " - f"Style constraints: polished academic paper illustration, crisp edges, consistent line quality, clean high-resolution vector-illustration / 2.5D hybrid only when useful. " - f"Global palette may include {palette}, but local reference colors override the global palette. Preserve color variety across slots. " - "Style consistency matters, but visual identity must vary by the reference crop: monitors look like monitors, video strips look like video strips, microphones look like microphones, waveforms look like waveforms, face cards look like face cards, pose skeletons look like pose skeletons, clocks look like clocks, clipboards look like clipboards, databases look like databases. " - "Allow only very small decorative non-critical UI marks or blurred pseudo-labels. " - "Critical scientific labels, formulas, variables, panel IDs, arrow labels, and metric values will be added later in PowerPoint and must not be generated in the image. " - f"Do not show: {avoid_showing}. " - "Do not add a white presentation tile, white mat, extra card background, or artificial border unless that exact border exists in the local reference crop. " - f"{NEGATIVE_PROMPT}" - ) - - -def _draw_placeholder(slot: dict, path: Path, style: dict, candidate_index: int = 1) -> None: - width, height = _target_size(slot["target_canvas_ratio"]) - palette = style.get("palette", ["#EAF5FF", "#DFF7EF", "#8EC9E8", "#66C6A4", "#163B4D"]) - bg = palette[(candidate_index - 1) % 2] - line = palette[4] if len(palette) > 4 else "#163B4D" - accent = palette[2 + ((candidate_index + len(slot["id"])) % max(1, min(2, len(palette) - 2)))] if len(palette) >= 4 else "#8EC9E8" - img = Image.new("RGB", (width, height), bg) - draw = ImageDraw.Draw(img) - - margin = max(8, int(min(width, height) * 0.035)) - draw.rounded_rectangle([margin, margin, width - margin, height - margin], radius=max(14, margin * 2), fill=bg, outline=line, width=max(3, margin // 4)) - - # Dense edge-to-edge detail so placeholder QA exercises high-fill layouts. - for i in range(8 + candidate_index): - x = int(width * (0.035 + i * 0.12)) % width - draw.line([x, margin, min(width - margin, x + int(width * 0.22)), height - margin], fill=accent, width=max(1, margin // 6)) - for i in range(6): - y = int(height * (0.10 + i * 0.15)) - draw.arc([margin + i * 5, y, width - margin - i * 4, y + int(height * 0.26)], 180, 360, fill=accent, width=max(1, margin // 7)) - - cx, cy = width // 2, height // 2 - main_w = int(width * (0.72 + 0.03 * ((candidate_index - 1) % 3))) - main_h = int(height * (0.64 + 0.03 * ((candidate_index - 1) % 2))) - kind = slot.get("composition_type", "full_frame_icon") - if kind in {"scene_thumbnail", "full_bleed_card"}: - draw.rounded_rectangle([cx - main_w // 2, cy - main_h // 2, cx + main_w // 2, cy + main_h // 2], radius=max(12, margin), fill="#FFFFFF", outline=line, width=max(3, margin // 4)) - panel_gap = max(4, margin // 3) - for row in range(2): - for col in range(2): - px0 = cx - main_w // 2 + margin + col * ((main_w - margin * 2) // 2) - py0 = cy - main_h // 2 + margin + row * ((main_h - margin * 2) // 2) - px1 = px0 + (main_w - margin * 2) // 2 - panel_gap - py1 = py0 + (main_h - margin * 2) // 2 - panel_gap - draw.rounded_rectangle([px0, py0, px1, py1], radius=max(6, margin // 2), fill=accent if (row + col + candidate_index) % 2 == 0 else "#DFF7EF", outline=line, width=max(1, margin // 8)) - draw.polygon([(px0 + panel_gap, py1 - panel_gap), (px0 + (px1 - px0) // 2, py0 + panel_gap), (px1 - panel_gap, py1 - panel_gap)], fill="#9AD7C4") - else: - radius = min(main_w, main_h) // 2 - draw.ellipse([cx - radius, cy - radius, cx + radius, cy + radius], fill="#FFFFFF", outline=line, width=max(4, margin // 3)) - for angle_idx in range(12): - ang = angle_idx * math.pi / 6 + candidate_index * 0.08 - x1 = cx + int(math.cos(ang) * radius * 0.18) - y1 = cy + int(math.sin(ang) * radius * 0.18) - x2 = cx + int(math.cos(ang) * radius * 0.82) - y2 = cy + int(math.sin(ang) * radius * 0.82) - draw.line([x1, y1, x2, y2], fill=line, width=max(3, margin // 4)) - draw.ellipse([x2 - margin // 2, y2 - margin // 2, x2 + margin // 2, y2 + margin // 2], fill=accent, outline=line) - inner = int(radius * 0.36) - draw.rounded_rectangle([cx - inner, cy - inner, cx + inner, cy + inner], radius=max(8, margin // 2), fill=accent, outline=line, width=max(2, margin // 6)) - - try: - font = ImageFont.truetype("arial.ttf", max(9, int(min(width, height) * 0.045))) - except Exception: - font = ImageFont.load_default() - draw.text((margin * 1.2, height - margin * 2.2), "UI", fill=line, font=font) - path.parent.mkdir(parents=True, exist_ok=True) - img.save(path) - - -def _supported_gemini_ratio(ratio: str) -> str: - value = _ratio_value(ratio) - supported = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "16:9": 16 / 9, "9:16": 9 / 16} - return min(supported, key=lambda key: abs(supported[key] - value)) - - -def _image2_model_name() -> str: - model = os.getenv("RFS_IMAGE_MODEL") or os.getenv("IMAGE_MODEL") or "image-2" - # Yunwu currently exposes GPT Image 2 under this model id. Keep "image-2" - # as the user-facing logical name while sending the listed model id. - if model == "image-2": - return "gpt-image-2" - return model - - -def _image2_size(ratio: str) -> str: - value = _ratio_value(ratio) - if value >= 1.25: - return "1536x1024" - if value <= 0.80: - return "1024x1536" - return "1024x1024" - - -def _write_image_response(data: dict, output_path: Path) -> bool: - items = data.get("data") if isinstance(data.get("data"), list) else [] - if not items: - return False - item = items[0] - output_path.parent.mkdir(parents=True, exist_ok=True) - if item.get("b64_json"): - output_path.write_bytes(base64.b64decode(item["b64_json"])) - return True - if item.get("url"): - response = requests.get(item["url"], timeout=180) - response.raise_for_status() - output_path.write_bytes(response.content) - return True - return False - - -def _call_image2(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: - api_base = os.getenv("API_BASE", "https://yunwu.ai/v1").rstrip("/") - api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") - model = _image2_model_name() - if not api_key: - raise RuntimeError("API_KEY/GEMINI_API_KEY is required for --asset-mode image2") - payload = { - "model": model, - "prompt": prompt, - "n": 1, - "size": _image2_size(ratio), - } - max_attempts = max(1, int(retries) + 1) - last_error: Exception | None = None - for attempt in range(1, max_attempts + 1): - try: - response = requests.post( - f"{api_base}/images/generations", - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=240, - ) - if response.status_code == 400 and payload.get("size") != "1024x1024": - # Some image gateways accept only square output for a model. The - # prompt and final ratio-normalization still preserve slot ratio. - payload = dict(payload) - payload["size"] = "1024x1024" - response = requests.post( - f"{api_base}/images/generations", - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=240, - ) - if response.status_code == 429 or 500 <= response.status_code < 600: - raise RuntimeError(f"image2 request returned HTTP {response.status_code}") - response.raise_for_status() - if _write_image_response(response.json(), output_path): - return True - raise RuntimeError("image2 response did not contain b64_json or url image data") - except Exception as exc: - last_error = exc - if attempt >= max_attempts: - break - time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) - if last_error: - raise last_error - return False - - -def _call_gemini(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: - api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") - url = os.getenv("GEMINI_GEN_IMG_URL") - if not api_key or not url: - raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode gemini") - payload = { - "contents": [{"role": "user", "parts": [{"text": prompt}]}], - "generationConfig": { - "responseModalities": ["IMAGE"], - "imageConfig": {"imageSize": "2K", "aspectRatio": _supported_gemini_ratio(ratio)}, - }, - } - max_attempts = max(1, int(retries) + 1) - last_error: Exception | None = None - for attempt in range(1, max_attempts + 1): - try: - response = requests.post( - url, - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=180, - ) - if response.status_code == 429 or 500 <= response.status_code < 600: - raise RuntimeError(f"Gemini image request returned HTTP {response.status_code}") - response.raise_for_status() - data = response.json() - for candidate in data.get("candidates", []): - for part in candidate.get("content", {}).get("parts", []): - inline = part.get("inlineData") or part.get("inline_data") - if inline and inline.get("data"): - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_bytes(base64.b64decode(inline["data"])) - return True - raise RuntimeError("Gemini image response did not contain inline image data") - except Exception as exc: - last_error = exc - if attempt >= max_attempts: - break - # Back off on rate limits, transient server errors, and network timeouts. - time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) - if last_error: - raise last_error - return False - - -def _enforce_target_canvas_ratio(path: Path, ratio: str) -> None: - """Resize to the exact slot canvas ratio without semantic cropping.""" - target_w, target_h = _target_size(ratio, max_side=1024, min_side=96) - with Image.open(path) as image: - converted = image.convert("RGBA") - if converted.size == (target_w, target_h): - return - resized = converted.resize((target_w, target_h), Image.LANCZOS) - path.parent.mkdir(parents=True, exist_ok=True) - resized.save(path) - - -def _content_bbox(rgba: Image.Image) -> tuple[list[int], float, float]: - width, height = rgba.size - pixels = rgba.load() - background = _median_corner_color(pixels, width, height) - xs: list[int] = [] - ys: list[int] = [] - color_threshold = 30.0 - alpha_threshold = 12 - for y in range(height): - for x in range(width): - r, g, b, a = pixels[x, y] - if a <= alpha_threshold: - continue - if a < 245 or _color_distance((r, g, b), background) > color_threshold: - xs.append(x) - ys.append(y) - if not xs or not ys: - return [0, 0, width, height], 0.0, 100.0 - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) - bbox_w = max(1, right - left + 1) - bbox_h = max(1, bottom - top + 1) - content_fill = bbox_w * bbox_h / float(width * height) * 100.0 - margins = ( - left / width * 100.0, - (width - right - 1) / width * 100.0, - top / height * 100.0, - (height - bottom - 1) / height * 100.0, - ) - return [left, top, right + 1, bottom + 1], content_fill, max(margins) - - -def _hex_rgb(value: str, fallback: tuple[int, int, int] = (229, 242, 248)) -> tuple[int, int, int]: - text = str(value or "").strip().lstrip("#") - if len(text) >= 6: - try: - return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) - except Exception: - return fallback - return fallback - - -def _fit_subject_to_canvas(path: Path, min_fill: float = 88.0, max_margin: float = 9.0, support_color: str | None = None) -> None: - """Remove only plain background slack and enlarge the complete subject.""" - with Image.open(path) as image: - rgba = image.convert("RGBA") - width, height = rgba.size - bbox, content_fill, empty_margin = _content_bbox(rgba) - if content_fill >= min_fill and empty_margin <= max_margin: - return - left, top, right, bottom = bbox - pad_x = max(4, int((right - left) * 0.06)) - pad_y = max(4, int((bottom - top) * 0.06)) - left = max(0, left - pad_x) - top = max(0, top - pad_y) - right = min(width, right + pad_x) - bottom = min(height, bottom + pad_y) - if right <= left or bottom <= top: - return - subject = rgba.crop((left, top, right, bottom)) - scale = min(width * 0.94 / max(subject.width, 1), height * 0.94 / max(subject.height, 1)) - if scale <= 1.02: - canvas = rgba - else: - new_w = max(1, min(width, int(subject.width * scale))) - new_h = max(1, min(height, int(subject.height * scale))) - subject = subject.resize((new_w, new_h), Image.LANCZOS) - pixels = rgba.load() - bg = tuple(int(v) for v in _median_corner_color(pixels, width, height)) + (255,) - canvas = Image.new("RGBA", (width, height), bg) - canvas.alpha_composite(subject, ((width - new_w) // 2, (height - new_h) // 2)) - _bbox, final_fill, final_margin = _content_bbox(canvas) - if (final_fill < min_fill or final_margin > max_margin) and support_color: - support = Image.new("RGBA", (width, height), (0, 0, 0, 0)) - draw = ImageDraw.Draw(support) - r, g, b = _hex_rgb(support_color) - inset_x = max(2, int(width * 0.025)) - inset_y = max(2, int(height * 0.025)) - draw.rounded_rectangle( - (inset_x, inset_y, width - inset_x, height - inset_y), - radius=max(8, min(width, height) // 12), - fill=(r, g, b, 82), - outline=(r, g, b, 150), - width=max(2, min(width, height) // 80), - ) - for i in range(5): - x = int(width * (0.10 + i * 0.18)) - draw.line((x, inset_y + 4, x + int(width * 0.10), height - inset_y - 4), fill=(r, g, b, 58), width=2) - canvas.alpha_composite(support) - canvas.save(path) - - -def _median_corner_color(pixels, width: int, height: int) -> tuple[float, float, float]: - sample = [] - band_x = max(1, min(width // 12, 28)) - band_y = max(1, min(height // 12, 28)) - regions = ( - (0, 0, band_x, band_y), - (width - band_x, 0, width, band_y), - (0, height - band_y, band_x, height), - (width - band_x, height - band_y, width, height), - ) - for x0, y0, x1, y1 in regions: - for y in range(y0, y1): - for x in range(x0, x1): - r, g, b, _a = pixels[x, y] - sample.append((r, g, b)) - if not sample: - return (255.0, 255.0, 255.0) - return ( - float(median([rgb[0] for rgb in sample])), - float(median([rgb[1] for rgb in sample])), - float(median([rgb[2] for rgb in sample])), - ) - - -def _color_distance(a: tuple[int, int, int], b: tuple[float, float, float]) -> float: - return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2) ** 0.5 - - -def _estimate_quality(path: Path, slot: dict, candidate_index: int, source: str) -> dict: - with Image.open(path) as image: - rgba = image.convert("RGBA") - width, height = rgba.size - pixels = rgba.load() - background = _median_corner_color(pixels, width, height) - - xs: list[int] = [] - ys: list[int] = [] - color_threshold = 30.0 - alpha_threshold = 12 - for y in range(height): - for x in range(width): - r, g, b, a = pixels[x, y] - if a <= alpha_threshold: - continue - if a < 245 or _color_distance((r, g, b), background) > color_threshold: - xs.append(x) - ys.append(y) - - if not xs or not ys: - content_fill = 0.0 - empty_margin = 100.0 - bbox = [0, 0, width, height] - else: - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) - bbox_w = max(1, right - left + 1) - bbox_h = max(1, bottom - top + 1) - bbox = [left, top, right + 1, bottom + 1] - content_fill = bbox_w * bbox_h / float(width * height) * 100.0 - margins = ( - left / width * 100.0, - (width - right - 1) / width * 100.0, - top / height * 100.0, - (height - bottom - 1) / height * 100.0, - ) - empty_margin = max(margins) - - image_ratio = width / max(height, 1) - target_ratio = _ratio_value(slot["target_canvas_ratio"]) - ratio_mismatch_percent = abs(image_ratio - target_ratio) / max(target_ratio, 0.001) * 100.0 - - issue_tags = [] - if content_fill < float(slot.get("min_content_fill_percent", 80)): - issue_tags.append("too_much_whitespace") - if empty_margin > float(slot.get("max_empty_margin_percent", 12)): - issue_tags.append("large_blank_canvas") - if content_fill < 65 and empty_margin > 18: - issue_tags.append("tiny_centered_subject") - if ratio_mismatch_percent > 10: - issue_tags.append("ratio_mismatch") - - fill_score = min(content_fill, 95.0) - max(0.0, content_fill - 95.0) * 1.5 - margin_penalty = max(0.0, empty_margin - 12.0) * 4.0 - ratio_penalty = max(0.0, ratio_mismatch_percent - 10.0) * 2.0 - score = fill_score - margin_penalty - ratio_penalty - - return { - "candidate_index": candidate_index, - "path": str(path), - "source": source, - "width": width, - "height": height, - "content_bbox_px": bbox, - "content_fill_percent": round(content_fill, 2), - "min_content_fill_percent": slot["min_content_fill_percent"], - "empty_margin_percent": round(empty_margin, 2), - "max_empty_margin_percent": slot["max_empty_margin_percent"], - "ratio_mismatch_percent": round(ratio_mismatch_percent, 2), - "edge_cutoff_status": "ok", - "ratio_status": "ok" if ratio_mismatch_percent <= 10 else "needs_ratio_regeneration", - "issue_tags": issue_tags, - "selection_score": round(score, 2), - "action": "ok_no_crop" if not issue_tags else "candidate_rejected", - "estimation_method": "corner-background-content-bbox", - } - - -def _simple_allowed_slot(slot: dict) -> bool: - text = " ".join( - str(slot.get(key, "")) - for key in ["id", "paper_concept", "macro_panel", "parent_panel", "complexity_kind", "composition_type"] - ).lower() - return str(slot.get("complexity_kind", "")).lower() == "legend_icon" or any( - term in text - for term in ["legend", "badge", "label", "ocean", "openness", "conscientiousness", "extraversion", "agreeableness", "neuroticism"] - ) - - -def _estimate_component_count(mask: list[list[bool]]) -> int: - if not mask or not mask[0]: - return 0 - height = len(mask) - width = len(mask[0]) - seen = [[False] * width for _ in range(height)] - components = 0 - min_area = max(6, int(width * height * 0.002)) - for y in range(height): - for x in range(width): - if seen[y][x] or not mask[y][x]: - continue - stack = [(x, y)] - seen[y][x] = True - area = 0 - while stack: - px, py = stack.pop() - area += 1 - for nx, ny in ((px + 1, py), (px - 1, py), (px, py + 1), (px, py - 1)): - if 0 <= nx < width and 0 <= ny < height and not seen[ny][nx] and mask[ny][nx]: - seen[ny][nx] = True - stack.append((nx, ny)) - if area >= min_area: - components += 1 - return components - - -def _estimate_complexity(path: Path, slot: dict, quality: dict) -> dict: - with Image.open(path) as image: - rgba = image.convert("RGBA") - width, height = rgba.size - small_w = 96 - small_h = max(16, int(96 * height / max(width, 1))) - if small_h > 128: - small_h = 128 - small_w = max(16, int(128 * width / max(height, 1))) - small = rgba.resize((small_w, small_h), Image.LANCZOS) - pixels = small.load() - background = _median_corner_color(pixels, small_w, small_h) - mask: list[list[bool]] = [] - quantized_colors: set[tuple[int, int, int]] = set() - edge_hits = 0 - comparisons = 0 - for y in range(small_h): - row: list[bool] = [] - for x in range(small_w): - r, g, b, a = pixels[x, y] - active = a > 12 and (a < 245 or _color_distance((r, g, b), background) > 30.0) - row.append(active) - if active: - quantized_colors.add((r // 24, g // 24, b // 24)) - if x > 0: - pr, pg, pb, pa = pixels[x - 1, y] - comparisons += 1 - if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: - edge_hits += 1 - if y > 0: - pr, pg, pb, pa = pixels[x, y - 1] - comparisons += 1 - if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: - edge_hits += 1 - mask.append(row) - component_count = _estimate_component_count(mask) - color_count = len(quantized_colors) - edge_density = edge_hits / max(comparisons, 1) * 100.0 - content_fill = float(quality.get("content_fill_percent", 0)) - spec_objects = len(slot.get("secondary_objects") if isinstance(slot.get("secondary_objects"), list) else []) - spec_micro = len(slot.get("micro_details") if isinstance(slot.get("micro_details"), list) else []) - object_count_estimate = max(component_count, 1 + min(4, spec_objects // 2)) - detail_score = min( - 100.0, - content_fill * 0.30 - + min(color_count, 24) * 1.25 - + min(edge_density, 28.0) * 1.15 - + min(component_count, 8) * 4.0 - + min(spec_objects + spec_micro, 8) * 2.5, - ) - target = float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)) - object_target = int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)) - simple_icon_risk = False - issue_tags: list[str] = [] - if not _simple_allowed_slot(slot): - if detail_score < target: - simple_icon_risk = True - issue_tags.append("too_simple") - if object_count_estimate < object_target: - simple_icon_risk = True - issue_tags.append("generic_icon") - if content_fill >= 85 and color_count <= 4 and edge_density < 3: - simple_icon_risk = True - issue_tags.append("single_object_on_blank_background") - if not str(slot.get("reference_crop_path", "")).strip(): - issue_tags.append("reference_crop_ignored") - if not str(slot.get("reference_style_profile_path", "")).strip(): - issue_tags.append("style_drift") - return { - "detail_score": round(detail_score, 2), - "detail_score_target": target, - "object_count_estimate": object_count_estimate, - "object_count_target": object_target, - "simple_icon_risk": simple_icon_risk, - "reference_crop_match": "planned_reference_crop_grounded" if str(slot.get("reference_crop_path", "")).strip() else "missing_reference_crop", - "style_match": "planned_reference_style_grounded" if str(slot.get("reference_style_profile_path", "")).strip() else "missing_reference_style", - "color_bin_count": color_count, - "edge_density_percent": round(edge_density, 2), - "component_count_estimate": component_count, - "complexity_kind": slot.get("complexity_kind") or "pipeline_module", - "required_visual_complexity": slot.get("required_visual_complexity") or "dense", - "complexity_issue_tags": sorted(set(issue_tags)), - } - - -def _select_candidate(candidate_reports: list[dict]) -> dict: - passing = [item for item in candidate_reports if not item.get("issue_tags")] - pool = passing or candidate_reports - return max(pool, key=lambda item: item["selection_score"]) - - -def _make_contact_sheet(asset_paths: Iterable[Path], output_path: Path, title: str = "selected assets") -> None: - paths = list(asset_paths) - thumbs = [] - cell_w, cell_h = 260, 228 - for path in paths: - img = Image.open(path).convert("RGB") - img.thumbnail((224, 162), Image.LANCZOS) - cell = Image.new("RGB", (cell_w, cell_h), "white") - draw = ImageDraw.Draw(cell) - cell.paste(img, ((cell_w - img.width) // 2, 14)) - draw.text((10, 184), path.stem[:34], fill=(20, 20, 20)) - thumbs.append(cell) - cols = 5 - rows = max(1, (len(thumbs) + cols - 1) // cols) - header_h = 34 - sheet = Image.new("RGB", (cols * cell_w, rows * cell_h + header_h), (245, 247, 248)) - draw = ImageDraw.Draw(sheet) - draw.text((12, 10), title, fill=(20, 20, 20)) - for index, thumb in enumerate(thumbs): - sheet.paste(thumb, ((index % cols) * cell_w, header_h + (index // cols) * cell_h)) - output_path.parent.mkdir(parents=True, exist_ok=True) - sheet.save(output_path) - - -def _generate_one_candidate(slot: dict, style: dict, path: Path, asset_mode: str, candidate_index: int, prompt: str, retries: int = 2) -> str: - support_color = "" - if isinstance(slot.get("reference_dominant_colors"), list) and slot.get("reference_dominant_colors"): - support_color = str(slot["reference_dominant_colors"][0]) - elif isinstance(style.get("reference_palette"), list) and style.get("reference_palette"): - support_color = str(style["reference_palette"][0]) - if asset_mode == "placeholder": - _draw_placeholder(slot, path, style, candidate_index=candidate_index) - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) - _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) - return "explicit_placeholder_candidate" - if asset_mode == "gemini": - ok = _call_gemini(prompt, slot["target_canvas_ratio"], path, retries=retries) - if not ok: - raise RuntimeError(f"Gemini returned no image for slot {slot['id']} candidate {candidate_index}") - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) - _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) - return "gemini_candidate_generation" - if asset_mode == "image2": - ok = _call_image2(prompt, slot["target_canvas_ratio"], path, retries=retries) - if not ok: - raise RuntimeError(f"image2 returned no image for slot {slot['id']} candidate {candidate_index}") - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) - _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) - return f"image2_candidate_generation:{_image2_model_name()}" - raise ValueError(f"Unsupported asset_mode: {asset_mode}") - - -def _generation_failure_report(slot: dict, candidate_index: int, path: Path, exc: Exception) -> dict: - return { - "candidate_index": candidate_index, - "path": str(path), - "source": "generation_failed", - "width": 0, - "height": 0, - "content_bbox_px": [0, 0, 0, 0], - "content_fill_percent": 0.0, - "min_content_fill_percent": slot["min_content_fill_percent"], - "empty_margin_percent": 100.0, - "max_empty_margin_percent": slot["max_empty_margin_percent"], - "ratio_mismatch_percent": 100.0, - "edge_cutoff_status": "unknown_generation_failed", - "ratio_status": "unknown_generation_failed", - "issue_tags": ["generation_failed"], - "selection_score": -9999.0, - "detail_score": 0.0, - "detail_score_target": float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)), - "object_count_estimate": 0, - "object_count_target": int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)), - "simple_icon_risk": True, - "reference_crop_match": "unknown_generation_failed", - "style_match": "unknown_generation_failed", - "complexity_issue_tags": ["generation_failed"], - "action": "candidate_rejected", - "error": str(exc), - } - - -def _generate_and_score_candidate(task: dict) -> dict: - slot = task["slot"] - candidate_path = task["path"] - try: - source = _generate_one_candidate( - slot, - task["style"], - candidate_path, - task["asset_mode"], - task["candidate_index"], - task["prompt"], - retries=task["asset_retries"], - ) - quality = _estimate_quality(candidate_path, slot, task["candidate_index"], source) - complexity = _estimate_complexity(candidate_path, slot, quality) - issue_tags = sorted(set(list(quality.get("issue_tags", [])) + list(complexity.get("complexity_issue_tags", [])))) - quality.update(complexity) - quality["issue_tags"] = issue_tags - complexity_penalty = 0.0 - if complexity.get("simple_icon_risk"): - complexity_penalty += 35.0 - complexity_penalty += max(0.0, float(complexity.get("detail_score_target", 0)) - float(complexity.get("detail_score", 0))) * 0.75 - quality["selection_score"] = round(float(quality.get("selection_score", 0)) + float(complexity.get("detail_score", 0)) * 0.35 - complexity_penalty, 2) - quality["action"] = "ok_no_crop" if not issue_tags else "candidate_rejected" - return quality - except Exception as exc: - return _generation_failure_report(slot, task["candidate_index"], candidate_path, exc) - - -def generate_assets( - program: dict, - style: dict, - out_dir: str | Path, - asset_mode: str = "gemini", - candidates_per_slot: int = 3, - asset_workers: int = 1, - asset_retries: int = 2, -) -> dict: - out = Path(out_dir) - assets_dir = ensure_dir(out / "assets") - candidates_root = ensure_dir(out / "asset_candidates") - candidates_per_slot = max(1, min(5, int(candidates_per_slot))) - asset_workers = max(1, min(12, int(asset_workers))) - asset_retries = max(0, min(5, int(asset_retries))) - - prompt_lines = [ - "# Summary", - "Slot-level image prompts generated after paper brief, reference slot analysis, style sheet, layout_plan.json, figure_program.json, and slot_prompt_plan.json.", - "Each prompt inherits the model-planned image_prompt_core for its slot before adding canvas, fill, text-control, and no-crop constraints.", - f"Each slot generates {candidates_per_slot} candidate image block(s) with up to {asset_workers} worker(s); the selected asset is copied into `assets/` without semantic cropping.", - "", - ] - report_items = [] - complexity_items = [] - selected_paths = [] - all_candidate_paths = [] - tasks = [] - ordered_slot_candidates: dict[str, list[dict]] = {} - - for slot in program["slots"]: - slot_candidate_dir = ensure_dir(candidates_root / slot["id"]) - ordered_slot_candidates[slot["id"]] = [] - prompt_lines.append(f"## {slot['id']}") - prompt_lines.append(f"Paper concept: {slot['paper_concept']}") - prompt_lines.append(f"Prompt plan id: {slot.get('prompt_plan_id', 'missing')}") - prompt_lines.append(f"Visual spec id: {slot.get('visual_spec_id', 'missing')}") - prompt_lines.append(f"Complexity kind: {slot.get('complexity_kind', 'pipeline_module')}") - prompt_lines.append(f"Image prompt core: {slot.get('image_prompt_core', '')}") - prompt_lines.append("") - - for candidate_index in range(1, candidates_per_slot + 1): - prompt = build_slot_prompt(slot, style, candidate_index=candidate_index) - candidate_path = slot_candidate_dir / f"candidate_{candidate_index:02d}.png" - prompt_lines.extend([f"### Candidate {candidate_index}", prompt, ""]) - task = { - "slot": slot, - "style": style, - "path": candidate_path, - "asset_mode": asset_mode, - "candidate_index": candidate_index, - "prompt": prompt, - "asset_retries": asset_retries, - } - tasks.append(task) - ordered_slot_candidates[slot["id"]].append(task) - - reports_by_key: dict[tuple[str, int], dict] = {} - if asset_workers == 1 or len(tasks) <= 1: - for task in tasks: - report = _generate_and_score_candidate(task) - reports_by_key[(task["slot"]["id"], task["candidate_index"])] = report - else: - with ThreadPoolExecutor(max_workers=asset_workers) as executor: - futures = {executor.submit(_generate_and_score_candidate, task): task for task in tasks} - for future in as_completed(futures): - task = futures[future] - reports_by_key[(task["slot"]["id"], task["candidate_index"])] = future.result() - - for slot in program["slots"]: - candidate_reports = [ - reports_by_key[(slot["id"], candidate_index)] - for candidate_index in range(1, candidates_per_slot + 1) - ] - successful_reports = [item for item in candidate_reports if Path(item.get("path", "")).exists() and "generation_failed" not in item.get("issue_tags", [])] - all_candidate_paths.extend(Path(item["path"]) for item in successful_reports) - if not successful_reports: - errors = "; ".join(item.get("error", "unknown generation failure") for item in candidate_reports) - raise RuntimeError(f"All candidate generations failed for slot {slot['id']}: {errors}") - selected = _select_candidate(candidate_reports) - selected_path = Path(selected["path"]) - final_asset_path = assets_dir / f"{slot['asset_id']}.png" - shutil.copyfile(selected_path, final_asset_path) - selected_paths.append(final_asset_path) - - selected_action = "ok_no_crop" if not selected.get("issue_tags") else "selected_best_but_needs_regeneration" - selected_reason = ( - "Selected because it had the best combined fill, margin, ratio, and visual-complexity score." - if not selected.get("issue_tags") - else "Selected as the least-bad candidate but validation should fail until regenerated." - ) - report_items.append({ - "asset_id": slot["asset_id"], - "slot_id": slot["id"], - "path": str(final_asset_path), - "selected_candidate_index": selected["candidate_index"], - "selected_candidate_path": selected["path"], - "source": selected["source"], - "content_fill_percent": selected["content_fill_percent"], - "min_content_fill_percent": slot["min_content_fill_percent"], - "empty_margin_percent": selected["empty_margin_percent"], - "max_empty_margin_percent": slot["max_empty_margin_percent"], - "ratio_mismatch_percent": selected["ratio_mismatch_percent"], - "edge_cutoff_status": selected["edge_cutoff_status"], - "ratio_status": selected["ratio_status"], - "issue_tags": selected.get("issue_tags", []), - "action": selected_action, - "detail_score": selected.get("detail_score"), - "detail_score_target": selected.get("detail_score_target"), - "object_count_estimate": selected.get("object_count_estimate"), - "object_count_target": selected.get("object_count_target"), - "simple_icon_risk": selected.get("simple_icon_risk"), - "reference_crop_match": selected.get("reference_crop_match"), - "style_match": selected.get("style_match"), - "complexity_issue_tags": selected.get("complexity_issue_tags", []), - "selected_reason": selected_reason, - "notes": "Selected from candidates; no semantic crop used; inserted into PPT with contain-fit.", - "rejected_candidates": [item for item in candidate_reports if item["candidate_index"] != selected["candidate_index"]], - }) - complexity_items.append({ - "asset_id": slot["asset_id"], - "slot_id": slot["id"], - "selected_candidate_index": selected["candidate_index"], - "selected_candidate_path": selected["path"], - "complexity_kind": selected.get("complexity_kind") or slot.get("complexity_kind"), - "required_visual_complexity": selected.get("required_visual_complexity") or slot.get("required_visual_complexity"), - "detail_score": selected.get("detail_score"), - "detail_score_target": selected.get("detail_score_target"), - "object_count_estimate": selected.get("object_count_estimate"), - "object_count_target": selected.get("object_count_target"), - "simple_icon_risk": selected.get("simple_icon_risk"), - "reference_crop_match": selected.get("reference_crop_match"), - "style_match": selected.get("style_match"), - "complexity_issue_tags": selected.get("complexity_issue_tags", []), - "selected_reason": selected_reason, - "candidate_complexity": [ - { - "candidate_index": item.get("candidate_index"), - "path": item.get("path"), - "detail_score": item.get("detail_score"), - "object_count_estimate": item.get("object_count_estimate"), - "simple_icon_risk": item.get("simple_icon_risk"), - "reference_crop_match": item.get("reference_crop_match"), - "style_match": item.get("style_match"), - "complexity_issue_tags": item.get("complexity_issue_tags", []), - "selection_score": item.get("selection_score"), - } - for item in candidate_reports - ], - }) - - write_text(out / "prompts.md", "\n".join(prompt_lines)) - report = { - "summary": "Asset quality report for multi-candidate slot-level generated blocks.", - "candidate_policy": { - "candidates_per_slot": candidates_per_slot, - "selection": "prefer passing candidates; otherwise choose highest fill/margin/ratio score and fail validation if below thresholds", - "no_crop": True, - }, - "assets": report_items, - } - write_json(out / "asset_quality_report.json", report) - complexity_report = { - "summary": "Asset complexity report for selected slot-level generated blocks.", - "policy": "Reject normal non-legend assets that look like sparse generic icons, ignore the local crop, or lack layered visual detail.", - "assets": complexity_items, - } - write_json(out / "asset_complexity_report.json", complexity_report) - _make_contact_sheet(selected_paths, out / "asset_contact_sheet.png", title="selected slot assets") - _make_contact_sheet(all_candidate_paths, out / "asset_candidate_contact_sheet.png", title="all generated candidates") - return report +_sys.modules[__name__] = _implementation diff --git a/rfs/asset_reviewer.py b/rfs/asset_reviewer.py index b553f90..cf14073 100644 --- a/rfs/asset_reviewer.py +++ b/rfs/asset_reviewer.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import hashlib import json import os import re @@ -12,6 +13,23 @@ from .utils import write_json +ASSET_REVIEW_CONTRACT_VERSION = "vlm-asset-review-v2" + + +def _critic_model_name(model: str | None = None) -> str: + return model or os.getenv("RFS_CRITIC_MODEL") or os.getenv("MODEL_VLM") or "gemini-3-pro-preview-thinking" + + +def _sha256_file(path: Path) -> str | None: + if not path.exists() or not path.is_file(): + return None + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def _extract_json(text: str) -> dict: cleaned = text.strip().replace("```json", "```") if cleaned.startswith("```"): @@ -29,7 +47,7 @@ def _extract_json(text: str) -> dict: def _call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None = None) -> dict: api_base = os.getenv("API_BASE", "").rstrip("/") api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") - model_name = model or os.getenv("RFS_CRITIC_MODEL") or os.getenv("MODEL_VLM") or "gemini-3-pro-preview-thinking" + model_name = _critic_model_name(model) if not api_base or not api_key: raise RuntimeError("VLM review requires API_BASE and API_KEY/GEMINI_API_KEY environment variables") @@ -69,6 +87,20 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non for item in complexity_report.get("assets", []) if isinstance(item, dict) and item.get("slot_id") } + prompt_plan_by_slot = {} + try: + prompt_plan = json.loads((out / "slot_prompt_plan.json").read_text(encoding="utf-8")) + prompt_plan_by_slot = { + str(item.get("slot_id")): item + for item in prompt_plan.get("slots", []) + if isinstance(item, dict) and item.get("slot_id") + } + except Exception: + prompt_plan_by_slot = {} + paper_native = any( + str(item.get("reference_content_priority") or "").casefold() == "paper_semantic_contract" + for item in prompt_plan_by_slot.values() + ) if mode == "off": review = {"summary": "Asset visual review skipped by configuration.", "mode": "off", "issues": []} @@ -106,6 +138,10 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non "detail_score": complexity_by_slot.get(str(item.get("slot_id")), {}).get("detail_score"), "simple_icon_risk": complexity_by_slot.get(str(item.get("slot_id")), {}).get("simple_icon_risk"), "complexity_issue_tags": complexity_by_slot.get(str(item.get("slot_id")), {}).get("complexity_issue_tags"), + "paper_concept": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("paper_concept"), + "must_show": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("must_show"), + "avoid_showing": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("avoid_showing"), + "semantic_content_authority": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("reference_content_priority"), } for item in report.get("assets", []) ] @@ -117,7 +153,7 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non Check for these issues: - too_simple: looks like a generic sparse icon instead of a rich research visual block - generic_icon: looks like a standalone pictogram rather than a reference-guided mini scientific scene/card -- reference_crop_ignored: does not visually resemble the local reference crop object +- {"paper_semantic_mismatch: visual content does not plausibly represent the stated paper concept or must_show items" if paper_native else "reference_crop_ignored: does not visually resemble the local reference crop object"} - single_object_on_blank_background: one object floats on a plain canvas - style_drift: local color/style does not match the reference figure - too_sparse: too much empty canvas or subject too small @@ -126,6 +162,8 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non - style_drift: style inconsistent across blocks - low_resolution: blur or artifacts +{"Paper-native policy: generated seed crops contain no scientific objects and must not be used for content comparison. Judge each asset against paper_concept, must_show, and avoid_showing. Do not penalize the collection merely for containing the mandatory 25-50 raster slot assets." if paper_native else "Reference-primary policy: compare visual objects and local style against the user reference crop contract."} + Return schema: {{ "summary": "...", @@ -138,9 +176,41 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non Asset list: {json.dumps(assets_brief, ensure_ascii=False)} """.strip() + fingerprint_assets = [] + for item in report.get("assets", []): + asset_path = Path(str(item.get("path") or "")) + if not asset_path.exists() and item.get("asset_id"): + asset_path = out / "assets" / f"{item.get('asset_id')}.png" + fingerprint_assets.append({ + "asset_id": item.get("asset_id"), + "slot_id": item.get("slot_id"), + "sha256": _sha256_file(asset_path), + "brief": next((entry for entry in assets_brief if entry.get("slot_id") == item.get("slot_id")), None), + }) + review_fingerprint = hashlib.sha256(json.dumps({ + "contract_version": ASSET_REVIEW_CONTRACT_VERSION, + "model": _critic_model_name(model), + "paper_native": paper_native, + "assets": fingerprint_assets, + }, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest() + existing_path = out / "asset_visual_review.json" + if existing_path.exists(): + try: + existing = json.loads(existing_path.read_text(encoding="utf-8")) + if existing.get("status") == "pass" and existing.get("fingerprint") == review_fingerprint: + review = dict(existing) + review["cache_hit"] = True + write_json(existing_path, review) + return review + except Exception: + pass review = _call_vlm_json(prompt, [contact_sheet], model=model) review.setdefault("summary", "VLM asset visual review for selected slot blocks.") review.setdefault("mode", "vlm") + review.setdefault("status", "needs_regeneration" if review.get("issues") else "pass") + review["model"] = _critic_model_name(model) + review["fingerprint"] = review_fingerprint + review["cache_hit"] = False else: raise ValueError(f"Unsupported asset review mode: {mode}") diff --git a/rfs/cli.py b/rfs/cli.py index 7839330..26a741f 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -1,32 +1,59 @@ -from __future__ import annotations +from __future__ import annotations import argparse +import importlib.util import json import os +import shutil +import subprocess import sys from pathlib import Path from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution -from .editable_rebuild import rebuild_editable -from .paper_to_image import run_paper_to_image -from .professional_rebuild import rebuild_editable_pro -from .professional_repair import vlm_professional_repair_adapter +from .paper_to_image import analyze_reference_figures, inspect_paper, learn_reference_style, record_paper_image_preference, run_fast_framework_prompt, run_paper_to_image, select_reference_exemplars +from .evaluation import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, run_pdf_extraction_stress_suite, score_benchmark_case, validate_benchmark_case +from .workflows import rebuild_editable, rebuild_editable_pro, run_paper_to_editable +from .professional.repair import vlm_professional_repair_adapter from .presentations_qa import run_presentations_qa -from .rebuild_vlm_adapters import build_rebuild_vlm_adapters +from .providers import build_rebuild_vlm_adapters from .rebuild_eval import evaluate_rebuild_vlm +from .rebuild_evolution import run_rebuild_evolution +from .rebuild_judge import run_rebuild_judge +from .analysis.rebuild.reference_text import _rapidocr_detector_limit from .utils import env_present, mask_secret -from .validator import validate_output +from .evaluation import validate_output from .workflow import make_framework +from .cli_commands.rebuild import add_rebuild_args, build_rebuild_kwargs, register_professional_rebuild, register_rebuild_core_commands +from .cli_commands.categories import EXPERIMENTAL_COMMANDS, MAINTENANCE_COMMANDS, PRODUCT_COMMANDS +from .cli_commands.maintenance import register_quality_commands, register_reference_and_inspection_commands +from .cli_commands.experimental import register_coevolution_commands, register_judge_command, register_judge_data_commands +from .judge_training import build_dataset, validate_dataset, register_checkpoint, evaluate_predictions, compare_shadow_predictions +from .cli_commands.product import register_fast_framework_prompt, register_make_framework, register_paper_to_editable, register_paper_to_image def _json_print(data: dict) -> None: print(json.dumps(data, indent=2, ensure_ascii=False)) +def _probe_executable(name: str, version_arg: str = "-v") -> dict: + path = shutil.which(name) + if not path: + return {"available": False, "path": None, "error": "not found on PATH"} + try: + completed = subprocess.run([path, version_arg], capture_output=True, text=True, timeout=5, check=False) + except Exception as exc: + return {"available": False, "path": path, "error": str(exc)} + output = "\n".join(part.strip() for part in (completed.stdout, completed.stderr) if part and part.strip()) + result = {"available": completed.returncode == 0, "path": path, "returncode": completed.returncode} + if completed.returncode != 0: + result["error"] = output.splitlines()[0] if output else "version probe failed" + return result + + def _doctor() -> dict: deps = {} - for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: + for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("pypdf", "pypdf"), ("pdfplumber", "pdfplumber"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: try: __import__(module) deps[name] = {"available": True} @@ -59,6 +86,34 @@ def _doctor() -> dict: "RFS_IMAGE_EDIT_URL": {"present": env_present("RFS_IMAGE_EDIT_URL"), "value": os.getenv("RFS_IMAGE_EDIT_URL") if env_present("RFS_IMAGE_EDIT_URL") else None}, "MODEL_VLM": {"present": env_present("MODEL_VLM"), "value": os.getenv("MODEL_VLM") if env_present("MODEL_VLM") else None}, } + easyocr_model_dir = Path(os.getenv("EASYOCR_MODULE_PATH", "").strip() or (Path.home() / ".EasyOCR" / "model")) + easyocr_files = {path.name: path.stat().st_size for path in easyocr_model_dir.glob("*.pth")} if easyocr_model_dir.exists() else {} + easyocr_models = { + "model_dir": str(easyocr_model_dir), + "detector_ready": "craft_mlt_25k.pth" in easyocr_files, + "en_ready": "english_g2.pth" in easyocr_files, + "ch_ready": "zh_sim_g2.pth" in easyocr_files, + "en_ch_ready": "craft_mlt_25k.pth" in easyocr_files and "zh_sim_g2.pth" in easyocr_files, + "allow_download": str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"}, + "files": easyocr_files, + } + optional_pdf = { + "pdftotext": _probe_executable("pdftotext"), + "pdftoppm": _probe_executable("pdftoppm"), + "easyocr": {"available": importlib.util.find_spec("easyocr") is not None, "models": easyocr_models}, + "rapidocr": { + "available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, + "runtime": "onnxruntime", + "worker_policy": "up to 4 page workers on 8+ CPUs, 2 on 4-7 CPUs, 1 below 4; deadline runs use killable worker processes; 2 intra-op threads for a single page", + "worker_override": os.getenv("RFS_OCR_WORKERS"), + "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), + "detector_limit_override": os.getenv("RFS_RAPIDOCR_DET_LIMIT"), + "effective_detector_limit": _rapidocr_detector_limit(), + }, + "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, + "wordninja": {"available": importlib.util.find_spec("wordninja") is not None, "purpose": "conservative English spacing repair for OCR lines"}, + "fast_contract_cache": {"available": True, "path": str(Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")))}, + } ok = all(item["available"] for item in deps.values()) return { "summary": "ResearchFigureStudio doctor report.", @@ -66,6 +121,7 @@ def _doctor() -> dict: "version": __version__, "python": sys.executable, "dependencies": deps, + "pdf_tools": optional_pdf, "powerpoint": powerpnt, "auth": auth, "notes": [ @@ -79,158 +135,92 @@ def _doctor() -> dict: "Use rfs presentations-qa as an optional inspection pass; RFS remains the authoritative PPTX compiler.", "Use rfs coevolve-image to run whole-image Creator Agent and Online/Frozen Judge refinement before PPTX conversion.", "Use rfs paper-to-image for evidence-grounded paper summarization, whole-image prompt compilation, candidate generation, and selected_image.png without PPTX.", + "Use rfs fast-framework-prompt for the one-call, cached paper-to-contract fast path; RFS_FAST_FRAMEWORK_MODEL defaults to gemini-2.5-flash.", "Production paper-to-image uses a content-free template blueprint and requires an Image2 edit endpoint; placeholder output is engineering-only.", ], } +def _add_rebuild_args(parser: argparse.ArgumentParser) -> None: + """Shared rebuild-editable arguments, also reused by rebuild-evolve.""" + return add_rebuild_args(parser) + """ + parser.add_argument("--reference", required=True, help="Reference image path.") + parser.add_argument("--out", required=True, help="Output directory.") + parser.add_argument("--asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api", help="Slot asset source. Policy follows generation-plan per-slot decisions.") + parser.add_argument("--asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") + parser.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") + parser.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") + parser.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") + parser.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") + parser.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") + parser.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") + parser.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM layout disagreements.") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM control disagreements.") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for visible controls missed after adjudication.") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for missed layout objects and controls.") + parser.add_argument("--vlm-timeout", type=int, help="Timeout in seconds for rebuild VLM calls. Defaults to RFS_VLM_TIMEOUT or 240.") + parser.add_argument("--vlm-retries", type=int, help="Retry count for rebuild VLM calls. Defaults to RFS_VLM_RETRIES or 2.") + parser.add_argument("--rebuild-critic-mode", choices=["off", "heuristic", "vlm"], default="off", help="Compiled-preview rebuild critic mode. Default: off.") + parser.add_argument("--rebuild-critic-iterations", type=int, default=0, help="Compiled-preview critic correction iterations, clamped to 0-3. Default: 0.") + parser.add_argument("--rebuild-critic-model", help="Optional VLM model for --rebuild-critic-mode vlm. Defaults to RFS_REBUILD_CRITIC_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") + parser.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") + parser.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") + parser.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") + parser.add_argument("--from-stage", help="Re-run from this stage onward (design_plan/build_program/localize_controls/text_layer/slot_semantics); stages before it reuse snapshots.") + parser.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle; easyocr remains available.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + """ + + def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="rfs", description="ResearchFigureStudio CLI") + parser = argparse.ArgumentParser( + prog="rfs", + description="ResearchFigureStudio CLI", + epilog=( + f"Product: {', '.join(PRODUCT_COMMANDS)}. " + f"Experimental: {', '.join(EXPERIMENTAL_COMMANDS)}. " + f"Maintenance/QA: {', '.join(MAINTENANCE_COMMANDS)}." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.") sub = parser.add_subparsers(dest="command", required=True) - doctor = sub.add_parser("doctor", help="Check dependencies, PowerPoint, and auth env vars.") - doctor.add_argument("--json", action="store_true", help="Emit JSON.") - - make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") - make.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") - make.add_argument("--reference", required=True, help="User-provided visual reference image path.") - make.add_argument("--out", required=True, help="Output directory.") - make.add_argument("--profile", default="ai-ml-paper", help="Figure profile. Default: ai-ml-paper.") - make.add_argument("--slot-count", type=int, default=36, help="Target slot count, clamped to 25-50. Default: 36.") - make.add_argument("--slot-source", choices=["paper", "reference-primary"], default="reference-primary", help="Slot content source. Default reference-primary makes the reference figure drive visual objects, layout, color, and flow logic.") - make.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset generation mode. Default image2 uses Yunwu Images API.") - make.add_argument("--candidates-per-slot", type=int, default=4, help="Candidate images per slot, clamped to 1-5. Default: 4.") - make.add_argument("--asset-workers", type=int, default=1, help="Parallel asset generation workers, clamped to 1-12. Default: 1.") - make.add_argument("--asset-retries", type=int, default=2, help="Retries per generated asset candidate, clamped to 0-5. Default: 2.") - make.add_argument("--asset-review-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Selected asset review mode. Default: heuristic.") - make.add_argument("--locator-mode", choices=["heuristic", "vlm"], default="heuristic", help="Layout coordinate source. Use vlm to locate slots from the reference image.") - make.add_argument("--locator-model", help="Optional VLM model for --locator-mode vlm. Defaults to RFS_LOCATOR_MODEL/MODEL_VLM.") - make.add_argument("--control-localizer-mode", choices=["off", "heuristic", "hybrid"], default="hybrid", help="Arrow/connector localization mode. Default hybrid uses CV candidates plus optional VLM binding; falls back to heuristic without API keys.") - make.add_argument("--arrow-style-mode", choices=["off", "reference", "aesthetic"], default="reference", help="Arrow styling/routing mode. Default reference preserves reference-image routes and adds softer PPT styling/QA metadata.") - make.add_argument("--prompt-plan-mode", choices=["heuristic", "vlm", "vlm-batch"], default="vlm", help="Reference-aware per-slot prompt planning mode. Default vlm uses one VLM call per slot; vlm-batch uses one batch VLM call; heuristic is offline only.") - make.add_argument("--prompt-plan-model", help="Optional VLM model for --prompt-plan-mode vlm/vlm-batch. Defaults to RFS_PROMPT_PLANNER_MODEL/MODEL_VLM.") - make.add_argument("--prompt-plan-workers", type=int, default=4, help="Parallel VLM workers for per-slot prompt planning, clamped to 1-12. Default: 4.") - make.add_argument("--complexity-profile", choices=["reference-dense", "balanced", "legend-simple"], default="reference-dense", help="Visual complexity policy for slot_visual_spec.json. Default reference-dense.") - make.add_argument("--critic-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Final reference-vs-render critic mode. Default: heuristic.") - make.add_argument("--critic-model", help="Optional VLM model for asset review and final critic. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") - make.add_argument("--critic-iterations", type=int, default=0, help="VLM layout correction iterations, clamped to 0-3. Default: 0.") - make.add_argument("--text-extractor-mode", choices=["heuristic", "ocr"], default="ocr", help="Editable text layer source. Default ocr uses local OCR when available and falls back to heuristic.") - make.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle.") - make.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - make.add_argument("--presentations-qa", action="store_true", help="Run optional Presentations plugin import/render/layout QA after export. This never mutates the PPTX.") - make.add_argument("--presentations-workspace", help="Optional workspace for Presentations QA scratch artifacts.") - make.add_argument("--presentations-scale", type=int, default=2, help="Preview render scale for Presentations QA. Default: 2.") - make.add_argument("--no-export", action="store_true", help="Skip PDF/PNG export and only create PPTX/artifacts.") - make.add_argument("--json", action="store_true", help="Emit JSON.") - - paper_image = sub.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate raster image candidates without PPTX.") - paper_image.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") - paper_image.add_argument("--out", required=True, help="Output directory for review, templates, prompts, candidates, and production-only selected_image.png.") - paper_image.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") - paper_image.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") - paper_image.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") - paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") - paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") - paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") - paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") - paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") - paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") - paper_image.add_argument("--language", default="English", help="Visible label language. Default: English.") - paper_image.add_argument("--image-model", help="Optional image model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") - paper_image.add_argument("--image-retries", type=int, default=2, help="Retries per image candidate, clamped to 0-5. Default: 2.") - paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") - paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") - paper_image.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") - paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - paper_image.add_argument("--json", action="store_true", help="Emit JSON.") - - rebuild = sub.add_parser("rebuild-editable", help="Rebuild a reference image into a reusable editable PowerPoint composition.") - rebuild.add_argument("--reference", required=True, help="Reference image path.") - rebuild.add_argument("--out", required=True, help="Output directory.") - rebuild.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") - rebuild.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") - rebuild.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") - rebuild.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") - rebuild.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") - rebuild.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") - rebuild.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") - rebuild.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") - rebuild.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") - rebuild.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") - rebuild.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") - rebuild.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") - rebuild.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") - rebuild.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") - rebuild.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - rebuild.add_argument("--json", action="store_true", help="Emit JSON.") - - rebuild_eval = sub.add_parser("rebuild-editable-eval", help="Compare heuristic and hybrid VLM rebuild-editable outputs for one reference image.") - rebuild_eval.add_argument("--reference", required=True, help="Reference image path.") - rebuild_eval.add_argument("--out", required=True, help="Evaluation output directory.") - rebuild_eval.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="crop", help="Asset mode for both cases. Default: crop to avoid image generation cost.") - rebuild_eval.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Text extraction mode for both cases. Default: ocr.") - rebuild_eval.add_argument("--export-preview", action="store_true", help="Export PNG previews when PowerPoint is available.") - rebuild_eval.add_argument("--json", action="store_true", help="Emit JSON.") - - rebuild_pro = sub.add_parser("rebuild-editable-pro", help="Rebuild a reference image through a VLM-authored controlled professional DSL.") - rebuild_pro.add_argument("--reference", required=True, help="Reference image path.") - rebuild_pro.add_argument("--out", required=True, help="Output directory.") - rebuild_pro.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") - rebuild_pro.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") - rebuild_pro.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") - rebuild_pro.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") - rebuild_pro.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") - rebuild_pro.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") - rebuild_pro.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Baseline layout extraction mode before professional DSL planning. Default: hybrid.") - rebuild_pro.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Baseline control extraction mode before professional DSL planning. Default: hybrid.") - rebuild_pro.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") - rebuild_pro.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") - rebuild_pro.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") - rebuild_pro.add_argument("--compile-only", action="store_true", help="Compile from existing professional_rebuild_script.dsl.json without rerunning analysis or VLM planning.") - rebuild_pro.add_argument("--repair-rounds", type=int, default=2, help="Preview repair rounds to record/run. V1 records conservative no-mutation repair reports by default.") - rebuild_pro.add_argument("--repair-mode", choices=["report", "vlm"], default="report", help="Professional repair mode. report records rounds without mutation; vlm applies controlled DSL patches.") - rebuild_pro.add_argument("--benchmark-out", help="Optional specialized rebuild output directory to compare against in professional_gap_report.json.") - rebuild_pro.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") - rebuild_pro.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - rebuild_pro.add_argument("--json", action="store_true", help="Emit JSON.") - - coevolve = sub.add_parser("coevolve-image", help="Refine a complete scientific image through Creator Agent and Online/Frozen Judges.") - coevolve.add_argument("--ground-truth", required=True, help="Structured Ground Truth JSON containing paper facts and human aesthetic preferences.") - coevolve.add_argument("--out", required=True, help="Output directory for rounds, training trajectories, and approved_image.png.") - coevolve.add_argument("--candidates", type=int, default=3, help="Initial whole-image candidate count. Default: 3.") - coevolve.add_argument("--repair-candidates", type=int, default=2, help="Candidate count for each repair round. Default: 2.") - coevolve.add_argument("--max-rounds", type=int, default=4, help="Maximum total rounds including initial generation. Default: 4.") - coevolve.add_argument("--online-judge-model", help="Online feedback Judge model. Defaults to RFS_ONLINE_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - coevolve.add_argument("--frozen-judge-model", help="Independent acceptance Judge model. Defaults to RFS_FROZEN_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - coevolve.add_argument("--image-model", help="Whole-image generation model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") - coevolve.add_argument("--image-retries", type=int, default=2, help="Image request retries. Default: 2.") - coevolve.add_argument("--json", action="store_true", help="Emit JSON.") - - coevolution_report = sub.add_parser("coevolution-report", help="Aggregate a co-evolution run into reproducible improvement metrics.") - coevolution_report.add_argument("--run", required=True, help="Existing co-evolution output directory.") - coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") - - validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") - validate.add_argument("--out", required=True, help="Output directory to validate.") - validate.add_argument("--json", action="store_true", help="Emit JSON.") - - presentations_qa = sub.add_parser("presentations-qa", help="Run optional Presentations plugin QA for an existing RFS output directory.") - presentations_qa.add_argument("--out", required=True, help="RFS output directory containing editable_composition.pptx.") - presentations_qa.add_argument("--pptx", help="Optional PPTX path. Defaults to /editable_composition.pptx.") - presentations_qa.add_argument("--workspace", help="Optional Presentations QA workspace. Defaults to /presentations_plugin_qa_workspace.") - presentations_qa.add_argument("--scale", type=int, default=2, help="Preview render scale. Default: 2.") - presentations_qa.add_argument("--skip-inspect", action="store_true", help="Write report from PPTX/package metadata without invoking the Presentations plugin.") - presentations_qa.add_argument("--json", action="store_true", help="Emit JSON.") + register_reference_and_inspection_commands(sub) + + register_fast_framework_prompt(sub) + + register_make_framework(sub) + + register_paper_to_image(sub) + + register_paper_to_editable(sub) + + register_rebuild_core_commands(sub) + + register_professional_rebuild(sub) + + register_coevolution_commands(sub) + + register_quality_commands(sub) + register_judge_command(sub) + register_judge_data_commands(sub) return parser def _print_human(data: dict) -> None: if "ok" in data: print(f"ok: {data['ok']}") - for key in ["out_dir", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "control_mode", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: + for key in ["out_dir", "records_path", "library_path", "report_path", "record_path", "selection_path", "brief_path", "style_reference", "user_style_brief", "combined_style_brief", "run_path", "chosen_candidate_id", "chosen_path", "preference_pairs_path", "feedback_path", "pair_count", "appended_pair_count", "record_count", "positive_count", "negative_count", "input_count", "scanned_file_count", "skipped_count", "failure_count", "analysis_mode", "model", "reference_library", "reference_top_k", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_policy", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "layout_adjudication_mode", "layout_adjudication_status", "control_mode", "control_adjudication_mode", "control_adjudication_status", "control_coverage_audit_mode", "control_coverage_audit_status", "combined_coverage_audit_mode", "combined_coverage_audit_status", "combined_missing_layout_count", "combined_missing_control_count", "vlm_timeout", "vlm_retries", "rebuild_critic_mode", "rebuild_critic_iterations", "rebuild_critic_status", "rebuild_critic_model", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: if key in data: print(f"{key}: {data[key]}") if data.get("presentations_qa"): @@ -251,6 +241,11 @@ def _print_human(data: dict) -> None: print(f"warning: {warn}") +def _rebuild_kwargs(args) -> dict: + """Compatibility helper for callers importing the historical CLI symbol.""" + return build_rebuild_kwargs(args, build_rebuild_vlm_adapters) + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) json_requested = False @@ -264,6 +259,62 @@ def main(argv: list[str] | None = None) -> int: try: if args.command == "doctor": result = _doctor() + elif args.command == "analyze-reference-figures": + result = analyze_reference_figures( + references=args.references, + out=args.out, + mode=args.mode, + model=args.model, + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + ) + elif args.command == "select-reference-exemplars": + result = select_reference_exemplars( + paper_review=args.paper_review, + library=args.library, + out=args.out, + selected_template=args.selected_template, + top_k=args.top_k, + ) + elif args.command == "learn-reference-style": + result = learn_reference_style( + style_reference=args.style_reference, + out=args.out, + mode=args.mode, + model=args.model, + ) + elif args.command == "record-paper-image-preference": + result = record_paper_image_preference( + run=args.run, + chosen=args.chosen, + reason=args.reason, + reviewer=args.reviewer, + tags=args.tags, + ) + elif args.command == "inspect-pdf": + result = inspect_paper( + paper=args.paper, + out=args.out, + deadline_seconds=args.deadline, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + ) + elif args.command == "fast-framework-prompt": + result = run_fast_framework_prompt( + paper=args.paper, + out=args.out, + deadline_seconds=args.deadline, + planner_mode=args.planner_mode, + planner_model=args.planner_model, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + preferences_path=args.preferences, + aspect_ratio=args.aspect_ratio, + language=args.language, + domain_profile=args.domain_profile, + editable_ppt=args.editable_ppt, + verify_ppt_roundtrip=args.verify_ppt_roundtrip, + ) elif args.command == "make-framework": result = make_framework( paper=args.paper, @@ -295,6 +346,8 @@ def main(argv: list[str] | None = None) -> int: presentations_workspace=args.presentations_workspace, presentations_scale=args.presentations_scale, export=not args.no_export, + semantic_contract=args.semantic_contract, + paper_contract_dir=args.paper_contract_dir, ) elif args.command == "paper-to-image": result = run_paper_to_image( @@ -303,6 +356,10 @@ def main(argv: list[str] | None = None) -> int: preferences_path=args.preferences, positive_references=args.positive_reference, negative_references=args.negative_reference, + reference_library=args.reference_library, + reference_top_k=args.reference_top_k, + user_style_brief=args.user_style_brief, + preference_history=args.preference_history, planner_mode=args.planner_mode, planner_model=args.planner_model, asset_mode=args.asset_mode, @@ -316,31 +373,73 @@ def main(argv: list[str] | None = None) -> int: domain_profile=args.domain_profile, template=args.template, repair_rounds=args.repair_rounds, + human_scientific_confirmed=args.human_scientific_confirmed, + repair_source=args.repair_source, + resume_candidates=args.resume_candidates, + editable_overlay_ppt=args.editable_overlay_ppt, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, ) - elif args.command == "rebuild-editable": - rebuild_adapters = build_rebuild_vlm_adapters(args.out) - result = rebuild_editable( - reference=args.reference, + elif args.command == "paper-to-editable": + result = run_paper_to_editable( + paper=args.paper, out=args.out, - asset_mode=args.asset_mode, - asset_workers=args.asset_workers, - asset_retries=args.asset_retries, - economy_mode=args.economy_mode, - text_mode=args.text_mode, - control_mode=args.control_mode, + preferences_path=args.preferences, + positive_references=args.positive_reference, + negative_references=args.negative_reference, + planner_mode=args.planner_mode, + planner_model=args.planner_model, + image_asset_mode=args.image_asset_mode, + image_candidates=args.image_candidates, + aspect_ratio=args.aspect_ratio, + language=args.language, + image_model=args.image_model, + image_retries=args.image_retries, + review_mode=args.review_mode, + review_model=args.review_model, + domain_profile=args.domain_profile, + template=args.template, + repair_rounds=args.repair_rounds, + human_scientific_confirmed=args.human_scientific_confirmed, + rebuild_asset_mode=args.rebuild_asset_mode, + rebuild_asset_policy=args.rebuild_asset_policy, layout_mode=args.layout_mode, - export_preview=args.export_preview, - regenerate_slots=args.regenerate_slots, - strict_asset_regeneration=args.strict_asset_regeneration, - skip_analysis=args.skip_analysis, - compile_only=args.compile_only, + control_mode=args.control_mode, + layout_adjudication_mode=args.layout_adjudication_mode if args.layout_mode in {"vlm", "hybrid"} else "off", + control_adjudication_mode=args.control_adjudication_mode if args.control_mode in {"vlm", "hybrid"} else "off", + control_coverage_audit_mode=args.control_coverage_audit_mode if args.control_mode in {"vlm", "hybrid"} else "off", + combined_coverage_audit_mode=args.combined_coverage_audit_mode if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else "off", + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + rebuild_critic_mode=args.rebuild_critic_mode, + rebuild_critic_iterations=args.rebuild_critic_iterations, + rebuild_critic_model=args.rebuild_critic_model, + text_mode=args.text_mode, + design_plan_mode=args.design_plan_mode, + export_preview=not args.no_export_preview, + allow_engineering_preview=args.allow_engineering_preview, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, - vlm_layout_adapter=rebuild_adapters["layout"] if args.layout_mode in {"vlm", "hybrid"} else None, - control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, - semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, + editable_route=args.editable_route, + framework_slot_count=args.framework_slot_count, + framework_asset_mode=args.framework_asset_mode, + framework_candidates_per_slot=args.framework_candidates_per_slot, + framework_asset_workers=args.framework_asset_workers, + ) + elif args.command == "rebuild-editable": + result = rebuild_editable( + reference=args.reference, + out=args.out, + **_rebuild_kwargs(args), + ) + elif args.command == "rebuild-evolve": + result = run_rebuild_evolution( + reference=args.reference, + out=args.out, + max_rounds=args.max_rounds, + judge_model=args.judge_model, + score_threshold=args.score_threshold, + **_rebuild_kwargs(args), ) elif args.command == "rebuild-editable-eval": result = evaluate_rebuild_vlm( @@ -351,10 +450,25 @@ def main(argv: list[str] | None = None) -> int: export_preview=args.export_preview, ) elif args.command == "rebuild-editable-pro": - rebuild_adapters = build_rebuild_vlm_adapters(args.out) + baseline_out = Path(args.out) / "professional_baseline" + rebuild_adapters = build_rebuild_vlm_adapters(baseline_out, vlm_timeout=args.vlm_timeout, vlm_retries=args.vlm_retries) + use_layout_vlm = args.layout_mode in {"vlm", "hybrid"} + use_control_vlm = args.control_mode in {"vlm", "hybrid"} + use_any_rebuild_vlm = use_layout_vlm or use_control_vlm + repair_adapter = None + if args.repair_mode == "vlm": + repair_adapter = lambda reference_path, preview_path, dsl, round_index: vlm_professional_repair_adapter( + reference_path, + preview_path, + dsl, + round_index, + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + ) result = rebuild_editable_pro( reference=args.reference, out=args.out, + baseline_out=baseline_out, asset_mode=args.asset_mode, asset_workers=args.asset_workers, asset_retries=args.asset_retries, @@ -369,11 +483,28 @@ def main(argv: list[str] | None = None) -> int: repair_rounds=args.repair_rounds, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, - vlm_layout_adapter=rebuild_adapters["layout"] if args.layout_mode in {"vlm", "hybrid"} else None, - control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, - semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, - repair_adapter=vlm_professional_repair_adapter if args.repair_mode == "vlm" else None, + vlm_layout_adapter=rebuild_adapters["layout"] if use_layout_vlm else None, + panel_layout_adapter=rebuild_adapters.get("panel_layout") if use_layout_vlm else None, + layout_adjudication_mode=args.layout_adjudication_mode if use_layout_vlm else "off", + layout_adjudicator=rebuild_adapters["layout_adjudicator"] if use_layout_vlm and args.layout_adjudication_mode == "vlm" else None, + control_adapter=rebuild_adapters["control"] if use_control_vlm else None, + control_adjudication_mode=args.control_adjudication_mode if use_control_vlm else "off", + control_adjudicator=rebuild_adapters["control_adjudicator"] if use_control_vlm and args.control_adjudication_mode == "vlm" else None, + control_coverage_audit_mode=args.control_coverage_audit_mode if use_control_vlm else "off", + coverage_audit_adapter=rebuild_adapters["coverage_audit"] if use_control_vlm and args.control_coverage_audit_mode == "vlm" else None, + combined_coverage_audit_mode=args.combined_coverage_audit_mode if use_any_rebuild_vlm else "off", + combined_coverage_audit_adapter=rebuild_adapters["combined_coverage_audit"] if use_any_rebuild_vlm and args.combined_coverage_audit_mode == "vlm" else None, + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + semantic_adapter=rebuild_adapters["semantic"] if use_any_rebuild_vlm else None, + design_plan_mode=args.design_plan_mode, + design_plan_model=args.design_plan_model, + design_adapter=rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, + text_grouping_mode=args.text_grouping_mode, + text_grouping_model=args.text_grouping_model, + repair_adapter=repair_adapter, benchmark_out=args.benchmark_out, + asset_policy=args.asset_policy, ) elif args.command == "coevolve-image": result = run_image_coevolution( @@ -389,6 +520,41 @@ def main(argv: list[str] | None = None) -> int: ) elif args.command == "coevolution-report": result = analyze_coevolution_run(args.run) + elif args.command == "benchmark": + if args.benchmark_action == "list": + result = list_benchmark_cases(args.root, suite=args.suite) + elif args.benchmark_action == "validate": + if not args.case: + parser.error("benchmark validate requires --case") + result = validate_benchmark_case(args.case) + elif args.benchmark_action == "fetch": + if not args.case: + parser.error("benchmark fetch requires --case") + result = fetch_benchmark_case(args.case, force=args.force) + elif args.benchmark_action == "run": + if not args.case or not args.out: + parser.error("benchmark run requires --case and --out") + result = run_benchmark_case(args.case, args.out) + elif args.benchmark_action == "fast": + if not args.case or not args.out: + parser.error("benchmark fast requires --case and --out") + result = run_fast_benchmark_case(args.case, args.out, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine, rasterize_pdf_dpi=args.rasterize_dpi) + elif args.benchmark_action == "fast-suite": + if not args.out: + parser.error("benchmark fast-suite requires --out") + result = run_fast_benchmark_suite(args.root, args.out, case_ids=args.case_id, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine, rasterize_pdf_dpi=args.rasterize_dpi) + elif args.benchmark_action == "pdf-suite": + if not args.out: + parser.error("benchmark pdf-suite requires --out") + result = run_pdf_extraction_stress_suite(args.out, ocr_engine=args.ocr_engine) + elif args.benchmark_action == "audit": + if not args.run_dir: + parser.error("benchmark audit requires --run; --case is optional but enables benchmark scoring") + result = audit_paper_image_run(args.run_dir, case_dir=args.case, candidate_image=args.candidate_image, review_model=args.review_model, ocr_engine=args.ocr_engine) + else: + if not args.case or not args.run_dir: + parser.error("benchmark score requires --case and --run") + result = score_benchmark_case(args.case, args.run_dir, args.out) elif args.command == "validate": result = validate_output(args.out) elif args.command == "presentations-qa": @@ -400,6 +566,35 @@ def main(argv: list[str] | None = None) -> int: run_inspect=not args.skip_inspect, ) result["ok"] = result.get("presentations_plugin_qa", {}).get("status") not in {"failed"} + elif args.command == "rebuild-judge": + result = run_rebuild_judge( + out=args.out, + reference=args.reference, + preview=args.preview, + model=args.model, + timeout=args.timeout, + retries=args.retries, + repair_plan=args.repair_plan, + ) + elif args.command == "judge": + if args.judge_action == "register": + manifest = json.loads(Path(args.manifest).read_text(encoding="utf-8")) + result = register_checkpoint(args.checkpoint, manifest, args.registry) + result["ok"] = True + elif args.judge_action == "evaluate": + result = evaluate_predictions(args.dataset, args.predictions, checkpoint=args.checkpoint, suite=args.suite) + elif args.judge_action == "shadow": + result = compare_shadow_predictions(args.dataset, args.predictions, args.bootstrap_predictions, checkpoint=args.checkpoint) + else: + parser.error("judge requires register, evaluate, or shadow") + elif args.command == "judge-data": + if args.judge_data_action == "build": + result = build_dataset(args.runs, args.out, dataset_version=args.dataset_version) + result["ok"] = not bool(result.get("errors")) + elif args.judge_data_action == "validate": + result = validate_dataset(args.dataset) + else: + parser.error("judge-data requires build or validate") else: parser.error("unknown command") return 2 diff --git a/rfs/cli_commands/__init__.py b/rfs/cli_commands/__init__.py new file mode 100644 index 0000000..3bd72dc --- /dev/null +++ b/rfs/cli_commands/__init__.py @@ -0,0 +1,5 @@ +"""Command-specific CLI assembly helpers. + +The public entry point remains :mod:`rfs.cli`; this package only keeps +argument-to-workflow assembly out of the top-level parser module. +""" diff --git a/rfs/cli_commands/categories.py b/rfs/cli_commands/categories.py new file mode 100644 index 0000000..a6b91c6 --- /dev/null +++ b/rfs/cli_commands/categories.py @@ -0,0 +1,36 @@ +"""Stable CLI command classification used by help and navigation tooling.""" + +PRODUCT_COMMANDS = ( + "paper-to-image", + "paper-to-editable", + "rebuild-editable", + "make-framework", + "fast-framework-prompt", +) +EXPERIMENTAL_COMMANDS = ( + "rebuild-editable-pro", + "rebuild-evolve", + "rebuild-editable-eval", + "coevolve-image", + "coevolution-report", + "rebuild-judge", + "judge-data", + "judge", +) +MAINTENANCE_COMMANDS = ( + "doctor", + "inspect-pdf", + "benchmark", + "validate", + "presentations-qa", + "analyze-reference-figures", + "select-reference-exemplars", + "learn-reference-style", + "record-paper-image-preference", +) + +COMMAND_CATEGORIES = { + **{name: "product" for name in PRODUCT_COMMANDS}, + **{name: "experimental" for name in EXPERIMENTAL_COMMANDS}, + **{name: "maintenance" for name in MAINTENANCE_COMMANDS}, +} diff --git a/rfs/cli_commands/experimental.py b/rfs/cli_commands/experimental.py new file mode 100644 index 0000000..28734da --- /dev/null +++ b/rfs/cli_commands/experimental.py @@ -0,0 +1,65 @@ +"""Registration helpers for experimental CLI commands.""" + +from __future__ import annotations + + +def register_judge_command(subparsers) -> None: + rebuild_judge = subparsers.add_parser("rebuild-judge", help="Run the process-supervision Judge over a rebuild-editable output directory.") + rebuild_judge.add_argument("--out", required=True, help="Rebuild output directory containing run_log.jsonl and rebuild_result.json.") + rebuild_judge.add_argument("--reference", help="Optional reference image override. Defaults to rebuild_result.json.") + rebuild_judge.add_argument("--preview", help="Optional preview image override. Defaults to rebuild_result.json.") + rebuild_judge.add_argument("--model", help="Optional VLM model override.") + rebuild_judge.add_argument("--timeout", type=int, help="VLM timeout in seconds.") + rebuild_judge.add_argument("--retries", type=int, help="VLM retry count.") + rebuild_judge.add_argument("--repair-plan", action="store_true", help="Also write a v1 coarse repair plan mapping fault stages to actions.") + rebuild_judge.add_argument("--json", action="store_true", help="Emit JSON.") + judge = subparsers.add_parser("judge", help="Register and evaluate trained Judge checkpoints.") + actions = judge.add_subparsers(dest="judge_action", required=True) + register = actions.add_parser("register", help="Register a checkpoint manifest without loading weights.") + register.add_argument("--checkpoint", required=True) + register.add_argument("--manifest", required=True, help="JSON manifest containing model_id, dataset_version, roles, and git_commit.") + register.add_argument("--registry", required=True) + register.add_argument("--json", action="store_true") + evaluate = actions.add_parser("evaluate", help="Evaluate recorded predictions on a fixed dataset.") + evaluate.add_argument("--checkpoint") + evaluate.add_argument("--dataset", required=True) + evaluate.add_argument("--predictions", required=True) + evaluate.add_argument("--suite", default="hidden_test") + evaluate.add_argument("--json", action="store_true") + shadow = actions.add_parser("shadow", help="Compare trained predictions with Bootstrap predictions.") + shadow.add_argument("--checkpoint") + shadow.add_argument("--dataset", required=True) + shadow.add_argument("--predictions", required=True, help="Trained Judge prediction JSONL.") + shadow.add_argument("--bootstrap-predictions", required=True) + shadow.add_argument("--json", action="store_true") + + +def register_judge_data_commands(subparsers) -> None: + parser = subparsers.add_parser("judge-data", help="Build or validate Shadow-ready Judge training JSONL data.") + actions = parser.add_subparsers(dest="judge_data_action", required=True) + build = actions.add_parser("build", help="Build JudgeExample records from Paperimage evolution runs.") + build.add_argument("--runs", required=True, help="Run directory or root containing paper-to-editable outputs.") + build.add_argument("--out", required=True, help="Dataset directory or .jsonl output path.") + build.add_argument("--dataset-version", default="judge-dataset-v001") + build.add_argument("--json", action="store_true", help="Emit JSON.") + validate = actions.add_parser("validate", help="Validate a JudgeExample JSONL file or dataset directory.") + validate.add_argument("--dataset", required=True, help="Dataset directory or JSONL path.") + validate.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_coevolution_commands(subparsers) -> None: + parser = subparsers.add_parser("coevolve-image", help="Refine a complete scientific image through Creator Agent and Online/Frozen Judges.") + parser.add_argument("--ground-truth", required=True, help="Structured Ground Truth JSON containing paper facts and human aesthetic preferences.") + parser.add_argument("--out", required=True, help="Output directory for rounds, training trajectories, and approved_image.png.") + parser.add_argument("--candidates", type=int, default=3, help="Initial whole-image candidate count. Default: 3.") + parser.add_argument("--repair-candidates", type=int, default=2, help="Candidate count for each repair round. Default: 2.") + parser.add_argument("--max-rounds", type=int, default=4, help="Maximum total rounds including initial generation. Default: 4.") + parser.add_argument("--online-judge-model", help="Online feedback Judge model. Defaults to RFS_ONLINE_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--frozen-judge-model", help="Independent acceptance Judge model. Defaults to RFS_FROZEN_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--image-model", help="Whole-image generation model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") + parser.add_argument("--image-retries", type=int, default=2, help="Image request retries. Default: 2.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + report = subparsers.add_parser("coevolution-report", help="Aggregate a co-evolution run into reproducible improvement metrics.") + report.add_argument("--run", required=True, help="Existing co-evolution output directory.") + report.add_argument("--json", action="store_true", help="Emit JSON.") diff --git a/rfs/cli_commands/maintenance.py b/rfs/cli_commands/maintenance.py new file mode 100644 index 0000000..a3fa865 --- /dev/null +++ b/rfs/cli_commands/maintenance.py @@ -0,0 +1,80 @@ +"""Registration helpers for maintenance and reference-library commands.""" + +from __future__ import annotations + + +def register_reference_and_inspection_commands(subparsers) -> None: + doctor = subparsers.add_parser("doctor", help="Check dependencies, PowerPoint, and auth env vars.") + doctor.add_argument("--json", action="store_true", help="Emit JSON.") + + analyze_references = subparsers.add_parser("analyze-reference-figures", help="Analyze project reference figures into reusable design records.") + analyze_references.add_argument("--references", default="reference-figures", help="Reference figure root containing excellent/ and negative/. Default: reference-figures.") + analyze_references.add_argument("--out", default="output/reference_learning_stage1", help="Output directory for reference analysis JSONL, library JSONL, and report.") + analyze_references.add_argument("--mode", choices=["heuristic", "vlm"], default="heuristic", help="Reference analysis mode. Default: heuristic.") + analyze_references.add_argument("--model", default="gpt-5.6-sol", help="VLM model for --mode vlm. Default: gpt-5.6-sol.") + analyze_references.add_argument("--vlm-timeout", type=int, help="Optional VLM timeout in seconds. Defaults to RFS_VLM_TIMEOUT.") + analyze_references.add_argument("--vlm-retries", type=int, help="Optional VLM retry count. Defaults to RFS_VLM_RETRIES.") + analyze_references.add_argument("--json", action="store_true", help="Emit JSON.") + + select_exemplars = subparsers.add_parser("select-reference-exemplars", help="Select reference design exemplars for a paper review.") + select_exemplars.add_argument("--paper-review", required=True, help="paper_review.json for the current paper.") + select_exemplars.add_argument("--library", required=True, help="Reference library JSONL from analyze-reference-figures.") + select_exemplars.add_argument("--out", required=True, help="Output directory for exemplar_selection.json and exemplar_style_brief.json.") + select_exemplars.add_argument("--selected-template", help="Optional selected_template.json for archetype/aspect matching.") + select_exemplars.add_argument("--top-k", type=int, default=5, help="Positive exemplar count, clamped to 1-20. Default: 5.") + select_exemplars.add_argument("--json", action="store_true", help="Emit JSON.") + + learn_style = subparsers.add_parser("learn-reference-style", help="Learn reusable visual style preferences from one user style reference image.") + learn_style.add_argument("--style-reference", required=True, help="User-provided style reference image. Content is not reused.") + learn_style.add_argument("--out", required=True, help="Output directory for user_style_record.json, user_style_brief.json, and user_style_report.json.") + learn_style.add_argument("--mode", choices=["heuristic", "vlm"], default="vlm", help="User style learning mode. Default: vlm.") + learn_style.add_argument("--model", default="gpt-5.6-sol", help="VLM model for --mode vlm. Default: gpt-5.6-sol.") + learn_style.add_argument("--json", action="store_true", help="Emit JSON.") + + record_preference = subparsers.add_parser("record-paper-image-preference", help="Record human preference feedback for an existing paper-to-image run.") + record_preference.add_argument("--run", required=True, help="Existing paper-to-image output directory.") + record_preference.add_argument("--chosen", required=True, help="Preferred candidate_id, such as candidate_02 or repair_01.") + record_preference.add_argument("--reason", required=True, help="Why the chosen candidate is preferred.") + record_preference.add_argument("--reviewer", default="human", help="Reviewer name or role. Default: human.") + record_preference.add_argument("--tags", default="", help="Comma-separated preference tags, such as layout,hierarchy,color.") + record_preference.add_argument("--json", action="store_true", help="Emit JSON.") + + inspect_pdf = subparsers.add_parser("inspect-pdf", help="Extract and diagnose a paper without calling a model.") + inspect_pdf.add_argument("--paper", required=True, help="Paper PDF or supported document path.") + inspect_pdf.add_argument("--out", required=True, help="Output directory for the document model and extraction report.") + inspect_pdf.add_argument("--deadline", type=int, default=180, help="Soft processing deadline in seconds. Default: 180.") + inspect_pdf.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") + inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_quality_commands(subparsers) -> None: + benchmark = subparsers.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score", "audit"]) + benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") + benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") + benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") + benchmark.add_argument("--case-id", action="append", default=[], help="Optional case id filter for fast-suite. Repeat to select multiple cases.") + benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") + benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") + benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") + benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") + benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") + benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") + benchmark.add_argument("--review-model", help="Optional frozen VLM model for benchmark audit.") + benchmark.add_argument("--candidate-image", help="Optional explicit image to re-audit inside an existing paper-to-image run.") + benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") + benchmark.add_argument("--rasterize-dpi", type=int, help="For benchmark fast/fast-suite, convert local PDF inputs to image-only scanned PDFs at this DPI before extraction.") + benchmark.add_argument("--json", action="store_true", help="Emit JSON.") + + validate = subparsers.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") + validate.add_argument("--out", required=True, help="Output directory to validate.") + validate.add_argument("--json", action="store_true", help="Emit JSON.") + + presentations_qa = subparsers.add_parser("presentations-qa", help="Run optional Presentations plugin QA for an existing RFS output directory.") + presentations_qa.add_argument("--out", required=True, help="RFS output directory containing editable_composition.pptx.") + presentations_qa.add_argument("--pptx", help="Optional PPTX path. Defaults to /editable_composition.pptx.") + presentations_qa.add_argument("--workspace", help="Optional Presentations QA workspace. Defaults to /presentations_plugin_qa_workspace.") + presentations_qa.add_argument("--scale", type=int, default=2, help="Preview render scale. Default: 2.") + presentations_qa.add_argument("--skip-inspect", action="store_true", help="Write report from PPTX/package metadata without invoking the Presentations plugin.") + presentations_qa.add_argument("--json", action="store_true", help="Emit JSON.") diff --git a/rfs/cli_commands/product.py b/rfs/cli_commands/product.py new file mode 100644 index 0000000..2b3554a --- /dev/null +++ b/rfs/cli_commands/product.py @@ -0,0 +1,139 @@ +"""Registration helpers for product workflow commands.""" + +from __future__ import annotations + + +def register_fast_framework_prompt(subparsers) -> None: + parser = subparsers.add_parser("fast-framework-prompt", help="Compile a paper-grounded framework prompt without generating images.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--out", required=True, help="Output directory for evidence, semantic contract, prompt, and overlay specification.") + parser.add_argument("--deadline", type=int, default=180, help="Soft end-to-end deadline in seconds. Default: 180.") + parser.add_argument("--preferences", help="Optional JSON style and output preferences.") + parser.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + parser.add_argument("--planner-model") + parser.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + parser.add_argument("--aspect-ratio", default="16:9") + parser.add_argument("--language", default="English") + parser.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + parser.add_argument("--editable-ppt", action="store_true", help="Also compile the semantic contract into native editable PowerPoint shapes, text, and connectors.") + parser.add_argument("--verify-ppt-roundtrip", action="store_true", help="Compile the editable PPTX, render it through PowerPoint, and write ppt_roundtrip_report.json.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_make_framework(subparsers) -> None: + parser = subparsers.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--reference", required=True, help="User-provided visual reference image path.") + parser.add_argument("--out", required=True, help="Output directory.") + contract_source = parser.add_mutually_exclusive_group() + contract_source.add_argument("--paper-contract-dir", help="Fast paper-to-image/fast-framework output directory containing figure_specification.json. Bypasses the legacy paper planner.") + contract_source.add_argument("--semantic-contract", help="Semantic contract JSON or figure_specification.json. Bypasses the legacy paper planner.") + parser.add_argument("--profile", default="ai-ml-paper", help="Figure profile. Default: ai-ml-paper.") + parser.add_argument("--slot-count", type=int, default=36, help="Target slot count, clamped to 25-50. Default: 36.") + parser.add_argument("--slot-source", choices=["paper", "reference-primary"], default="reference-primary", help="Slot content source. Default reference-primary makes the reference figure drive visual objects, layout, color, and flow logic.") + parser.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset generation mode. Default image2 uses Yunwu Images API.") + parser.add_argument("--candidates-per-slot", type=int, default=4, help="Candidate images per slot, clamped to 1-5. Default: 4.") + parser.add_argument("--asset-workers", type=int, default=1, help="Parallel asset generation workers, clamped to 1-12. Default: 1.") + parser.add_argument("--asset-retries", type=int, default=2, help="Retries per generated asset candidate, clamped to 0-5. Default: 2.") + parser.add_argument("--asset-review-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Selected asset review mode. Default: heuristic.") + parser.add_argument("--locator-mode", choices=["heuristic", "vlm"], default="heuristic", help="Layout coordinate source. Use vlm to locate slots from the reference image.") + parser.add_argument("--locator-model", help="Optional VLM model for --locator-mode vlm. Defaults to RFS_LOCATOR_MODEL/MODEL_VLM.") + parser.add_argument("--control-localizer-mode", choices=["off", "heuristic", "hybrid"], default="hybrid", help="Arrow/connector localization mode. Default hybrid uses CV candidates plus optional VLM binding; falls back to heuristic without API keys.") + parser.add_argument("--arrow-style-mode", choices=["off", "reference", "aesthetic"], default="reference", help="Arrow styling/routing mode. Default reference preserves reference-image routes and adds softer PPT styling/QA metadata.") + parser.add_argument("--prompt-plan-mode", choices=["heuristic", "vlm", "vlm-batch"], default="vlm", help="Reference-aware per-slot prompt planning mode. Default vlm uses one VLM call per slot; vlm-batch uses one batch VLM call; heuristic is offline only.") + parser.add_argument("--prompt-plan-model", help="Optional VLM model for --prompt-plan-mode vlm/vlm-batch. Defaults to RFS_PROMPT_PLANNER_MODEL/MODEL_VLM.") + parser.add_argument("--prompt-plan-workers", type=int, default=4, help="Parallel VLM workers for per-slot prompt planning, clamped to 1-12. Default: 4.") + parser.add_argument("--complexity-profile", choices=["reference-dense", "balanced", "legend-simple"], default="reference-dense", help="Visual complexity policy for slot_visual_spec.json. Default reference-dense.") + parser.add_argument("--critic-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Final reference-vs-render critic mode. Default: heuristic.") + parser.add_argument("--critic-model", help="Optional VLM model for asset review and final critic. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--critic-iterations", type=int, default=0, help="VLM layout correction iterations, clamped to 0-3. Default: 0.") + parser.add_argument("--text-extractor-mode", choices=["heuristic", "ocr"], default="ocr", help="Editable text layer source. Default ocr uses local OCR when available and falls back to heuristic.") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle; easyocr remains available.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--presentations-qa", action="store_true", help="Run optional Presentations plugin import/render/layout QA after export. This never mutates the PPTX.") + parser.add_argument("--presentations-workspace", help="Optional workspace for Presentations QA scratch artifacts.") + parser.add_argument("--presentations-scale", type=int, default=2, help="Preview render scale for Presentations QA. Default: 2.") + parser.add_argument("--no-export", action="store_true", help="Skip PDF/PNG export and only create PPTX/artifacts.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_paper_to_image(subparsers) -> None: + parser = subparsers.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate reviewed raster candidates with an optional editable PPT overlay.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--out", required=True, help="Output directory for review, templates, prompts, candidates, and production-only selected_image.png.") + parser.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") + parser.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") + parser.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") + parser.add_argument("--reference-library", help="Optional reference library JSONL for exemplar selection, prompt guidance, and exemplar-aware candidate review.") + parser.add_argument("--reference-top-k", type=int, default=5, help="Positive exemplar count for --reference-library, clamped to 1-20. Default: 5.") + parser.add_argument("--user-style-brief", help="Optional user_style_brief.json from learn-reference-style. Adds style preferences only.") + parser.add_argument("--preference-history", help="Optional previous paper-to-image run directory or preference_pairs.jsonl for human visual preference-aware review.") + parser.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") + parser.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") + parser.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") + parser.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + parser.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") + parser.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") + parser.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") + parser.add_argument("--language", default="English", help="Visible label language. Default: English.") + parser.add_argument("--image-model", help="Optional image model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") + parser.add_argument("--image-retries", type=int, default=1, help="Retries per image candidate for retryable HTTP/provider errors, clamped to 0-5. Network timeouts are not blindly retried. Default: 1.") + parser.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") + parser.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--repair-rounds", type=int, default=3, help="Localized Image2 edit repair rounds, clamped to 0-4 with no-regression acceptance. Default: 3.") + parser.add_argument("--human-scientific-confirmed", action="store_true", help="Record human scientific review of the selected whole image during the Bootstrap Judge period.") + parser.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") + parser.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") + parser.add_argument("--editable-overlay-ppt", action="store_true", help="Place the selected geometry-normalized Image2 visual substrate beneath native editable PPT labels and connectors. Requires deterministic overlay mode and a raw candidate that passes substrate geometry normalization.") + parser.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_paper_to_editable(subparsers) -> None: + parser = subparsers.add_parser("paper-to-editable", help="Generate a paper-grounded visual reference and rebuild it as an editable PowerPoint figure.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--out", required=True, help="Output directory for paper analysis, image candidates, semantic contracts, and editable PPTX.") + parser.add_argument("--preferences") + parser.add_argument("--positive-reference", action="append", default=[]) + parser.add_argument("--negative-reference", action="append", default=[]) + parser.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + parser.add_argument("--planner-model") + parser.add_argument("--image-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") + parser.add_argument("--image-candidates", type=int, default=3) + parser.add_argument("--aspect-ratio", default="auto") + parser.add_argument("--language", default="English") + parser.add_argument("--image-model") + parser.add_argument("--image-retries", type=int, default=2) + parser.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") + parser.add_argument("--review-model") + parser.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + parser.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + parser.add_argument("--repair-rounds", type=int, default=3) + parser.add_argument("--human-scientific-confirmed", action="store_true", help="Record that a human reviewed the selected whole-image scientific content during the Bootstrap Judge period.") + parser.add_argument("--editable-route", choices=["rebuild", "framework"], default="rebuild", help="Editable compiler route.") + parser.add_argument("--framework-slot-count", type=int, default=25) + parser.add_argument("--framework-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") + parser.add_argument("--framework-candidates-per-slot", type=int, default=1) + parser.add_argument("--framework-asset-workers", type=int, default=3) + parser.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api") + parser.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--vlm-timeout", type=int) + parser.add_argument("--vlm-retries", type=int) + parser.add_argument("--rebuild-critic-mode", choices=["off", "heuristic", "vlm"], default="off") + parser.add_argument("--rebuild-critic-iterations", type=int, default=0) + parser.add_argument("--rebuild-critic-model") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") + parser.add_argument("--allow-engineering-preview", action="store_true") + parser.add_argument("--no-export-preview", action="store_true") + parser.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + parser.add_argument("--json", action="store_true", help="Emit JSON.") diff --git a/rfs/cli_commands/rebuild.py b/rfs/cli_commands/rebuild.py new file mode 100644 index 0000000..496e122 --- /dev/null +++ b/rfs/cli_commands/rebuild.py @@ -0,0 +1,144 @@ +"""Adapters for editable-rebuild CLI arguments.""" + +from __future__ import annotations + +from collections.abc import Callable + + +def add_rebuild_args(parser) -> None: + parser.add_argument("--reference", required=True, help="Reference image path.") + parser.add_argument("--out", required=True, help="Output directory.") + parser.add_argument("--asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api", help="Slot asset source. Policy follows generation-plan per-slot decisions.") + parser.add_argument("--asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") + parser.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") + parser.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") + parser.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") + parser.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") + parser.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") + parser.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") + parser.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM layout disagreements.") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM control disagreements.") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for visible controls missed after adjudication.") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for missed layout objects and controls.") + parser.add_argument("--vlm-timeout", type=int, help="Timeout in seconds for rebuild VLM calls. Defaults to RFS_VLM_TIMEOUT or 240.") + parser.add_argument("--vlm-retries", type=int, help="Retry count for rebuild VLM calls. Defaults to RFS_VLM_RETRIES or 2.") + parser.add_argument("--rebuild-critic-mode", choices=["off", "heuristic", "vlm"], default="off", help="Compiled-preview rebuild critic mode. Default: off.") + parser.add_argument("--rebuild-critic-iterations", type=int, default=0, help="Compiled-preview critic correction iterations, clamped to 0-3. Default: 0.") + parser.add_argument("--rebuild-critic-model", help="Optional VLM model for --rebuild-critic-mode vlm. Defaults to RFS_REBUILD_CRITIC_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") + parser.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") + parser.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") + parser.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") + parser.add_argument("--from-stage", help="Re-run from this stage onward (design_plan/build_program/localize_controls/text_layer/slot_semantics); stages before it reuse snapshots.") + parser.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle; easyocr remains available.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def build_rebuild_kwargs(args, adapter_factory: Callable) -> dict: + """Build ``rebuild_editable`` kwargs without changing CLI semantics.""" + + rebuild_adapters = adapter_factory(args.out, vlm_timeout=args.vlm_timeout, vlm_retries=args.vlm_retries) + use_layout_vlm = args.layout_mode in {"vlm", "hybrid"} + use_control_vlm = args.control_mode in {"vlm", "hybrid"} + use_any_rebuild_vlm = use_layout_vlm or use_control_vlm + return { + "asset_mode": args.asset_mode, + "asset_workers": args.asset_workers, + "asset_retries": args.asset_retries, + "economy_mode": args.economy_mode, + "text_mode": args.text_mode, + "control_mode": args.control_mode, + "layout_mode": args.layout_mode, + "export_preview": args.export_preview, + "regenerate_slots": args.regenerate_slots, + "strict_asset_regeneration": args.strict_asset_regeneration, + "skip_analysis": args.skip_analysis, + "from_stage": args.from_stage, + "compile_only": args.compile_only, + "ocr_engine": args.ocr_engine, + "ocr_lang": args.ocr_lang, + "vlm_layout_adapter": rebuild_adapters["layout"] if use_layout_vlm else None, + "panel_layout_adapter": rebuild_adapters.get("panel_layout") if use_layout_vlm else None, + "layout_adjudication_mode": args.layout_adjudication_mode if use_layout_vlm else "off", + "layout_adjudicator": rebuild_adapters["layout_adjudicator"] if use_layout_vlm and args.layout_adjudication_mode == "vlm" else None, + "control_adapter": rebuild_adapters["control"] if use_control_vlm else None, + "control_adjudication_mode": args.control_adjudication_mode if use_control_vlm else "off", + "control_adjudicator": rebuild_adapters["control_adjudicator"] if use_control_vlm and args.control_adjudication_mode == "vlm" else None, + "control_coverage_audit_mode": args.control_coverage_audit_mode if use_control_vlm else "off", + "coverage_audit_adapter": rebuild_adapters["coverage_audit"] if use_control_vlm and args.control_coverage_audit_mode == "vlm" else None, + "combined_coverage_audit_mode": args.combined_coverage_audit_mode if use_any_rebuild_vlm else "off", + "combined_coverage_audit_adapter": rebuild_adapters["combined_coverage_audit"] if use_any_rebuild_vlm and args.combined_coverage_audit_mode == "vlm" else None, + "vlm_timeout": args.vlm_timeout, + "vlm_retries": args.vlm_retries, + "rebuild_critic_mode": args.rebuild_critic_mode, + "rebuild_critic_iterations": args.rebuild_critic_iterations, + "rebuild_critic_model": args.rebuild_critic_model, + "semantic_adapter": rebuild_adapters["semantic"] if use_any_rebuild_vlm else None, + "asset_policy": args.asset_policy, + "design_plan_mode": args.design_plan_mode, + "design_plan_model": args.design_plan_model, + "design_adapter": rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, + "text_grouping_mode": args.text_grouping_mode, + "text_grouping_model": args.text_grouping_model, + } + + +def register_rebuild_core_commands(subparsers) -> None: + rebuild = subparsers.add_parser("rebuild-editable", help="Rebuild a reference image into a reusable editable PowerPoint composition.") + add_rebuild_args(rebuild) + + evolve = subparsers.add_parser("rebuild-evolve", help="Run rebuild-editable in a process-supervision evolution loop.") + add_rebuild_args(evolve) + evolve.add_argument("--max-rounds", type=int, default=3, help="Maximum rebuild+judge rounds. Default: 3.") + evolve.add_argument("--judge-model", help="Process Judge model. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") + evolve.add_argument("--score-threshold", type=float, help="Stop when the Judge overall score reaches this threshold.") + + evaluation = subparsers.add_parser("rebuild-editable-eval", help="Compare heuristic and hybrid VLM rebuild-editable outputs for one reference image.") + evaluation.add_argument("--reference", required=True, help="Reference image path.") + evaluation.add_argument("--out", required=True, help="Evaluation output directory.") + evaluation.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="crop", help="Asset mode for both cases. Default: crop to avoid image generation cost.") + evaluation.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Text extraction mode for both cases. Default: ocr.") + evaluation.add_argument("--export-preview", action="store_true", help="Export PNG previews when PowerPoint is available.") + evaluation.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_professional_rebuild(subparsers) -> None: + parser = subparsers.add_parser("rebuild-editable-pro", help="Rebuild a reference image through a VLM-authored controlled professional DSL.") + parser.add_argument("--reference", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api") + parser.add_argument("--asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api") + parser.add_argument("--asset-workers", type=int, default=4) + parser.add_argument("--asset-retries", type=int, default=1) + parser.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True) + parser.add_argument("--no-economy-mode", dest="economy_mode", action="store_false") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") + parser.add_argument("--design-plan-model") + parser.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic") + parser.add_argument("--text-grouping-model") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--vlm-timeout", type=int) + parser.add_argument("--vlm-retries", type=int) + parser.add_argument("--export-preview", action="store_true") + parser.add_argument("--regenerate-slots") + parser.add_argument("--strict-asset-regeneration", action="store_true") + parser.add_argument("--compile-only", action="store_true") + parser.add_argument("--repair-rounds", type=int, default=2) + parser.add_argument("--repair-mode", choices=["report", "vlm"], default="report") + parser.add_argument("--benchmark-out") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + parser.add_argument("--json", action="store_true") diff --git a/rfs/composition/__init__.py b/rfs/composition/__init__.py new file mode 100644 index 0000000..74d6b19 --- /dev/null +++ b/rfs/composition/__init__.py @@ -0,0 +1,7 @@ +"""Deterministic editable-figure composition and preview rendering.""" + +from .pptx import compile_ppt +from .preview import render_rebuild_preview +from .rebuild import build_figure_program, build_text_layer, style_and_route_arrows + +__all__ = ["build_figure_program", "build_text_layer", "compile_ppt", "render_rebuild_preview", "style_and_route_arrows"] diff --git a/rfs/composition/pptx.py b/rfs/composition/pptx.py new file mode 100644 index 0000000..0d6262f --- /dev/null +++ b/rfs/composition/pptx.py @@ -0,0 +1,720 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.dml import MSO_LINE_DASH_STYLE +from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE +from pptx.enum.text import PP_ALIGN, MSO_ANCHOR, MSO_AUTO_SIZE +from pptx.oxml import parse_xml +from pptx.util import Inches, Pt + +from ..utils import pct_to_inches, write_json + + +def _rgb(hex_color: str) -> RGBColor: + value = hex_color.strip().lstrip("#") or "000000" + return RGBColor(int(value[:2], 16), int(value[2:4], 16), int(value[4:6], 16)) + + +def _apply_arrow(connector, size: str = "sm") -> None: + ln = connector.line._get_or_add_ln() + ln.append(parse_xml(f'')) + + +def _apply_line_cap(connector, cap: str = "round") -> None: + value = {"round": "rnd", "square": "sq"}.get(str(cap).lower()) + if not value: + return + ln = connector.line._get_or_add_ln() + ln.set("cap", value) + + +def _connector_type_for_route(route_style: str): + if str(route_style).lower() in {"soft_curve", "dashed_loop", "dashed_spline_like"}: + return MSO_CONNECTOR.CURVE + return MSO_CONNECTOR.STRAIGHT + + +def _set_text( + shape, + text: str, + font_size: float = 10, + bold: bool = False, + color: str = "#163B4D", + align=PP_ALIGN.CENTER, + font_family: str | None = None, + fit_to_shape: bool = False, +) -> None: + tf = shape.text_frame + tf.clear() + tf.word_wrap = True + if fit_to_shape: + tf.auto_size = MSO_AUTO_SIZE.TEXT_TO_FIT_SHAPE + tf.vertical_anchor = MSO_ANCHOR.MIDDLE + p = tf.paragraphs[0] + p.alignment = align + run = p.add_run() + run.text = text + run.font.size = Pt(float(font_size)) + run.font.bold = bold + if font_family: + run.font.name = str(font_family) + run.font.color.rgb = _rgb(color) + + +def _add_round_rect(slide, x: float, y: float, w: float, h: float, fill: str, stroke: str, width_pt: float = 1.2): + shape = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x), Inches(y), Inches(w), Inches(h)) + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(fill) + shape.line.color.rgb = _rgb(stroke) + shape.line.width = Pt(width_pt) + shape.shadow.inherit = False + return shape + + +def _add_card_frame(slide, card: dict, width_in: float, height_in: float): + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + shape_type = MSO_SHAPE.RECTANGLE if str(card.get("shape_kind") or "rounded_rect").lower() == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + if float(card.get("fill_transparency", 1.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(card.get("fill_color") or "#FFFFFF")) + shape.line.color.rgb = _rgb(str(card.get("stroke_color") or "#59AFCB")) + shape.line.width = Pt(float(card.get("stroke_width_pt") or 1.5)) + dash_style = str(card.get("dash_style") or "solid").lower() + if dash_style in {"dash", "dashed"}: + shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH + elif dash_style in {"dot", "dotted"}: + shape.line.dash_style = getattr(MSO_LINE_DASH_STYLE, "ROUND_DOT", MSO_LINE_DASH_STYLE.DASH) + shape.shadow.inherit = False + return shape + + +def _add_label( + slide, + text: str, + x: float, + y: float, + w: float, + h: float, + font_size: float = 9, + bold: bool = False, + align=PP_ALIGN.CENTER, + color: str = "#163B4D", + font_family: str | None = None, + fit_to_shape: bool = False, +): + box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) + _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family, fit_to_shape=fit_to_shape) + return box + + +def _add_title_block(slide, program: dict, width_in: float, height_in: float) -> None: + title_block = program.get("title_block") + if not isinstance(title_block, dict): + return + title = str(title_block.get("title", "")).strip() + subtitle = str(title_block.get("subtitle", "")).strip() + bbox = title_block.get("bbox_percent") + if not title or not isinstance(bbox, dict): + return + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + title_h = min(h * 0.58, 0.42) + subtitle_h = max(0.18, min(h - title_h, 0.26)) + _add_label( + slide, + title, + x, + y, + w, + title_h, + font_size=int(title_block.get("title_font_size", 24)), + bold=True, + align=PP_ALIGN.LEFT, + ) + if subtitle: + subtitle_box = _add_label( + slide, + subtitle, + x, + y + title_h * 0.94, + w, + subtitle_h, + font_size=int(title_block.get("subtitle_font_size", 11)), + bold=False, + align=PP_ALIGN.LEFT, + ) + for paragraph in subtitle_box.text_frame.paragraphs: + for run in paragraph.runs: + run.font.color.rgb = _rgb(str(title_block.get("subtitle_color", "#333333"))) + + +def _short_caption(text: str) -> str: + replacements = { + "participant seated before 49-inch screen": "Participant setup", + "3D virtual interviewer on screen": "Virtual interviewer", + "wide-angle camera capturing full body": "Full-body camera", + "virtual interview room setup": "Interview room", + "interview question prompt on screen": "Question prompt", + "raw interview video recording": "Raw video", + "287 participant video collection": "287 participants", + "36 interview questions": "36 questions", + "full-body video thumbnails": "Full-body clips", + "participant clip grid": "Clip grid", + "FFmpeg extracts audio from video": "FFmpeg audio", + "audio waveform stream": "Audio stream", + "FunASR speech recognition": "FunASR ASR", + "spoken text with timestamps": "Text + timestamps", + "MTCNN face detection": "MTCNN face", + "face clips": "Face clips", + "AlphaPose full-body skeleton extraction": "AlphaPose pose", + "full-body pose skeleton stream": "Pose stream", + "frame sampling from video": "Frame sampling", + "sampled video frames": "Video frames", + "timestamp alignment clock": "Alignment clock", + "aligned audio and text streams": "Audio + text", + "aligned face and pose streams": "Face + pose", + "aligned frame stream": "Frame stream", + "face modality": "Face", + "frame modality": "Frame", + "pose modality": "Pose", + "audio modality": "Audio", + "text modality": "Text", + "NEO-FFI-3 questionnaire": "NEO-FFI-3", + "OCEAN score vector": "OCEAN scores", + "Openness score": "Openness", + "Conscientiousness score": "Conscientiousness", + "Extraversion score": "Extraversion", + "Agreeableness score": "Agreeableness", + "Neuroticism score": "Neuroticism", + } + if text in replacements: + return replacements[text] + if len(text) <= 22: + return text + words = text.replace("/", " ").split() + return " ".join(words[:3]) if len(words) > 3 else text[:22] + + +def _picture_contain_box(image_path: Path, x: float, y: float, w: float, h: float) -> tuple[float, float, float, float, float]: + with Image.open(image_path) as img: + iw, ih = img.size + image_ratio = iw / max(ih, 1) + slot_ratio = w / max(h, 0.001) + if image_ratio > slot_ratio: + fit_w = w + fit_h = w / image_ratio + else: + fit_h = h + fit_w = h * image_ratio + left = x + (w - fit_w) / 2 + top = y + (h - fit_h) / 2 + fill_percent = fit_w * fit_h / max(w * h, 0.001) * 100 + return left, top, fit_w, fit_h, fill_percent + + +def _add_picture_contain(slide, image_path: Path, x: float, y: float, w: float, h: float): + left, top, fit_w, fit_h, _fill_percent = _picture_contain_box(image_path, x, y, w, h) + return slide.shapes.add_picture(str(image_path), Inches(left), Inches(top), width=Inches(fit_w), height=Inches(fit_h)) + + +def _panel_map(program: dict) -> dict[str, dict]: + return {panel["id"]: panel for panel in program.get("panels", [])} + + +def _text_program_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + items = text_program.get("items", []) + return [item for item in items if isinstance(item, dict) and item.get("visible", True)] + + +def _text_item_for_target(program: dict, target_id: str, role: str) -> dict | None: + for item in _text_program_items(program): + if str(item.get("target_id")) == target_id and str(item.get("role")) == role: + return item + return None + + +def _align_from_text_item(item: dict): + value = str(item.get("align") or "").lower() + if value == "left": + return PP_ALIGN.LEFT + if value == "right": + return PP_ALIGN.RIGHT + return PP_ALIGN.CENTER + + +def _object_map(program: dict) -> dict[str, dict]: + objects = {panel["id"]: panel for panel in program.get("panels", [])} + objects.update({slot["id"]: slot for slot in program.get("slots", [])}) + objects.update({node["id"]: node for node in program.get("semantic_nodes", []) if isinstance(node, dict) and node.get("id")}) + return objects + + +def _bbox_center(bbox: dict, canvas_w: float, canvas_h: float) -> tuple[float, float]: + x, y, w, h = pct_to_inches(bbox, canvas_w, canvas_h) + return x + w / 2, y + h / 2 + + +def _arrow_endpoints(source: dict, target: dict, canvas_w: float, canvas_h: float) -> tuple[float, float, float, float]: + sbox = source["bbox_percent"] + tbox = target["bbox_percent"] + sx, sy = _bbox_center(sbox, canvas_w, canvas_h) + tx, ty = _bbox_center(tbox, canvas_w, canvas_h) + s_left, s_top, s_w, s_h = pct_to_inches(sbox, canvas_w, canvas_h) + t_left, t_top, t_w, t_h = pct_to_inches(tbox, canvas_w, canvas_h) + + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + if dx >= 0: + return s_left + s_w + 0.03, sy, t_left - 0.03, ty + return s_left - 0.03, sy, t_left + t_w + 0.03, ty + if dy >= 0: + return sx, s_top + s_h + 0.03, tx, t_top - 0.03 + return sx, s_top - 0.03, tx, t_top + t_h + 0.03 + + +def _arrow_points_from_path(path: list, width_in: float, height_in: float) -> list[tuple[float, float]]: + points = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((float(point[0]) * width_in, float(point[1]) * height_in)) + return points + + +def _draw_program_arrows(slide, program: dict, width_in: float, height_in: float) -> list[dict]: + objects_by_id = _object_map(program) + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + token_map = {str(item.get("token_id")): item for item in style.get("color_tokens", []) if isinstance(item, dict)} + rendered: list[dict] = [] + for arrow in program.get("arrows", []): + if arrow.get("type") == "custom_bus": + continue + source = objects_by_id.get(arrow.get("source") or arrow.get("source_id")) + target = objects_by_id.get(arrow.get("target") or arrow.get("target_id")) + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if len(path) >= 2 and all(isinstance(point, list) and len(point) >= 2 for point in path): + points = _arrow_points_from_path(path, width_in, height_in) + elif source and target: + x1, y1, x2, y2 = _arrow_endpoints(source, target, width_in, height_in) + points = [(x1, y1), (x2, y2)] + else: + continue + token = token_map.get(str(arrow.get("style_token_id"))) + arrow_color = str(token.get("hex")) if token else str(arrow.get("stroke_color") or "#1F6F8B") + control_kind = str(arrow.get("control_kind") or arrow.get("type", "")).lower() + dashed = control_kind in {"dashed_loop", "dashed", "loop"} + if str(arrow.get("line_pattern", "")).lower() in {"dash", "dashed"}: + dashed = True + line_width = float(arrow.get("stroke_width_pt") or style.get("arrow_weight_pt") or 1.7) + route_style = str(arrow.get("route_style") or "") + connector_type = _connector_type_for_route(route_style) + halo_width = float(arrow.get("halo_width_pt") or 0.0) + halo_color = str(arrow.get("halo_color") or "#FFFFFF") + arrowhead_size = str(arrow.get("arrowhead_size") or "sm").lower() + if arrowhead_size not in {"sm", "med", "lg"}: + arrowhead_size = "sm" + segment_count = 0 + for idx, ((x1, y1), (x2, y2)) in enumerate(zip(points[:-1], points[1:])): + if halo_width > 0: + halo = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + halo.line.color.rgb = _rgb(halo_color) + halo.line.width = Pt(max(line_width + 1.2, halo_width)) + _apply_line_cap(halo, str(arrow.get("line_cap") or "round")) + if dashed: + halo.line.dash_style = MSO_LINE_DASH_STYLE.DASH + connector = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + connector.name = f"RFS Connector {arrow.get('id') or 'unnamed'} {idx + 1}" + connector.line.color.rgb = _rgb(arrow_color) + connector.line.width = Pt(line_width) + _apply_line_cap(connector, str(arrow.get("line_cap") or "round")) + if dashed: + connector.line.dash_style = MSO_LINE_DASH_STYLE.DASH + if idx == len(points) - 2: + _apply_arrow(connector, arrowhead_size) + segment_count += 1 + rendered.append({ + "arrow_id": arrow.get("id"), + "control_kind": control_kind or "straight_arrow", + "semantic_role": arrow.get("semantic_role"), + "route_style": route_style, + "bundle_id": arrow.get("bundle_id"), + "lane_index": arrow.get("lane_index"), + "line_cap": arrow.get("line_cap", "round"), + "line_pattern": "dash" if dashed else "solid", + "connector_type": "curve" if connector_type == MSO_CONNECTOR.CURVE else "straight", + "halo_width_pt": halo_width, + "halo_color": halo_color if halo_width > 0 else None, + "stroke_width_pt": line_width, + "arrowhead_size": arrowhead_size, + "routing_algorithm": arrow.get("routing_algorithm"), + "route_generation_status": arrow.get("route_generation_status"), + "segment_count": segment_count, + "point_count": len(points), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok" if segment_count else "not_rendered", + }) + return rendered + + +def _ocean_label(text: str) -> str: + low = text.lower() + if "openness" in low: + return "Openness" + if "conscientiousness" in low: + return "Conscientiousness" + if "extraversion" in low: + return "Extraversion" + if "agreeableness" in low: + return "Agreeableness" + if "neuroticism" in low: + return "Neuroticism" + return _short_caption(text) + + +def compile_ppt(program: dict, out_dir: str | Path) -> Path: + out = Path(out_dir) + canvas = program["canvas"] + width_in = float(canvas["width_in"]) + height_in = float(canvas["height_in"]) + style = program.get("style", {}) + palette = style.get("palette") or style.get("reference_palette") or ["#2D6FB7", "#E17721", "#6B57C8", "#1B9A94", "#4B9B52", "#D44E5D"] + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + + prs = Presentation() + prs.slide_width = Inches(width_in) + prs.slide_height = Inches(height_in) + slide = prs.slides.add_slide(prs.slide_layouts[6]) + + background_visual = None + background = program.get("background_image") + if isinstance(background, dict) and str(background.get("path") or "").strip(): + background_path = Path(str(background["path"])) + if not background_path.is_absolute(): + background_path = out / background_path + if not background_path.exists(): + raise FileNotFoundError(f"PPT background visual substrate is missing: {background_path}") + left, top, fit_w, fit_h, fill_percent = _picture_contain_box(background_path, 0.0, 0.0, width_in, height_in) + picture = slide.shapes.add_picture( + str(background_path), + Inches(left), + Inches(top), + width=Inches(fit_w), + height=Inches(fit_h), + ) + picture.name = str(background.get("name") or "RFS Image2 Visual Substrate") + background_visual = { + "path": str(background_path), + "role": str(background.get("role") or "image2_visual_substrate"), + "contains_scientific_text": bool(background.get("contains_scientific_text", False)), + "contains_connectors": bool(background.get("contains_connectors", False)), + "canvas_area_fill_percent": round(fill_percent, 2), + "fit_policy": "contain_no_crop", + "rendered_as": "ppt_picture_background", + } + + _add_title_block(slide, program, width_in, height_in) + + panels_by_id = _panel_map(program) + panel_shapes = {} + has_text_program = bool(_text_program_items(program)) + ocr_panel_title_targets = { + str(item.get("target_id")) + for item in _text_program_items(program) + if str(item.get("role")) == "panel_title" and "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() + } + semantic_panel_title_targets = { + str(item.get("target_id")) + for item in _text_program_items(program) + if str(item.get("role")) == "panel_title" and item.get("semantic_panel_title") + } + ocr_panel_title_targets -= semantic_panel_title_targets + for idx, panel in enumerate(program["panels"]): + x, y, w, h = pct_to_inches(panel["bbox_percent"], width_in, height_in) + local_style = panel_styles.get(panel["id"], {}) if isinstance(panel_styles.get(panel["id"], {}), dict) else {} + fill = local_style.get("fill_color") or palette[idx % len(palette)] + stroke = local_style.get("stroke_color") or palette[(idx + 1) % len(palette)] + header_color = local_style.get("header_color") or stroke + shape = _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=1.4) + panel_shapes[panel["id"]] = shape + header_h = min(0.34, h * 0.17) + header = _add_round_rect(slide, x, y, w, header_h, header_color, header_color, width_pt=0.8) + title_text_item = _text_item_for_target(program, panel["id"], "panel_title") + _set_text( + header, + "" if str(panel["id"]) in ocr_panel_title_targets else str(title_text_item.get("text") or panel["title"]) if title_text_item else panel["title"], + font_size=float(title_text_item.get("font_size_pt")) if title_text_item else (9 if w < 2.0 else 10), + bold=True, + color=str(title_text_item.get("color_hex") or "#FFFFFF") if title_text_item else "#FFFFFF", + font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, + fit_to_shape=True, + ) + + rendered_cards = [] + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index", 12))): + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + _add_card_frame(slide, card, width_in, height_in) + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + if card.get("title") and not has_text_program: + _add_label(slide, str(card.get("title")), x + 0.03, y + 0.03, max(0.05, w - 0.06), min(0.22, h * 0.32), font_size=7, bold=True) + rendered_cards.append({ + "card_id": card.get("id"), + "semantic_role": card.get("semantic_role"), + "shape_kind": card.get("shape_kind", "rounded_rect"), + "bbox_percent": card.get("bbox_percent"), + "dash_style": card.get("dash_style", "solid"), + "fill_transparency": float(card.get("fill_transparency", 1.0)), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok", + }) + + rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) + + # Paper semantic nodes are rendered after connectors so every edge stays + # behind the editable node shape and its exact paper label. + rendered_semantic_nodes = [] + for node in sorted(program.get("semantic_nodes", []), key=lambda item: int(item.get("z_index", 30))): + if not isinstance(node, dict) or not isinstance(node.get("bbox_percent"), dict): + continue + x, y, w, h = pct_to_inches(node["bbox_percent"], width_in, height_in) + shape_kind = str(node.get("shape_kind") or "rounded_rect").casefold() + shape_type = MSO_SHAPE.RECTANGLE if shape_kind == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + shape.name = f"RFS Semantic Node {node.get('id') or 'unnamed'}" + if float(node.get("fill_transparency", 0.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(node.get("fill_color") or "#F5FAFC")) + shape.line.color.rgb = _rgb(str(node.get("stroke_color") or "#1F6F8B")) + shape.line.width = Pt(float(node.get("stroke_width_pt") or 1.6)) + shape.shadow.inherit = False + label_bbox = node.get("label_bbox_percent") + _set_text( + shape, + "" if isinstance(label_bbox, dict) else str(node.get("label") or node.get("name") or node.get("id") or ""), + font_size=float(node.get("font_size_pt") or 16), + bold=bool(node.get("bold", True)), + color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), + font_family=str(node.get("font_family") or "Arial"), + ) + label_rendered_as = "node_shape_text" + if isinstance(label_bbox, dict): + lx, ly, lw, lh = pct_to_inches(label_bbox, width_in, height_in) + label_shape = _add_round_rect( + slide, + lx, + ly, + lw, + lh, + str(node.get("label_fill_color") or node.get("fill_color") or "#F5FAFC"), + str(node.get("label_stroke_color") or "#FFFFFF"), + width_pt=float(node.get("label_stroke_width_pt") or 0.5), + ) + label_shape.name = f"RFS Semantic Label {node.get('id') or 'unnamed'}" + _set_text( + label_shape, + str(node.get("label") or node.get("name") or node.get("id") or ""), + font_size=float(node.get("font_size_pt") or 16), + bold=bool(node.get("bold", True)), + color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), + font_family=str(node.get("font_family") or "Arial"), + ) + label_rendered_as = "separate_editable_header_shape" + rendered_semantic_nodes.append({ + "node_id": node.get("id"), + "label": node.get("label") or node.get("name"), + "field": node.get("field"), + "role": node.get("role"), + "bbox_percent": node.get("bbox_percent"), + "shape_kind": shape_kind, + "fill_transparency": float(node.get("fill_transparency", 0.0)), + "label_bbox_percent": label_bbox, + "label_rendered_as": label_rendered_as, + "editable_in": "pptx", + "render_policy": "ppt_transparent_node_plus_editable_header" if isinstance(label_bbox, dict) else "ppt_shape_with_editable_text", + "status": "ok", + }) + + # Slot image layer and editable captions. + composition_items = [] + caption_queue = [] + ordered_slots = sorted(program["slots"], key=lambda item: int(item.get("z_index", 20))) + for slot in ordered_slots: + x, y, w, h = pct_to_inches(slot["bbox_percent"], width_in, height_in) + asset_path = out / "assets" / f"{slot['asset_id']}.png" + tile_added = False + if slot.get("composition_type") == "symbol_cutout": + if asset_path.exists(): + _add_picture_contain(slide, asset_path, x, y, w, h) + _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) + else: + fill_percent = 0.0 + if bool(slot.get("show_slot_caption", False)) and not has_text_program: + parent = panels_by_id.get(slot.get("panel_id")) + if parent: + px, py, pw, ph = pct_to_inches(parent["bbox_percent"], width_in, height_in) + label_x = x + w + 0.035 + label_w = max(0.28, min(0.86, px + pw - label_x - 0.03)) + caption_queue.append(( + _ocean_label(slot.get("display_label") or slot["paper_concept"]), + label_x, + y + h * 0.10, + label_w, + h * 0.80, + 5 if label_w < 0.55 else 6, + PP_ALIGN.LEFT, + )) + composition_items.append({ + "slot_id": slot["id"], + "asset_id": slot["asset_id"], + "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + "tile_frame_added": tile_added, + "caption_inside_image_slot": False, + "slot_bbox_percent": slot["bbox_percent"], + "image_slot_area_fill_percent": round(fill_percent, 2), + "status": "ok" if fill_percent >= 95 else "image_area_below_95", + }) + continue + if asset_path.exists(): + _add_picture_contain(slide, asset_path, x, y, w, h) + _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) + else: + fill_percent = 0.0 + if bool(slot.get("show_slot_caption", False)) and not has_text_program: + caption_h = min(0.20, h * 0.18) + caption = slot.get("display_label") or _short_caption(slot["paper_concept"]) + caption_queue.append((caption, x + w * 0.03, y + h + 0.01, w * 0.94, caption_h, 5 if w < 0.8 else 6, PP_ALIGN.CENTER)) + composition_items.append({ + "slot_id": slot["id"], + "asset_id": slot["asset_id"], + "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + "tile_frame_added": tile_added, + "caption_inside_image_slot": False, + "slot_bbox_percent": slot["bbox_percent"], + "image_slot_area_fill_percent": round(fill_percent, 2), + "status": "ok" if fill_percent >= 95 else "image_area_below_95", + }) + + for caption, x, y, w, h, font_size, align in caption_queue: + _add_label(slide, caption, x, y, w, h, font_size=font_size, align=align) + + rendered_text_items = [] + for item in _text_program_items(program): + is_ocr_text = "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() + targets_actual_panel = str(item.get("target_id") or "") in panels_by_id + if str(item.get("role")) == "panel_title" and targets_actual_panel and not is_ocr_text: + rendered_text_items.append({ + "text_id": item.get("id"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "rendered_as": "panel_header_text", + "editable_in": "pptx", + }) + continue + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + _add_label( + slide, + str(item.get("text") or ""), + x, + y, + w, + h, + font_size=float(item.get("font_size_pt") or 6), + bold=bool(item.get("bold")), + align=_align_from_text_item(item), + color=str(item.get("color_hex") or "#263747"), + font_family=str(item.get("font_family_guess") or "") or None, + fit_to_shape=bool(item.get("semantic_entity_id") or item.get("semantic_panel_title")), + ) + rendered_text_items.append({ + "text_id": item.get("id"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "color_hex": item.get("color_hex"), + "font_family_guess": item.get("font_family_guess"), + "fit_strategy": item.get("fit_strategy"), + "ocr_confidence": item.get("ocr_confidence"), + "editable_in": "pptx", + "rendered_as": "ppt_textbox", + }) + + for label in program.get("labels", []): + bbox = label.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + _add_label( + slide, + str(label.get("text") or ""), + x, + y, + w, + h, + font_size=float(label.get("font_size_pt") or 9), + bold=bool(label.get("bold", True)), + align=PP_ALIGN.LEFT if str(label.get("align")).lower() == "left" else PP_ALIGN.CENTER, + color=str(label.get("color_hex") or "#263747"), + ) + + # Shared resource bus as editable connectors. + shared = panels_by_id.get("shared_resource_library") + if shared: + sx, sy, sw, sh = pct_to_inches(shared["bbox_percent"], width_in, height_in) + bus_y = sy - 0.30 + line = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(sx + 0.1), Inches(bus_y), Inches(sx + sw - 0.1), Inches(bus_y)) + line.line.color.rgb = _rgb("#1F6F8B") + line.line.width = Pt(1.2) + for panel in program["panels"]: + if panel["id"] == "shared_resource_library": + continue + cx, _cy = _bbox_center(panel["bbox_percent"], width_in, height_in) + if not (sx + 0.1 <= cx <= sx + sw - 0.1): + continue + down = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(cx), Inches(bus_y), Inches(cx), Inches(sy)) + down.line.color.rgb = _rgb("#1F6F8B") + down.line.width = Pt(1.0) + if not has_text_program: + _add_label(slide, "shared resources", sx + 0.2, bus_y - 0.22, 1.5, 0.18, font_size=7, align=PP_ALIGN.LEFT) + + if program.get("show_visible_title"): + title = program.get("paper_brief", {}).get("title_guess") or "Research System Figure" + _add_label(slide, title[:90], width_in - 5.2, height_in - 0.50, 4.9, 0.38, font_size=7, bold=True, align=PP_ALIGN.RIGHT) + + pptx_path = out / "editable_composition.pptx" + prs.save(pptx_path) + write_json(out / "composition_quality_report.json", { + "summary": "PPT composition quality report checking frameless slot insertion, no extra white tiles, and image area fill inside each reference slot.", + "policy": { + "slot_frame_policy": "frameless_slot", + "picture_fill_policy": "direct_full_slot_contain_no_tile", + "min_image_slot_area_fill_percent": 95, + "caption_inside_image_slot_allowed": False, + }, + "slots": composition_items, + "cards": rendered_cards, + "arrows": rendered_arrows, + "semantic_nodes": rendered_semantic_nodes, + "text": rendered_text_items, + "background_visual": background_visual, + }) + return pptx_path diff --git a/rfs/composition/preview.py b/rfs/composition/preview.py new file mode 100644 index 0000000..72e0547 --- /dev/null +++ b/rfs/composition/preview.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image, ImageColor, ImageDraw, ImageFont + + +def _hex_color(value: object, fallback: str = "#000000") -> str: + if not isinstance(value, str): + return fallback + text = value.strip() + if not text: + return fallback + if not text.startswith("#"): + text = "#" + text + try: + ImageColor.getrgb(text) + return text + except ValueError: + return fallback + + +def _bbox_px(bbox: dict, width: int, height: int) -> tuple[int, int, int, int]: + x = int(round(float(bbox.get("x", 0)) * width)) + y = int(round(float(bbox.get("y", 0)) * height)) + w = int(round(float(bbox.get("w", 0)) * width)) + h = int(round(float(bbox.get("h", 0)) * height)) + return x, y, max(1, w), max(1, h) + + +def _font(size_px: int, bold: bool = False) -> ImageFont.ImageFont: + candidates = [ + "arialbd.ttf" if bold else "arial.ttf", + "calibrib.ttf" if bold else "calibri.ttf", + "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", + ] + for name in candidates: + try: + return ImageFont.truetype(name, max(4, int(size_px))) + except OSError: + continue + return ImageFont.load_default() + + +def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: int) -> None: + if not item.get("visible", True): + return + if str(item.get("layer_ownership") or "editable_text_layer") != "editable_text_layer": + return + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + text = str(item.get("text") or "") + if not text: + return + font_size_pt = float(item.get("font_size_pt") or 8) + size_px = max(4, int(round(font_size_pt * canvas_h / 387.0))) + bold = bool(item.get("bold")) + font = _font(size_px, bold=bold) + color = _hex_color(item.get("color_hex"), "#263747") + align = str(item.get("align") or "center").lower() + measure = draw.multiline_textbbox if "\n" in text else draw.textbbox + text_bbox = measure((0, 0), text, font=font, align="center") if "\n" in text else measure((0, 0), text, font=font) + text_w = max(1, text_bbox[2] - text_bbox[0]) + text_h = max(1, text_bbox[3] - text_bbox[1]) + while size_px > 4 and (text_w > w or text_h > h): + size_px -= 1 + font = _font(size_px, bold=bold) + text_bbox = measure((0, 0), text, font=font, align="center") if "\n" in text else measure((0, 0), text, font=font) + text_w = max(1, text_bbox[2] - text_bbox[0]) + text_h = max(1, text_bbox[3] - text_bbox[1]) + if align == "left": + tx = x + elif align == "right": + tx = x + w - text_w + else: + tx = x + (w - text_w) / 2 + ty = y + (h - text_h) / 2 + if "\n" in text: + draw.multiline_text((tx, ty), text, fill=color, font=font, align="center", spacing=0) + else: + draw.text((tx, ty), text, fill=color, font=font) + + +def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: int) -> None: + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + return + points: list[tuple[int, int]] = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((int(round(float(point[0]) * canvas_w)), int(round(float(point[1]) * canvas_h)))) + if len(points) < 2: + return + color = _hex_color(arrow.get("stroke_color") or arrow.get("outline_color"), "#3F5063") + line_width = max(1, int(round(float(arrow.get("stroke_width_pt") or 1.5) * canvas_h / 277.0))) + render_style = str(arrow.get("render_style") or "").lower() + if render_style == "filled_block_arrow": + fill = _hex_color(arrow.get("fill_color"), "#AFC6DE") + start, end = points[0], points[-1] + thickness = max(8, int(round(float(arrow.get("block_arrow_thickness_percent") or 0.052) * canvas_h))) + if abs(end[0] - start[0]) >= abs(end[1] - start[1]): + y = int(round((start[1] + end[1]) / 2)) + left, right = sorted([start[0], end[0]]) + draw.rounded_rectangle((left, y - thickness // 2, right, y + thickness // 2), radius=thickness // 3, fill=fill, outline=color, width=line_width) + else: + x = int(round((start[0] + end[0]) / 2)) + top, bottom = sorted([start[1], end[1]]) + draw.rounded_rectangle((x - thickness // 2, top, x + thickness // 2, bottom), radius=thickness // 3, fill=fill, outline=color, width=line_width) + return + draw.line(points, fill=color, width=line_width, joint="curve") + + +def _draw_card(draw: ImageDraw.ImageDraw, card: dict, canvas_w: int, canvas_h: int) -> None: + bbox = card.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + stroke = _hex_color(card.get("stroke_color"), "#59AFCB") + fill = None if float(card.get("fill_transparency", 1.0)) >= 1.0 else _hex_color(card.get("fill_color"), "#FFFFFF") + line_width = max(1, int(round(float(card.get("stroke_width_pt") or 1.5) * canvas_h / 360.0))) + radius = max(0, int(round(float(card.get("corner_radius") or 0.08) * min(w, h)))) + rect = (x, y, x + w, y + h) + if str(card.get("shape_kind") or "rounded_rect").lower() == "rect": + draw.rectangle(rect, fill=fill, outline=stroke, width=line_width) + else: + draw.rounded_rectangle(rect, radius=radius, fill=fill, outline=stroke, width=line_width) + + +def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str | Path | None = None) -> Path: + out = Path(out_dir) + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + width = int(canvas.get("width_px") or 1600) + height = int(canvas.get("height_px") or 900) + background = _hex_color(canvas.get("background"), "#FFFFFF") + image = Image.new("RGB", (width, height), background) + draw = ImageDraw.Draw(image) + + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + palette = style.get("palette") or ["#F7F8F7", "#D2DCDE", "#A09596"] + + for idx, panel in enumerate(program.get("panels", [])): + bbox = panel.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + local_style = panel_styles.get(panel.get("id"), {}) if isinstance(panel_styles.get(panel.get("id")), dict) else {} + fill = _hex_color(local_style.get("fill_color"), palette[idx % len(palette)]) + stroke = _hex_color(local_style.get("stroke_color"), "#FFFFFF") + draw.rounded_rectangle((x, y, x + w, y + h), radius=max(2, min(12, h // 18)), fill=fill, outline=stroke, width=2) + + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index") or 12)): + if isinstance(card, dict): + _draw_card(draw, card, width, height) + + asset_by_id = {str(asset.get("id")): asset for asset in program.get("assets", []) if isinstance(asset, dict)} + for slot in sorted(program.get("slots", []), key=lambda item: int(item.get("z_index") or 20)): + bbox = slot.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + asset = asset_by_id.get(str(slot.get("asset_id") or slot.get("id"))) + asset_path = out / str(asset.get("path")) if asset else out / "assets" / f"{slot.get('asset_id')}.png" + if asset_path.exists(): + with Image.open(asset_path) as asset_img: + asset_img = asset_img.convert("RGB") + asset_img.thumbnail((w, h), Image.Resampling.LANCZOS) + left = x + (w - asset_img.width) // 2 + top = y + (h - asset_img.height) // 2 + image.paste(asset_img, (left, top)) + + for arrow in program.get("arrows", []): + if isinstance(arrow, dict): + _draw_arrow(draw, arrow, width, height) + + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} + text_items = text_program.get("items", []) if isinstance(text_program, dict) else [] + visible_items = [item for item in text_items if isinstance(item, dict)] + for item in sorted(visible_items, key=lambda item: (float(item.get("z_index") or 80), str(item.get("id") or ""))): + _draw_text(draw, item, width, height) + + target = Path(preview_path) if preview_path else out / "rebuild_preview.png" + target.parent.mkdir(parents=True, exist_ok=True) + image.save(target) + return target diff --git a/rfs/composition/rebuild/__init__.py b/rfs/composition/rebuild/__init__.py new file mode 100644 index 0000000..fbbc87f --- /dev/null +++ b/rfs/composition/rebuild/__init__.py @@ -0,0 +1,7 @@ +"""Canonical editable-rebuild composition APIs.""" + +from .arrows import style_and_route_arrows +from .program import build_figure_program +from .text import build_text_layer + +__all__ = ["build_figure_program", "build_text_layer", "style_and_route_arrows"] diff --git a/rfs/composition/rebuild/arrows.py b/rfs/composition/rebuild/arrows.py new file mode 100644 index 0000000..06153ad --- /dev/null +++ b/rfs/composition/rebuild/arrows.py @@ -0,0 +1,707 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ...utils import write_json + +ROUTER_VERSION = "reference-constrained-orthogonal-v1" +AESTHETIC_ROUTER_VERSION = "reference-tunnel-aesthetic-v2" +DEFAULT_REFERENCE_TUNNEL_PERCENT = 0.024 + + +def _bbox_center(bbox: dict[str, Any]) -> list[float]: + return [ + round(float(bbox["x"]) + float(bbox["w"]) / 2, 4), + round(float(bbox["y"]) + float(bbox["h"]) / 2, 4), + ] + + +def _anchor_point(obj: dict | None, anchor: str) -> list[float] | None: + if not obj or not isinstance(obj.get("bbox_percent"), dict): + return None + bbox = obj["bbox_percent"] + x = float(bbox["x"]) + y = float(bbox["y"]) + w = float(bbox["w"]) + h = float(bbox["h"]) + points = { + "left_mid": [x, y + h / 2], + "right_mid": [x + w, y + h / 2], + "top_mid": [x + w / 2, y], + "bottom_mid": [x + w / 2, y + h], + "center": [x + w / 2, y + h / 2], + } + point = points.get(anchor) or points["center"] + return [round(point[0], 4), round(point[1], 4)] + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _bbox_center(sbox) + tx, ty = _bbox_center(tbox) + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + s_anchor = "right_mid" if dx >= 0 else "left_mid" + t_anchor = "left_mid" if dx >= 0 else "right_mid" + else: + s_anchor = "bottom_mid" if dy >= 0 else "top_mid" + t_anchor = "top_mid" if dy >= 0 else "bottom_mid" + s = _anchor_point(source, s_anchor) + t = _anchor_point(target, t_anchor) + return [s, t] if s and t else [] + + +def _clean_path(points: list[list[float] | None]) -> list[list[float]]: + cleaned: list[list[float]] = [] + for point in points: + if not point or len(point) < 2: + continue + x = min(1.0, max(0.0, float(point[0]))) + y = min(1.0, max(0.0, float(point[1]))) + rounded = [round(x, 4), round(y, 4)] + if not cleaned or abs(cleaned[-1][0] - rounded[0]) > 0.001 or abs(cleaned[-1][1] - rounded[1]) > 0.001: + cleaned.append(rounded) + if len(cleaned) <= 2: + return cleaned + simplified = [cleaned[0]] + for index in range(1, len(cleaned) - 1): + prev = simplified[-1] + cur = cleaned[index] + nxt = cleaned[index + 1] + same_x = abs(prev[0] - cur[0]) < 0.001 and abs(cur[0] - nxt[0]) < 0.001 + same_y = abs(prev[1] - cur[1]) < 0.001 and abs(cur[1] - nxt[1]) < 0.001 + if not (same_x or same_y): + simplified.append(cur) + simplified.append(cleaned[-1]) + return simplified + + +def _segments(points: list[list[float]]) -> list[tuple[list[float], list[float]]]: + return [(points[i], points[i + 1]) for i in range(max(0, len(points) - 1))] + + +def _path_length(points: list[list[float]]) -> float: + total = 0.0 + for a, b in _segments(points): + total += ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 + return round(total, 4) + + +def _path_normal(points: list[list[float]]) -> list[float]: + if len(points) < 2: + return [0.0, 0.0] + dx = float(points[-1][0]) - float(points[0][0]) + dy = float(points[-1][1]) - float(points[0][1]) + length = (dx * dx + dy * dy) ** 0.5 + if length < 0.0001: + return [0.0, 0.0] + return [-dy / length, dx / length] + + +def _point_distance(a: list[float], b: list[float]) -> float: + return ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 + + +def _point_segment_distance(point: list[float], a: list[float], b: list[float]) -> float: + px, py = float(point[0]), float(point[1]) + ax, ay = float(a[0]), float(a[1]) + bx, by = float(b[0]), float(b[1]) + dx = bx - ax + dy = by - ay + denom = dx * dx + dy * dy + if denom < 0.0000001: + return _point_distance(point, a) + t = max(0.0, min(1.0, ((px - ax) * dx + (py - ay) * dy) / denom)) + proj = [ax + t * dx, ay + t * dy] + return _point_distance(point, proj) + + +def _max_distance_to_polyline(points: list[list[float]], reference: list[list[float]]) -> float: + segments = _segments(reference) + if not segments: + return 0.0 + distances = [] + for point in points: + distances.append(min(_point_segment_distance(point, a, b) for a, b in segments)) + return round(max(distances, default=0.0), 4) + + +def _lane_offset(lane_index: int, lane_count: int, tunnel_percent: float) -> float: + if lane_count <= 1: + return 0.0 + centered = float(lane_index) - (float(lane_count) - 1.0) / 2.0 + step = min(0.006, max(0.002, tunnel_percent / max(lane_count, 1))) + return max(-tunnel_percent * 0.82, min(tunnel_percent * 0.82, centered * step)) + + +def _shift_point(point: list[float], normal: list[float], offset: float) -> list[float]: + return [ + round(min(1.0, max(0.0, float(point[0]) + normal[0] * offset)), 4), + round(min(1.0, max(0.0, float(point[1]) + normal[1] * offset)), 4), + ] + + +def _aesthetic_tunnel_path( + points: list[list[float]], + lane_index: int, + lane_count: int, + role: str, + tunnel_percent: float = DEFAULT_REFERENCE_TUNNEL_PERCENT, +) -> tuple[list[list[float]], dict[str, Any]]: + original = _clean_path(points) + if len(original) < 2: + return original, { + "routing_algorithm": AESTHETIC_ROUTER_VERSION, + "route_generation_status": "aesthetic_no_path", + "reference_tunnel_percent": tunnel_percent, + "reference_path_delta_max": 0.0, + "reference_tunnel_preserved": True, + } + normal = _path_normal(original) + offset = _lane_offset(lane_index, lane_count, tunnel_percent) + adjusted = [list(point) for point in original] + if abs(offset) > 0.0001 and len(original) == 2: + mid = [ + (float(original[0][0]) + float(original[1][0])) / 2.0, + (float(original[0][1]) + float(original[1][1])) / 2.0, + ] + adjusted = [original[0], _shift_point(mid, normal, offset), original[-1]] + elif role in {"main_flow", "module_flow"}: + # Single-lane arrows keep the exact reference geometry; the PPT renderer + # still softens them through curve connectors, halo, and line caps. + adjusted = original + adjusted = _clean_path(adjusted) + delta = _max_distance_to_polyline(adjusted, original) + return adjusted, { + "routing_algorithm": AESTHETIC_ROUTER_VERSION, + "route_generation_status": "aesthetic_tunnel_adjusted" if adjusted != original else "aesthetic_style_only", + "reference_tunnel_percent": tunnel_percent, + "reference_path_delta_max": delta, + "reference_tunnel_preserved": delta <= tunnel_percent, + "lane_offset_percent": round(offset, 4), + } + + +def _orientation(a: list[float], b: list[float], c: list[float]) -> float: + return (float(b[1]) - float(a[1])) * (float(c[0]) - float(b[0])) - (float(b[0]) - float(a[0])) * (float(c[1]) - float(b[1])) + + +def _share_endpoint(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: + pairs = ((a1, b1), (a1, b2), (a2, b1), (a2, b2)) + return any(abs(p[0] - q[0]) < 0.002 and abs(p[1] - q[1]) < 0.002 for p, q in pairs) + + +def _segments_cross(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: + if _share_endpoint(a1, a2, b1, b2): + return False + # Fast reject by segment bounding boxes. + if max(min(a1[0], a2[0]), min(b1[0], b2[0])) > min(max(a1[0], a2[0]), max(b1[0], b2[0])): + return False + if max(min(a1[1], a2[1]), min(b1[1], b2[1])) > min(max(a1[1], a2[1]), max(b1[1], b2[1])): + return False + o1 = _orientation(a1, a2, b1) + o2 = _orientation(a1, a2, b2) + o3 = _orientation(b1, b2, a1) + o4 = _orientation(b1, b2, a2) + return o1 * o2 < 0 and o3 * o4 < 0 + + +def _point_inside_bbox(point: list[float], bbox: dict[str, Any], pad: float = 0.0) -> bool: + x = float(bbox["x"]) - pad + y = float(bbox["y"]) - pad + w = float(bbox["w"]) + pad * 2 + h = float(bbox["h"]) + pad * 2 + return x <= float(point[0]) <= x + w and y <= float(point[1]) <= y + h + + +def _segment_bbox_overlap(a: list[float], b: list[float], bbox: dict[str, Any], pad: float = 0.004) -> bool: + if _point_inside_bbox(a, bbox, pad=pad) or _point_inside_bbox(b, bbox, pad=pad): + return True + x0 = float(bbox["x"]) - pad + y0 = float(bbox["y"]) - pad + x1 = float(bbox["x"]) + float(bbox["w"]) + pad + y1 = float(bbox["y"]) + float(bbox["h"]) + pad + edges = [ + ([x0, y0], [x1, y0]), + ([x1, y0], [x1, y1]), + ([x1, y1], [x0, y1]), + ([x0, y1], [x0, y0]), + ] + return any(_segments_cross(a, b, e0, e1) for e0, e1 in edges) + + +def _candidate_lanes(source: dict, target: dict, obstacles: dict[str, dict], axis: str, limit: int = 18) -> list[float]: + sbox = source["bbox_percent"] + tbox = target["bbox_percent"] + if axis == "x": + base = [ + float(sbox["x"]), + float(sbox["x"]) + float(sbox["w"]), + float(sbox["x"]) + float(sbox["w"]) / 2, + float(tbox["x"]), + float(tbox["x"]) + float(tbox["w"]), + float(tbox["x"]) + float(tbox["w"]) / 2, + ] + for obj in obstacles.values(): + bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None + if not bbox: + continue + base.extend([float(bbox["x"]) - 0.014, float(bbox["x"]) + float(bbox["w"]) + 0.014]) + else: + base = [ + float(sbox["y"]), + float(sbox["y"]) + float(sbox["h"]), + float(sbox["y"]) + float(sbox["h"]) / 2, + float(tbox["y"]), + float(tbox["y"]) + float(tbox["h"]), + float(tbox["y"]) + float(tbox["h"]) / 2, + ] + for obj in obstacles.values(): + bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None + if not bbox: + continue + base.extend([float(bbox["y"]) - 0.014, float(bbox["y"]) + float(bbox["h"]) + 0.014]) + midpoint = (sum(base[:6]) / 6) if len(base) >= 6 else 0.5 + lanes = sorted({round(min(0.985, max(0.015, value)), 4) for value in base}, key=lambda value: abs(value - midpoint)) + return sorted(lanes[:limit]) + + +def _crossing_count(points: list[list[float]], existing_segments: list[tuple[list[float], list[float]]]) -> int: + count = 0 + for a, b in _segments(points): + for c, d in existing_segments: + if _segments_cross(a, b, c, d): + count += 1 + return count + + +def _obstacle_overlaps(points: list[list[float]], obstacles: dict[str, dict]) -> list[str]: + overlaps = [] + for obj_id, obj in obstacles.items(): + bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None + if not bbox: + continue + if any(_segment_bbox_overlap(a, b, bbox) for a, b in _segments(points)): + overlaps.append(obj_id) + return overlaps + + +def _score_candidate(points: list[list[float]], obstacles: dict[str, dict], existing_segments: list[tuple[list[float], list[float]]]) -> tuple[float, dict[str, Any]]: + overlaps = _obstacle_overlaps(points, obstacles) + crossings = _crossing_count(points, existing_segments) + bends = max(0, len(points) - 2) + length = _path_length(points) + near_edge_penalty = sum(1 for x, y in points if x < 0.012 or x > 0.988 or y < 0.012 or y > 0.988) + score = len(overlaps) * 1000 + crossings * 80 + bends * 8 + length * 16 + near_edge_penalty * 10 + return score, { + "candidate_score": round(score, 4), + "obstacle_overlap_count": len(overlaps), + "obstacle_overlap_ids": overlaps[:12], + "crossing_count": crossings, + "bend_count": bends, + "path_length": length, + } + + +def _best_fallback_path( + source: dict | None, + target: dict | None, + obstacles: dict[str, dict], + existing_segments: list[tuple[list[float], list[float]]], +) -> tuple[list[list[float]], dict[str, Any]]: + if not source or not target: + return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_source_or_target"} + direct = _edge_path(source, target) + if len(direct) < 2: + return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_anchor_points"} + + sx, sy = direct[0] + tx, ty = direct[-1] + x_lanes = _candidate_lanes(source, target, obstacles, "x") + y_lanes = _candidate_lanes(source, target, obstacles, "y") + candidates = [_clean_path(direct)] + candidates.extend(_clean_path([direct[0], [x, sy], [x, ty], direct[-1]]) for x in x_lanes) + candidates.extend(_clean_path([direct[0], [sx, y], [tx, y], direct[-1]]) for y in y_lanes) + # Two-lane orthogonal paths can move around large central obstacles without + # changing the source-target semantics from the reference-derived program. + for x in x_lanes[:12]: + for y in y_lanes[:12]: + candidates.append(_clean_path([direct[0], [x, sy], [x, y], [tx, y], direct[-1]])) + candidates.append(_clean_path([direct[0], [sx, y], [x, y], [x, ty], direct[-1]])) + + best_path: list[list[float]] = direct + best_meta: dict[str, Any] = {} + best_score = float("inf") + seen: set[tuple[tuple[float, float], ...]] = set() + for candidate in candidates: + if len(candidate) < 2: + continue + key = tuple((point[0], point[1]) for point in candidate) + if key in seen: + continue + seen.add(key) + score, meta = _score_candidate(candidate, obstacles, existing_segments) + if score < best_score: + best_score = score + best_path = candidate + best_meta = meta + best_meta.update({ + "routing_algorithm": ROUTER_VERSION, + "route_generation_status": "fallback_route_selected", + "candidate_count": len(seen), + }) + return best_path, best_meta + + +def _infer_role(arrow: dict, out_counts: dict[str, int], in_counts: dict[str, int], panel_ids: set[str]) -> str: + kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() + source = str(arrow.get("source_id") or arrow.get("source") or "") + target = str(arrow.get("target_id") or arrow.get("target") or "") + if "loop" in kind or "dashed" in kind: + return "feedback_loop" + # Branch/convergence semantics should win over panel membership. Some RFS + # programs use panel-like IDs for central modules, and treating those first + # collapses every connector into the same main-flow style. + if out_counts.get(source, 0) > 1: + return "branch" + if in_counts.get(target, 0) > 1: + return "convergence" + if source in panel_ids or target in panel_ids: + return "main_flow" + return "module_flow" + + +def _style_for_role(role: str, arrow: dict, mode: str = "reference") -> dict[str, Any]: + kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() + if str(mode).lower() == "aesthetic": + if role == "main_flow": + return {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "line_cap": "round", "halo_width_pt": 4.4, "halo_color": "#FFFFFF"} + if role == "branch": + return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} + if role == "convergence": + return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} + if role == "feedback_loop": + return {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} + if "elbow" in kind: + return {"route_style": "rounded_elbow", "stroke_width_pt": 1.6, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} + return {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} + if role == "main_flow": + return {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med", "line_cap": "round"} + if role == "branch": + return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} + if role == "convergence": + return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} + if role == "feedback_loop": + return {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash"} + if "elbow" in kind: + return {"route_style": "rounded_elbow", "stroke_width_pt": 1.65, "arrowhead_size": "sm", "line_cap": "round"} + return {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round"} + + +def _reference_locked(arrow: dict) -> bool: + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if len(path) < 2: + return False + source = str(arrow.get("binding_source") or arrow.get("detected_by") or "").lower() + if any(term in source for term in ("reference", "opencv", "vlm", "explicit", "candidate", "layout")): + return True + # In reference-primary mode, existing normalized paths are treated as a + # reference contract unless explicitly marked as fallback. + return str(arrow.get("route_policy", "")).lower() != "fallback_reroute_allowed" + + +def _route_metrics(arrows: list[dict], objects: dict[str, dict]) -> tuple[dict[str, dict], int]: + metrics: dict[str, dict] = {} + all_segments: list[tuple[str, list[float], list[float]]] = [] + for arrow in arrows: + points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + sid = str(arrow.get("source_id") or arrow.get("source") or "") + tid = str(arrow.get("target_id") or arrow.get("target") or "") + obstacles = [] + for obj_id, obj in objects.items(): + if obj_id in {sid, tid} or not isinstance(obj.get("bbox_percent"), dict): + continue + if any(_segment_bbox_overlap(a, b, obj["bbox_percent"]) for a, b in _segments(points)): + obstacles.append(obj_id) + metrics[str(arrow.get("id"))] = { + "path_length": _path_length(points), + "bend_count": max(0, len(points) - 2), + "obstacle_overlap_count": len(obstacles), + "obstacle_overlap_ids": obstacles[:12], + "point_count": len(points), + } + for a, b in _segments(points): + all_segments.append((str(arrow.get("id")), a, b)) + + crossing_count = 0 + crossings_by_arrow = {str(arrow.get("id")): 0 for arrow in arrows} + for idx, (aid, a1, a2) in enumerate(all_segments): + for bid, b1, b2 in all_segments[idx + 1:]: + if aid == bid: + continue + if _segments_cross(a1, a2, b1, b2): + crossing_count += 1 + crossings_by_arrow[aid] = crossings_by_arrow.get(aid, 0) + 1 + crossings_by_arrow[bid] = crossings_by_arrow.get(bid, 0) + 1 + for aid, count in crossings_by_arrow.items(): + if aid in metrics: + metrics[aid]["crossing_count"] = count + return metrics, crossing_count + + +def style_and_route_arrows( + program: dict, + out_dir: str | Path, + mode: str = "reference", +) -> dict: + """Add reference-preserving arrow aesthetics and QA artifacts. + + The reference image remains the hard constraint. Existing reference-derived + routes are not freely rerouted; this stage mostly adds style, bundling + metadata, and diagnostics. Only missing/fallback paths are synthesized. + """ + out = Path(out_dir) + if str(mode).lower() == "off": + write_json(out / "arrow_style_profile.json", {"summary": "Arrow styling skipped.", "mode": "off"}) + write_json(out / "selected_arrow_routes.json", {"summary": "Arrow routing skipped.", "mode": "off", "routes": []}) + write_json(out / "arrow_quality_report.json", {"summary": "Arrow quality skipped.", "mode": "off", "status": "skipped"}) + return program + + panels = program.get("panels", []) if isinstance(program.get("panels"), list) else [] + slots = program.get("slots", []) if isinstance(program.get("slots"), list) else [] + program_style = program.get("style") if isinstance(program.get("style"), dict) else {} + color_tokens = program_style.get("color_tokens", []) if isinstance(program_style.get("color_tokens"), list) else [] + default_style_token_id = next( + ( + str(item.get("token_id") or "") + for item in color_tokens + if isinstance(item, dict) and any(term in str(item.get("usage") or "").casefold() for term in ("arrow", "connector", "line")) + ), + "", + ) + if not default_style_token_id: + default_style_token_id = next((str(item.get("token_id") or "") for item in color_tokens if isinstance(item, dict) and item.get("token_id")), "") + objects = {str(item.get("id")): item for item in panels + slots if isinstance(item, dict) and item.get("id")} + slot_objects = {str(item.get("id")): item for item in slots if isinstance(item, dict) and item.get("id")} + panel_ids = {str(panel.get("id")) for panel in panels if isinstance(panel, dict)} + arrows = [dict(item) for item in program.get("arrows", []) if isinstance(item, dict)] + out_counts: dict[str, int] = {} + in_counts: dict[str, int] = {} + for arrow in arrows: + source = str(arrow.get("source_id") or arrow.get("source") or "") + target = str(arrow.get("target_id") or arrow.get("target") or "") + out_counts[source] = out_counts.get(source, 0) + 1 + in_counts[target] = in_counts.get(target, 0) + 1 + + reference_segments: list[tuple[list[float], list[float]]] = [] + for original in arrows: + original_points = original.get("path_percent") if isinstance(original.get("path_percent"), list) else [] + if _reference_locked(original) and len(original_points) >= 2: + reference_segments.extend(_segments(_clean_path(original_points))) + + grouped: dict[str, list[dict]] = {} + enriched = [] + committed_fallback_segments: list[tuple[list[float], list[float]]] = [] + for index, arrow in enumerate(arrows, start=1): + source = str(arrow.get("source_id") or arrow.get("source") or "") + target = str(arrow.get("target_id") or arrow.get("target") or "") + arrow["source_id"] = source + arrow["target_id"] = target + arrow["source"] = source + arrow["target"] = target + if not str(arrow.get("style_token_id") or "").strip(): + arrow["style_token_id"] = default_style_token_id + points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + locked = _reference_locked(arrow) + route_meta: dict[str, Any] = {"routing_algorithm": "preserve_reference_path", "route_generation_status": "reference_locked"} + fallback_allowed = str(arrow.get("route_policy", "")).lower() == "fallback_reroute_allowed" + if (len(points) < 2 or (fallback_allowed and not locked)) and mode in {"reference", "aesthetic"}: + obstacles = {obj_id: obj for obj_id, obj in slot_objects.items() if obj_id not in {source, target}} + points, route_meta = _best_fallback_path( + objects.get(source), + objects.get(target), + obstacles, + reference_segments + committed_fallback_segments, + ) + locked = False + else: + points = _clean_path(points) + arrow["path_percent"] = [[round(float(p[0]), 4), round(float(p[1]), 4)] for p in points if isinstance(p, list) and len(p) >= 2] + role = _infer_role(arrow, out_counts, in_counts, panel_ids) + style = _style_for_role(role, arrow, mode=mode) + recompute_generated_style = str(arrow.get("aesthetic_policy", "")).startswith("reference_first") + for key, value in style.items(): + if recompute_generated_style or key not in arrow or not str(arrow.get(key, "")).strip(): + arrow[key] = value + if recompute_generated_style or not str(arrow.get("semantic_role", "")).strip(): + arrow["semantic_role"] = role + arrow.setdefault("aesthetic_policy", "reference_first_soft_editable_ppt_connector") + arrow.setdefault("reference_locked", locked) + arrow.setdefault("reference_path_preserved", locked) + arrow.setdefault("route_policy", "preserve_reference_path" if locked else "synthesize_missing_path_only") + arrow.setdefault("routing_algorithm", route_meta.get("routing_algorithm", ROUTER_VERSION)) + arrow.setdefault("route_generation_status", route_meta.get("route_generation_status", "fallback_route_selected" if not locked else "reference_locked")) + if "candidate_count" in route_meta: + arrow.setdefault("candidate_count", route_meta["candidate_count"]) + if "candidate_score" in route_meta: + arrow.setdefault("candidate_score", route_meta["candidate_score"]) + arrow.setdefault("corner_radius_percent", 0.018 if arrow.get("route_style") in {"rounded_elbow", "bundled_elbow"} else 0.0) + if role == "branch": + bundle_id = f"from_{source}" + elif role == "convergence": + bundle_id = f"to_{target}" + elif role == "feedback_loop": + bundle_id = f"loop_{source}_{target}" + else: + bundle_id = f"flow_{index:02d}" + arrow.setdefault("bundle_id", bundle_id) + grouped.setdefault(bundle_id, []).append(arrow) + enriched.append(arrow) + if not locked and len(arrow["path_percent"]) >= 2: + committed_fallback_segments.extend(_segments(arrow["path_percent"])) + + for bundle_id, items in grouped.items(): + items.sort(key=lambda item: (item.get("target_id", ""), item.get("id", ""))) + for lane_index, arrow in enumerate(items): + arrow["lane_index"] = lane_index + arrow["lane_count"] = len(items) + + if str(mode).lower() == "aesthetic": + tunnel_percent = DEFAULT_REFERENCE_TUNNEL_PERCENT + for arrow in enriched: + original_path = _clean_path(arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else []) + arrow["reference_original_path_percent"] = original_path + if arrow.get("reference_locked") and len(original_path) >= 2: + offset_allowed = bool(arrow.get("aesthetic_offset_allowed")) or str(arrow.get("route_policy", "")).lower() == "aesthetic_tunnel_allowed" + if offset_allowed: + adjusted_path, aesthetic_meta = _aesthetic_tunnel_path( + original_path, + int(arrow.get("lane_index") or 0), + int(arrow.get("lane_count") or 1), + str(arrow.get("semantic_role") or ""), + tunnel_percent=tunnel_percent, + ) + arrow["path_percent"] = adjusted_path + arrow["routing_algorithm"] = aesthetic_meta["routing_algorithm"] + arrow["route_generation_status"] = aesthetic_meta["route_generation_status"] + arrow["reference_tunnel_percent"] = aesthetic_meta["reference_tunnel_percent"] + arrow["reference_path_delta_max"] = aesthetic_meta["reference_path_delta_max"] + arrow["reference_tunnel_preserved"] = aesthetic_meta["reference_tunnel_preserved"] + arrow["lane_offset_percent"] = aesthetic_meta["lane_offset_percent"] + arrow["reference_path_preserved"] = bool(aesthetic_meta["reference_tunnel_preserved"]) + else: + arrow["routing_algorithm"] = AESTHETIC_ROUTER_VERSION + arrow["route_generation_status"] = "aesthetic_style_only" + arrow["reference_tunnel_percent"] = tunnel_percent + arrow["reference_path_delta_max"] = 0.0 + arrow["reference_tunnel_preserved"] = True + arrow["lane_offset_percent"] = 0.0 + arrow["reference_path_preserved"] = True + + metrics, total_crossings = _route_metrics(enriched, slot_objects) + routes = [] + for arrow in enriched: + aid = str(arrow.get("id")) + m = metrics.get(aid, {}) + clutter_score = ( + int(m.get("crossing_count", 0)) * 12 + + int(m.get("bend_count", 0)) * 3 + + int(m.get("obstacle_overlap_count", 0)) * 18 + ) + arrow["aesthetic_score"] = max(0, 100 - clutter_score) + routes.append({ + "id": aid, + "source_id": arrow.get("source_id"), + "target_id": arrow.get("target_id"), + "semantic_role": arrow.get("semantic_role"), + "route_style": arrow.get("route_style"), + "bundle_id": arrow.get("bundle_id"), + "lane_index": arrow.get("lane_index"), + "lane_count": arrow.get("lane_count"), + "reference_locked": arrow.get("reference_locked"), + "reference_path_preserved": arrow.get("reference_path_preserved"), + "path_percent": arrow.get("path_percent"), + "style_token_id": arrow.get("style_token_id"), + "stroke_width_pt": arrow.get("stroke_width_pt"), + "arrowhead_size": arrow.get("arrowhead_size"), + "line_cap": arrow.get("line_cap"), + "line_pattern": arrow.get("line_pattern", "solid"), + "routing_algorithm": arrow.get("routing_algorithm"), + "route_generation_status": arrow.get("route_generation_status"), + "reference_original_path_percent": arrow.get("reference_original_path_percent"), + "reference_tunnel_percent": arrow.get("reference_tunnel_percent"), + "reference_path_delta_max": arrow.get("reference_path_delta_max"), + "reference_tunnel_preserved": arrow.get("reference_tunnel_preserved"), + "lane_offset_percent": arrow.get("lane_offset_percent"), + "halo_width_pt": arrow.get("halo_width_pt"), + "halo_color": arrow.get("halo_color"), + "candidate_count": arrow.get("candidate_count"), + "candidate_score": arrow.get("candidate_score"), + "metrics": m, + "aesthetic_score": arrow["aesthetic_score"], + }) + + arrow_style_profile = { + "summary": "Reference-first arrow styling profile for editable PPT connectors.", + "mode": mode, + "reference_priority": "reference_image_hard_constraint", + "routing_principle": "preserve reference-derived source-target logic and path geometry; only synthesize missing fallback paths", + "routing_algorithm": ROUTER_VERSION, + "aesthetic_routing_algorithm": AESTHETIC_ROUTER_VERSION, + "reference_tunnel_percent": DEFAULT_REFERENCE_TUNNEL_PERCENT, + "fallback_routing_policy": "only arrows with missing paths or route_policy=fallback_reroute_allowed may use obstacle-aware orthogonal routing", + "aesthetic_mode_policy": "mode=aesthetic may adjust reference-locked paths only inside the reference tunnel and must keep reference_tunnel_preserved=true", + "style_rules": { + "main_flow": {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med"}, + "branch": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, + "convergence": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, + "feedback_loop": {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm"}, + "module_flow": {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm"}, + }, + "aesthetic_style_rules": { + "main_flow": {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "halo_width_pt": 4.4}, + "branch": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, + "convergence": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, + "feedback_loop": {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "halo_width_pt": 3.0}, + "module_flow": {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "halo_width_pt": 3.0}, + }, + "ppt_editability": "all arrows render as PPT connector shapes, not raster assets", + "rounded_line_policy": "use round line caps and editable connector segments; reference-locked paths are not geometrically rewritten", + } + selected_routes = { + "summary": "Selected reference-preserving arrow routes and style assignments.", + "mode": mode, + "route_count": len(routes), + "routes": routes, + } + tunnel_violations = [route["id"] for route in routes if route.get("reference_tunnel_preserved") is False] + quality_report = { + "summary": "Arrow routing and styling quality report.", + "mode": mode, + "status": "pass" if all(route["aesthetic_score"] >= 35 for route in routes) and not tunnel_violations else "needs_review", + "arrow_count": len(routes), + "total_crossing_count": total_crossings, + "total_obstacle_overlap_count": sum(int(route["metrics"].get("obstacle_overlap_count", 0)) for route in routes), + "fallback_route_count": sum(1 for route in routes if route.get("route_generation_status") == "fallback_route_selected"), + "aesthetic_tunnel_adjusted_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_tunnel_adjusted"), + "aesthetic_style_only_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_style_only"), + "reference_tunnel_violations": tunnel_violations, + "average_aesthetic_score": round(sum(float(route["aesthetic_score"]) for route in routes) / max(len(routes), 1), 2), + "reference_path_overrides": [route["id"] for route in routes if route.get("reference_locked") and not route.get("reference_path_preserved")], + "routes": routes, + } + program["arrows"] = enriched + program["control_shapes"] = [dict(item) for item in enriched] + program.setdefault("style", {})["arrow_style_profile_path"] = "arrow_style_profile.json" + write_json(out / "arrow_style_profile.json", arrow_style_profile) + write_json(out / "selected_arrow_routes.json", selected_routes) + write_json(out / "arrow_quality_report.json", quality_report) + write_json(out / "figure_program.json", program) + return program diff --git a/rfs/composition/rebuild/export.py b/rfs/composition/rebuild/export.py new file mode 100644 index 0000000..f4b8a7a --- /dev/null +++ b/rfs/composition/rebuild/export.py @@ -0,0 +1,112 @@ +"""Canonical rebuild compile-only and preview export operations.""" + +from __future__ import annotations + +import json +import platform +from pathlib import Path + +from ..pptx import compile_ppt +from ..preview import render_rebuild_preview +from ...utils import read_json, write_json, write_text + + +def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: + x = max(0.0, min(0.995, x)) + y = max(0.0, min(0.995, y)) + w = max(0.001, min(w, 1.0 - x)) + h = max(0.001, min(h, 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: + ratio = width_px / max(height_px, 1) + width = max(10.0, min(15.6, 7.5 * ratio)) + return round(width, 3), round(width / ratio, 3) + + +def _style_from_layout(layout: dict) -> dict: + palette = layout.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52", "#6B57C8", "#D44E5D"] + background = str(layout.get("canvas", {}).get("background") or "#FFFFFF") + style = { + "palette": palette, + "reference_palette": palette, + "color_tokens": [{"token_id": f"palette_{i:02d}", "hex": color} for i, color in enumerate(palette)], + "panel_styles": {"reference_canvas": {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}}, + "arrow_weight_pt": 1.7, + } + for panel in layout.get("panels", []): + if panel.get("id") and panel["id"] != "reference_canvas": + style["panel_styles"].setdefault(str(panel["id"]), {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}) + return style + + +def _export_preview(pptx_path: Path, out: Path) -> Path | None: + preview = out / "rebuild_preview.png" + if platform.system() != "Windows": + write_text(out / "preview_export_error.txt", "PowerPoint preview export is only available on Windows with Microsoft PowerPoint installed.") + program_path = out / "figure_program.json" + return render_rebuild_preview(read_json(program_path), out, preview) if program_path.exists() else None + try: + import win32com.client + app = win32com.client.Dispatch("PowerPoint.Application") + app.Visible = True + deck = app.Presentations.Open(str(pptx_path), WithWindow=False) + deck.Slides(1).Export(str(preview), "PNG") + deck.Close() + app.Quit() + return preview + except Exception as exc: + write_text(out / "preview_export_error.txt", str(exc)) + program_path = out / "figure_program.json" + return render_rebuild_preview(read_json(program_path), out, preview) if program_path.exists() else None + + +def _ppt_package_counts(pptx_path: Path, reference_path: Path) -> dict: + from pptx import Presentation + prs = Presentation(str(pptx_path)) + shapes = [shape for slide in prs.slides for shape in slide.shapes] + return {"ppt_page_count": len(prs.slides), "shape_count": len(shapes), "picture_count": sum(getattr(shape, "shape_type", None) == 13 for shape in shapes), "text_shape_count": sum(bool(getattr(shape, "has_text_frame", False) and shape.text_frame.text.strip()) for shape in shapes), "connector_count": sum("CONNECTOR" in str(getattr(shape, "shape_type", "")) or "LINE" in str(getattr(shape, "shape_type", "")) for shape in shapes), "contains_full_reference_image": False, "reference_path": str(reference_path)} + + +def _load_json_or_empty(path: Path) -> dict: + if not path.exists(): return {} + try: data = json.loads(path.read_text(encoding="utf-8")) + except Exception: return {} + return data if isinstance(data, dict) else {} + + +def _program_from_contracts(out: Path) -> dict: + geometry = _load_json_or_empty(out / "reference_geometry.json") + controls = _load_json_or_empty(out / "reference_controls.json") + text_program = _load_json_or_empty(out / "text_program.json") + canvas = geometry.get("canvas") if isinstance(geometry.get("canvas"), dict) else {"width_in": 10.0, "height_in": 5.625, "background": "#FFFFFF"} + if "width_in" not in canvas or "height_in" not in canvas: + width_px = int(canvas.get("width_px") or 1600) + height_px = int(canvas.get("height_px") or 900) + width_in, height_in = _canvas_inches(width_px, height_px) + canvas = {**canvas, "width_in": width_in, "height_in": height_in, "background": canvas.get("background") or "#FFFFFF"} + layout = {"canvas": canvas, "palette": geometry.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52"], "panels": geometry.get("panels") or [{"id": "reference_canvas", "title": "Editable Rebuild", "bbox_percent": _bbox(0.025, 0.065, 0.95, 0.84), "editable_in": "pptx"}]} + slots = geometry.get("slots") or _load_json_or_empty(out / "slot_inventory.json").get("slots") or [] + program = {"canvas": canvas, "style": _style_from_layout(layout), "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, "panels": layout["panels"], "cards": geometry.get("cards") or [], "slots": slots, "assets": [{"id": slot.get("asset_id") or slot.get("id"), "path": f"assets/{slot.get('asset_id') or slot.get('id')}.png", "source": "slot_asset"} for slot in slots], "arrows": controls.get("arrows") or [], "labels": [], "groups": [], "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}]} + if isinstance(text_program.get("items"), list): + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + return program + + +def _compile_existing_contracts(reference_path: Path, out: Path, export_preview: bool = False) -> dict: + program = _program_from_contracts(out) + write_json(out / "figure_program.json", program) + pptx_path = compile_ppt(program, out) + preview = _export_preview(pptx_path, out) if export_preview else None + counts = _ppt_package_counts(pptx_path, reference_path) + report_path = out / "composition_quality_report.json" + report = _load_json_or_empty(report_path) + report["rebuild_editable_summary"] = counts + report["compile_only"] = True + write_json(report_path, report) + return {"summary": "Editable rebuild recompiled from existing JSON contracts.", "ok": True, "out_dir": str(out), "reference": str(reference_path), "pptx": str(pptx_path), "preview": str(preview) if preview else None, "asset_count": len(program.get("assets", [])), "slot_count": len(program.get("slots", [])), "connector_count": len(program.get("arrows", [])), "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, "compile_only": True} + + +__all__ = ["_compile_existing_contracts", "_export_preview", "_load_json_or_empty", "_ppt_package_counts", "_program_from_contracts"] diff --git a/rfs/composition/rebuild/program.py b/rfs/composition/rebuild/program.py new file mode 100644 index 0000000..a6b9879 --- /dev/null +++ b/rfs/composition/rebuild/program.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from ...analysis.rebuild.layout import plan_reference_layout +from ...utils import ratio_string, write_json + + +def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: + x = max(0.0, min(0.995, x)) + y = max(0.0, min(0.995, y)) + w = max(0.001, min(w, 1.0 - x)) + h = max(0.001, min(h, 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _style_from_layout(layout: dict) -> dict: + palette = layout.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52", "#6B57C8", "#D44E5D"] + background = str(layout.get("canvas", {}).get("background") or "#FFFFFF") + style = { + "palette": palette, + "reference_palette": palette, + "color_tokens": [{"token_id": f"palette_{i:02d}", "hex": color} for i, color in enumerate(palette)], + "panel_styles": {"reference_canvas": {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}}, + "arrow_weight_pt": 1.7, + } + for panel in layout.get("panels", []): + if panel.get("id") and panel["id"] != "reference_canvas": + style["panel_styles"].setdefault(str(panel["id"]), {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}) + return style + + +def _build_program(reference_path: Path, out: Path, layout_mode: str = "hybrid", vlm_layout_adapter: Callable | None = None, panel_layout_adapter: Callable | None = None, design_plan: dict | None = None, layout_adjudication_mode: str = "off", layout_adjudicator: Callable | None = None) -> tuple[dict, dict]: + """Create the editable-rebuild figure program from the layout contract.""" + layout = plan_reference_layout(reference_path, out, mode=layout_mode, vlm_adapter=vlm_layout_adapter, panel_layout_adapter=panel_layout_adapter, design_plan=design_plan, layout_adjudication_mode=layout_adjudication_mode, layout_adjudicator=layout_adjudicator) + return { + "canvas": layout["canvas"], "style": _style_from_layout(layout), + "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, + "panels": layout.get("panels", []), "cards": layout.get("cards", []), "slots": layout.get("slots", []), + "assets": [], "arrows": [], "labels": [], "groups": [], + "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}], + }, layout + + +def _center(bbox: dict) -> tuple[float, float]: + return float(bbox["x"]) + float(bbox["w"]) / 2, float(bbox["y"]) + float(bbox["h"]) / 2 + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _center(sbox) + tx, ty = _center(tbox) + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + start = [float(sbox["x"]) + (float(sbox["w"]) if dx >= 0 else 0.0), sy] + end = [float(tbox["x"]) if dx >= 0 else float(tbox["x"]) + float(tbox["w"]), ty] + else: + start = [sx, float(sbox["y"]) + (float(sbox["h"]) if dy >= 0 else 0.0)] + end = [tx, float(tbox["y"]) if dy >= 0 else float(tbox["y"]) + float(tbox["h"])] + return [[round(start[0], 4), round(start[1], 4)], [round(end[0], 4), round(end[1], 4)]] + + +def _control_record_from_arrow(arrow: dict) -> dict: + points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + xs = [float(point[0]) for point in points if isinstance(point, list) and len(point) >= 2] + ys = [float(point[1]) for point in points if isinstance(point, list) and len(point) >= 2] + if not xs or not ys: + xs, ys = [0.0, 0.001], [0.0, 0.001] + x0, x1 = min(xs), max(xs) + y0, y1 = min(ys), max(ys) + w = max(0.001, x1 - x0) + h = max(0.001, y1 - y0) + return { + "id": arrow.get("id"), + "type": "ppt_control", + "control_kind": arrow.get("control_kind") or arrow.get("type") or "straight_arrow", + "bbox_percent": {"x": round(x0, 4), "y": round(y0, 4), "w": round(w, 4), "h": round(h, 4)}, + "center_percent": {"x": round(x0 + w / 2, 4), "y": round(y0 + h / 2, 4)}, + "width_percent": round(w, 4), + "height_percent": round(h, 4), + "aspect_ratio_decimal": round(w / max(h, 0.001), 3), + "aspect_ratio_w_h": ratio_string(w, h), + "source_id": arrow.get("source_id") or arrow.get("source"), + "target_id": arrow.get("target_id") or arrow.get("target"), + "source_anchor": arrow.get("source_anchor") or "auto", + "target_anchor": arrow.get("target_anchor") or "auto", + "path_percent": points, + "style_token_id": arrow.get("style_token_id"), + "candidate_label": arrow.get("candidate_label"), + "detected_by": arrow.get("detected_by"), + "binding_source": arrow.get("binding_source"), + "confidence": arrow.get("confidence"), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + } + + +def build_figure_program(paper_brief: dict, inventory: dict, style: dict, out_dir: str | Path, layout_plan: dict) -> dict: + panels = layout_plan["panels"] + slots = layout_plan["slots"] + objects_by_id = {item.get("id"): item for item in panels + slots if isinstance(item, dict)} + color_tokens = style.get("color_tokens", []) if isinstance(style.get("color_tokens"), list) else [] + default_arrow_token = next((str(item.get("token_id")) for item in color_tokens if "arrow" in str(item.get("usage", "")).lower() or "connector" in str(item.get("usage", "")).lower()), "") + if not default_arrow_token and color_tokens: + default_arrow_token = str(color_tokens[0].get("token_id") or "") + arrows = [] + for item in layout_plan.get("arrows", []): + if not isinstance(item, dict): + continue + arrow = dict(item) + arrow["source"] = arrow.get("source") or arrow.get("source_id") or "" + arrow["target"] = arrow.get("target") or arrow.get("target_id") or "" + arrow["source_id"] = arrow.get("source_id") or arrow.get("source") + arrow["target_id"] = arrow.get("target_id") or arrow.get("target") + arrow["source_anchor"] = arrow.get("source_anchor") or "auto" + arrow["target_anchor"] = arrow.get("target_anchor") or "auto" + arrow["editable_in"] = "pptx" + arrow["render_policy"] = arrow.get("render_policy") or "ppt_shape_not_image_asset" + arrow["style_token_id"] = arrow.get("style_token_id") or default_arrow_token + if not arrow.get("path_percent"): + arrow["path_percent"] = _edge_path(objects_by_id.get(arrow["source"]), objects_by_id.get(arrow["target"])) + arrows.append(arrow) + + assets = [ + { + "id": slot["asset_id"], + "slot_id": slot["id"], + "path": f"assets/{slot['asset_id']}.png", + "source": "slot_image_generation", + "fit_policy": "contain_no_crop", + "z_index": slot.get("z_index", 20), + "reference_crop_path": slot.get("reference_crop_path"), + "local_color_token_ids": slot.get("local_color_token_ids", []), + "style_profile_path": style.get("reference_style_profile_path", "reference_style_profile.json"), + } + for slot in slots + ] + + labels = [] + for panel in panels: + labels.append({ + "id": f"label_{panel['id']}", + "text": panel["title"], + "target_id": panel["id"], + "role": "panel_title", + "editable_in": "pptx", + }) + for slot in slots: + labels.append({ + "id": f"label_{slot['id']}", + "text": slot["paper_concept"], + "target_id": slot["id"], + "role": "slot_caption", + "editable_in": "pptx", + }) + + canvas_ratio = float(inventory.get("canvas_aspect_ratio") or 16 / 9) + height_in = 7.5 + width_in = height_in * canvas_ratio + if width_in > 16.0: + width_in = 16.0 + height_in = width_in / canvas_ratio + program = { + "summary": "Structured figure program used as the only layout source for PPT composition.", + "canvas": {"width_in": round(width_in, 3), "height_in": height_in, "ratio": round(canvas_ratio, 4), "background": "#FFFFFF"}, + "paper_brief": { + "title_guess": paper_brief.get("title_guess"), + "figure_goal": paper_brief.get("figure_goal"), + "variables": paper_brief.get("variables", []), + }, + "style": { + "palette": style.get("palette", []), + "reference_palette": style.get("reference_palette", []), + "color_tokens": style.get("color_tokens", []), + "reference_style_profile_path": style.get("reference_style_profile_path", "reference_style_profile.json"), + "panel_styles": style.get("panel_styles", {}), + "slot_frame_policy": style.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": style.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + }, + "locator": { + "mode": layout_plan.get("locator_mode"), + "reference_path": layout_plan.get("reference_path") or inventory.get("reference_path"), + }, + "panels": panels, + "slots": slots, + "assets": assets, + "labels": labels, + "arrows": arrows, + "control_shapes": layout_plan.get("control_shapes", []) or [_control_record_from_arrow(arrow) for arrow in arrows], + "ppt_shapes": layout_plan.get("ppt_shapes", []), + "text_regions": layout_plan.get("text_regions", []), + "groups": [ + { + "id": "top_pipeline", + "members": [p["id"] for p in panels if p["id"] != "shared_resource_library"], + "editable_in": "pptx", + }, + { + "id": "shared_resources", + "members": [p["id"] for p in panels if p["id"] == "shared_resource_library"], + "editable_in": "pptx", + }, + ], + "export_targets": [ + {"type": "pptx", "path": "editable_composition.pptx", "role": "main_editable_source"}, + {"type": "pdf", "path": "review.pdf", "role": "review_export"}, + {"type": "png", "path": "final_600dpi.png", "role": "publication_preview"}, + ], + } + controls_doc = { + "summary": "Reference arrows, connectors, loops, and other non-image controls measured for editable PPT rendering.", + "reference_path": layout_plan.get("reference_path") or inventory.get("reference_path"), + "candidate_path": inventory.get("reference_control_candidates_path") or "reference_control_candidates.json", + "slot_overlay_path": inventory.get("slot_overlay_path"), + "control_overlay_path": inventory.get("reference_control_overlay_path"), + "requested_mode": (inventory.get("control_localizer") or {}).get("requested_mode"), + "effective_mode": (inventory.get("control_localizer") or {}).get("effective_mode"), + "controls": [_control_record_from_arrow(arrow) for arrow in arrows], + "ppt_arrows": [_control_record_from_arrow(arrow) for arrow in arrows], + "render_policy": "ppt_shape_not_image_asset", + } + write_json(Path(out_dir) / "reference_controls.json", controls_doc) + write_json(Path(out_dir) / "figure_program.json", program) + return program diff --git a/rfs/composition/rebuild/text.py b/rfs/composition/rebuild/text.py new file mode 100644 index 0000000..dc0cc07 --- /dev/null +++ b/rfs/composition/rebuild/text.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import math +from pathlib import Path +from typing import Callable + +from PIL import Image + +from ...analysis.rebuild.reference_text import extract_reference_text +from ...text_grouping import group_text_regions, write_text_grouping_artifacts +from ...text_layer_ownership import apply_text_layer_ownership, write_text_layer_ownership_artifacts +from ...utils import write_json + + +def _bbox_center(bbox: dict) -> dict: + x = float(bbox["x"]) + y = float(bbox["y"]) + w = float(bbox["w"]) + h = float(bbox["h"]) + return {"x": round(x + w / 2, 4), "y": round(y + h / 2, 4)} + + +def _clamp_bbox(bbox: dict) -> dict: + x = max(0.0, min(0.995, float(bbox["x"]))) + y = max(0.0, min(0.995, float(bbox["y"]))) + w = max(0.001, min(float(bbox["w"]), 1.0 - x)) + h = max(0.001, min(float(bbox["h"]), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _region_record(region_id: str, text: str, role: str, bbox: dict, color_hex: str, source: str, target_id: str) -> dict: + bbox = _clamp_bbox(bbox) + return { + "id": region_id, + "text": text, + "role": role, + "target_id": target_id, + "bbox_percent": bbox, + "center_percent": _bbox_center(bbox), + "width_percent": round(float(bbox["w"]), 4), + "height_percent": round(float(bbox["h"]), 4), + "estimated_font_ratio": round(float(bbox["h"]) * 0.62, 5), + "color_hex": color_hex, + "source": source, + "editable_in": "pptx", + } + + +def _rgb_tuple(hex_color: str) -> tuple[int, int, int]: + text = str(hex_color or "#000000").strip().lstrip("#") + if len(text) < 6: + return 0, 0, 0 + return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) + + +def _hex(rgb: tuple[int, int, int]) -> str: + return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" + + +def _luminance(rgb: tuple[int, int, int]) -> float: + return 0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2] + + +def _sample_text_color(reference_path: str | Path, bbox: dict, fallback: str = "#263747") -> str: + try: + with Image.open(reference_path) as image: + image = image.convert("RGB") + iw, ih = image.size + x0 = max(0, min(iw - 1, int(float(bbox["x"]) * iw))) + y0 = max(0, min(ih - 1, int(float(bbox["y"]) * ih))) + x1 = max(x0 + 1, min(iw, int((float(bbox["x"]) + float(bbox["w"])) * iw))) + y1 = max(y0 + 1, min(ih, int((float(bbox["y"]) + float(bbox["h"])) * ih))) + crop = image.crop((x0, y0, x1, y1)) + pixels = list(crop.getdata()) + except Exception: + return fallback + if not pixels: + return fallback + dark = [px for px in pixels if _luminance(px) < 215] + candidates = dark or pixels + candidates.sort(key=_luminance) + return _hex(candidates[len(candidates) // 3]) + + +def _nearest_token_id(style: dict, color_hex: str) -> str: + tokens = style.get("color_tokens", []) if isinstance(style.get("color_tokens"), list) else [] + if not tokens: + return "" + target = _rgb_tuple(color_hex) + best_id = "" + best_dist = math.inf + for token in tokens: + if not isinstance(token, dict): + continue + token_hex = str(token.get("hex") or "") + if not token_hex: + continue + rgb = _rgb_tuple(token_hex) + dist = sum((target[i] - rgb[i]) ** 2 for i in range(3)) + if dist < best_dist: + best_dist = dist + best_id = str(token.get("token_id") or "") + return best_id + + +def _short_label(text: str) -> str: + replacements = { + "ffmpeg video-to-audio extraction": "FFmpeg", + "funasr speech recognition microphone": "FunASR", + "funasr speech recognition": "FunASR", + "mtcnn face detection": "MTCNN", + "alphapose full-body skeleton extraction": "AlphaPose", + "frame sampling stack": "Frame sampling", + "frame sampling from video": "Frame sampling", + "audio waveform output": "audio waveform", + "spoken text with timestamps": "transcript + time", + "raw interview video recording": "Raw interview video", + "participant full-body video thumbnail grid": "Sampled full-body clips", + "face modality": "Face", + "frame modality": "Frame", + "pose modality": "Pose", + "audio modality": "Audio", + "text modality": "Text", + "neo-ffi-3 questionnaire checklist": "NEO-FFI-3 questionnaire", + "multimodal personality dataset database": "Multimodal signals + Big Five labels", + } + normalized = " ".join(str(text or "").strip().split()).lower() + if normalized in replacements: + return replacements[normalized] + raw = str(text or "").strip() + if len(raw) <= 24: + return raw + words = raw.replace("/", " ").split() + return " ".join(words[:3]) + + +def _slot_role(slot: dict) -> str: + text = f"{slot.get('id', '')} {slot.get('paper_concept', '')} {slot.get('display_label', '')}".lower() + if "legend" in text: + return "legend_label" + if "modality" in text: + return "modality_label" + if any(term in text for term in ("ffmpeg", "funasr", "mtcnn", "alphapose", "frame sampling", "waveform", "transcript")): + return "method_label" + if any(term in text for term in ("openness", "conscientious", "extraversion", "agreeableness", "neuroticism", "ocean")): + return "trait_label" + if bool(slot.get("show_slot_caption")) or str(slot.get("display_label", "")).strip(): + return "slot_caption" + return "hidden_slot_label" + + +def _slot_text_visible(slot: dict, role: str) -> bool: + if role == "hidden_slot_label": + return False + if bool(slot.get("show_slot_caption")) or str(slot.get("display_label", "")).strip(): + return True + return role in {"legend_label", "modality_label", "method_label", "trait_label"} + + +def _parent_panel(slot: dict, panels: dict[str, dict]) -> dict | None: + for key in ("parent_panel", "panel_id", "macro_panel_id"): + value = str(slot.get(key) or "") + if value in panels: + return panels[value] + return None + + +def _caption_bbox_for_slot(slot: dict, panels: dict[str, dict]) -> dict: + box = slot["bbox_percent"] + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + panel = _parent_panel(slot, panels) + role = _slot_role(slot) + if role in {"modality_label", "trait_label"} and panel: + pbox = panel["bbox_percent"] + label_x = x + w + 0.004 + panel_right = float(pbox["x"]) + float(pbox["w"]) + return _clamp_bbox({ + "x": label_x, + "y": y + h * 0.18, + "w": max(0.014, panel_right - label_x - 0.004), + "h": h * 0.58, + }) + label_h = max(0.012, min(0.026, h * 0.20)) + label_y = y + h + 0.006 + if panel: + pbox = panel["bbox_percent"] + panel_bottom = float(pbox["y"]) + float(pbox["h"]) + if label_y + label_h > panel_bottom - 0.004: + label_y = y - label_h - 0.004 + return _clamp_bbox({ + "x": x + w * 0.03, + "y": label_y, + "w": w * 0.94, + "h": label_h, + }) + + +def _panel_title_bbox(panel: dict, canvas_height_in: float) -> dict: + bbox = panel["bbox_percent"] + header_h = min(float(bbox["h"]) * 0.17, 0.34 / max(canvas_height_in, 0.001)) + return _clamp_bbox({"x": bbox["x"], "y": bbox["y"], "w": bbox["w"], "h": header_h}) + + +def _font_pt_from_reference_region(region: dict, canvas_height_in: float) -> float: + role = str(region.get("role") or "") + multiplier = 0.44 if role == "panel_title" else 0.62 + return round(max(1.0, float(region["height_percent"]) * canvas_height_in * 72 * multiplier), 2) + + +def _alignment_item(item: dict, region: dict) -> dict: + ib = item["bbox_percent"] + rb = region["bbox_percent"] + ic = item["center_percent"] + rc = region["center_percent"] + center_delta = math.sqrt((float(ic["x"]) - float(rc["x"])) ** 2 + (float(ic["y"]) - float(rc["y"])) ** 2) + width_delta = abs(float(ib["w"]) - float(rb["w"])) + height_delta = abs(float(ib["h"]) - float(rb["h"])) + font_delta = abs(float(item["estimated_font_ratio"]) - float(region["estimated_font_ratio"])) + status = "pass" + if center_delta > 0.05 or width_delta > 0.08 or height_delta > 0.06 or item.get("editable_in") != "pptx": + status = "fail" + return { + "label_id": item["id"], + "source_reference_text_id": region["id"], + "role": item["role"], + "center_delta_percent": round(center_delta, 5), + "width_delta_percent": round(width_delta, 5), + "height_delta_percent": round(height_delta, 5), + "font_ratio_delta": round(font_delta, 5), + "color_status": "reference_token" if item.get("color_token_id") else "sampled_reference_color", + "editable_in": item.get("editable_in"), + "status": status, + } + + +def _heuristic_regions(reference_path: str | Path, program: dict, canvas_height_in: float) -> list[dict]: + panels = {panel["id"]: panel for panel in program.get("panels", []) if isinstance(panel, dict) and panel.get("id")} + regions: list[dict] = [] + for panel in program.get("panels", []): + if not isinstance(panel, dict) or not panel.get("bbox_percent"): + continue + bbox = _panel_title_bbox(panel, canvas_height_in) + regions.append(_region_record( + f"ref_text_panel_{panel['id']}", + str(panel.get("title") or panel.get("paper_concept") or panel["id"]), + "panel_title", + bbox, + "#FFFFFF", + "reference_panel_header_geometry", + panel["id"], + )) + + for slot in program.get("slots", []): + if not isinstance(slot, dict) or not slot.get("bbox_percent"): + continue + role = _slot_role(slot) + if not _slot_text_visible(slot, role): + continue + label = str(slot.get("display_label") or "").strip() or _short_label(str(slot.get("paper_concept") or slot.get("id") or "")) + bbox = _caption_bbox_for_slot(slot, panels) + color = _sample_text_color(reference_path, bbox, fallback="#263747") + regions.append(_region_record( + f"ref_text_slot_{slot['id']}", + label, + role, + bbox, + color, + "reference_slot_text_region_inferred", + slot["id"], + )) + + for arrow in program.get("arrows", []): + label = str(arrow.get("label") or "").strip() + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if not label or len(path) < 2: + continue + xs = [float(point[0]) for point in path if isinstance(point, list) and len(point) >= 2] + ys = [float(point[1]) for point in path if isinstance(point, list) and len(point) >= 2] + if not xs or not ys: + continue + mid_x = (min(xs) + max(xs)) / 2 + mid_y = (min(ys) + max(ys)) / 2 + bbox = {"x": mid_x - 0.025, "y": mid_y - 0.01, "w": 0.05, "h": 0.02} + color = _sample_text_color(reference_path, bbox, fallback="#263747") + regions.append(_region_record( + f"ref_text_arrow_{arrow.get('id')}", + label, + "arrow_label", + bbox, + color, + "reference_arrow_label_region_inferred", + str(arrow.get("id") or ""), + )) + return regions + + +def _overlap(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + return (ix * iy) / max(float(b["w"]) * float(b["h"]), 0.000001) + + +def _merge_ocr_with_fallback(ocr_regions: list[dict], fallback_regions: list[dict]) -> list[dict]: + if not ocr_regions: + return fallback_regions + merged = list(ocr_regions) + for fallback in fallback_regions: + target_id = str(fallback.get("target_id") or "") + role = str(fallback.get("role") or "") + duplicate = False + for region in ocr_regions: + if str(region.get("target_id") or "") == target_id and str(region.get("role") or "") == role: + duplicate = True + break + if _overlap(region["bbox_percent"], fallback["bbox_percent"]) > 0.45: + duplicate = True + break + if not duplicate and role == "panel_title": + fallback = dict(fallback) + fallback["source"] = "reference_panel_header_geometry_fallback_after_ocr" + merged.append(fallback) + return merged + + +def build_text_layer( + reference_path: str | Path, + program: dict, + style: dict, + out_dir: str | Path, + text_extractor_mode: str = "ocr", + ocr_engine: str = "easyocr", + ocr_lang: str = "en_ch", + ocr_adapter: Callable[[str | Path, str], list[dict]] | None = None, + text_grouping_mode: str = "heuristic", + text_grouping_model: str | None = None, + text_grouping_adapter: Callable | None = None, +) -> dict: + """Build reference-first editable text artifacts and attach them to the program.""" + + out = Path(out_dir) + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + canvas_height_in = float(canvas.get("height_in") or 7.5) + + fallback_regions = _heuristic_regions(reference_path, program, canvas_height_in) + ocr_regions, ocr_report = extract_reference_text( + reference_path, + program, + mode=text_extractor_mode, + engine=ocr_engine, + lang=ocr_lang, + ocr_adapter=ocr_adapter, + ) + raw_geometry = { + "summary": "Raw line-level OCR geometry before paragraph grouping.", + "reference_path": str(reference_path), + "detection_mode": "ocr" if ocr_regions else "ocr_empty_or_unavailable", + "ocr_engine": ocr_engine, + "ocr_lang": ocr_lang, + "text_region_count": len(ocr_regions), + "text_regions": ocr_regions, + } + grouped_ocr_regions, grouping_plan, grouping_report = group_text_regions( + ocr_regions, + mode=text_grouping_mode if ocr_regions else "off", + adapter=text_grouping_adapter, + reference_path=reference_path, + program=program, + model=text_grouping_model, + ) + write_text_grouping_artifacts(out, raw_geometry, grouping_plan, grouping_report) + regions = _merge_ocr_with_fallback(grouped_ocr_regions, fallback_regions) + regions, ownership_plan, ownership_report = apply_text_layer_ownership(regions, program, mode="heuristic") + write_text_layer_ownership_artifacts(out, ownership_plan, ownership_report) + if not ocr_regions: + ocr_report["text_region_count"] = len(regions) + ocr_report.setdefault("warnings", []) + ocr_report["warnings"].append("using_heuristic_text_layer_fallback") + + items: list[dict] = [] + for region in regions: + if region.get("layer_ownership") != "editable_text_layer": + continue + font_size = float(region.get("font_size_pt") or _font_pt_from_reference_region(region, canvas_height_in)) + token_id = _nearest_token_id(style, region["color_hex"]) + item = { + "id": f"text_{region['id']}", + "text": region["text"], + "role": region["role"], + "target_id": region["target_id"], + "source_reference_text_id": region["id"], + "reference_binding": region["source"], + "bbox_percent": region["bbox_percent"], + "center_percent": region["center_percent"], + "width_percent": region["width_percent"], + "height_percent": region["height_percent"], + "estimated_font_ratio": region["estimated_font_ratio"], + "font_size_pt": font_size, + "color_hex": region["color_hex"], + "color_token_id": token_id, + "bold": str(region.get("font_weight_guess") or "").lower() == "bold" or region["role"] in {"panel_title", "method_label", "modality_label"}, + "align": "left" if region["role"] in {"modality_label", "trait_label", "arrow_label"} else "center", + "font_family_guess": region.get("font_family_guess") or ("Microsoft YaHei" if any(ord(char) > 127 for char in str(region.get("text", ""))) else "Arial"), + "font_weight_guess": region.get("font_weight_guess") or ("bold" if region["role"] in {"panel_title", "method_label", "modality_label"} else "regular"), + "fit_strategy": "ocr_bbox_exact" if str(region.get("source", "")).startswith("reference_ocr") else "reference_geometry_bbox", + "ocr_confidence": region.get("confidence"), + "editable_in": "pptx", + "visible": True, + "layer_ownership": region.get("layer_ownership"), + } + items.append(item) + + geometry = { + "summary": "Reference-first text geometry extracted or inferred from the reference layout; no publication font threshold is applied.", + "reference_path": str(reference_path), + "policy": "reference_image_is_highest_authority_for_text_size_position_color_and_hierarchy", + "detection_mode": "ocr" if ocr_regions else "reference_geometry_and_local_color_sampling", + "ocr_engine": ocr_engine, + "ocr_lang": ocr_lang, + "reference_text_geometry_raw_path": "reference_text_geometry_raw.json", + "text_grouping_plan_path": "text_grouping_plan.json", + "text_grouping_report_path": "text_grouping_report.json", + "text_grouping_mode": text_grouping_mode, + "text_grouping_status": grouping_report.get("status"), + "text_layer_ownership_plan_path": "text_layer_ownership_plan.json", + "text_layer_ownership_report_path": "text_layer_ownership_report.json", + "text_regions": regions, + } + text_program = { + "summary": "Reference-first editable PPT text program derived from reference_text_geometry.json.", + "policy": "match_reference_text_geometry; reference_layout_over_typography_defaults", + "reference_text_geometry_path": "reference_text_geometry.json", + "items": items, + } + region_by_id = {region["id"]: region for region in regions} + report_items = [_alignment_item(item, region_by_id[item["source_reference_text_id"]]) for item in items] + alignment_report = { + "summary": "Text alignment report comparing PPT text program geometry against reference-derived text geometry.", + "policy": "reference_alignment_only; no fixed typography failure condition", + "status": "pass" if all(item["status"] == "pass" for item in report_items) else "fail", + "max_center_delta_percent": max((item["center_delta_percent"] for item in report_items), default=0.0), + "max_width_delta_percent": max((item["width_delta_percent"] for item in report_items), default=0.0), + "max_height_delta_percent": max((item["height_delta_percent"] for item in report_items), default=0.0), + "items": report_items, + } + readability_note = { + "summary": "Optional readability note only; it does not override reference-first text sizing.", + "status": "informational_only", + "policy": "do_not_modify_text_size_unless_user_requests_readability_over_reference_matching", + "note": "Some labels may appear small after manuscript scaling if the reference figure uses small labels.", + } + + write_json(out / "reference_text_geometry.json", geometry) + write_json(out / "text_program.json", text_program) + write_json(out / "text_alignment_report.json", alignment_report) + write_json(out / "ocr_text_quality_report.json", ocr_report) + write_json(out / "publication_readability_note.json", readability_note) + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + program["reference_text_geometry_path"] = "reference_text_geometry.json" + program["text_alignment_report_path"] = "text_alignment_report.json" + write_json(out / "figure_program.json", program) + return program diff --git a/rfs/contracts/__init__.py b/rfs/contracts/__init__.py new file mode 100644 index 0000000..c866d03 --- /dev/null +++ b/rfs/contracts/__init__.py @@ -0,0 +1,5 @@ +"""Stable data-contract APIs shared across ResearchFigureStudio workflows.""" + +from .semantic import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract, load_paper_semantic_contract + +__all__ = ["apply_paper_semantic_contract", "compile_paper_brief_from_semantic_contract", "load_paper_semantic_contract"] diff --git a/rfs/contracts/semantic.py b/rfs/contracts/semantic.py new file mode 100644 index 0000000..9a9b6e7 --- /dev/null +++ b/rfs/contracts/semantic.py @@ -0,0 +1,1288 @@ +from __future__ import annotations + +import re +from itertools import combinations +from pathlib import Path +from typing import Any + +from ..utils import read_json, write_json, write_text + + +def load_paper_semantic_contract(run_dir: str | Path) -> dict[str, Any]: + root = Path(run_dir) + if root.is_file(): + payload = read_json(root) + if root.name == "figure_specification.json" or "panel_graphs" in payload or "relations" in payload: + return { + "summary": "Paper-grounded semantic contract loaded from a figure specification.", + "source_run_dir": str(root.parent.resolve()), + "figure_specification": payload, + } + return payload + + def optional(name: str) -> dict[str, Any]: + path = root / name + return read_json(path) if path.exists() else {} + + specification_path = root / "figure_specification.json" + if not specification_path.exists(): + raise FileNotFoundError(f"semantic contract directory has no figure_specification.json: {root}") + return { + "summary": "Paper-grounded semantic contract for editable figure reconstruction.", + "source_run_dir": str(root.resolve()), + "paper_review": optional("paper_review.json"), + "figure_specification": read_json(root / "figure_specification.json"), + "paper_summary": optional("paper_summary.json"), + "design_plan": optional("design_plan.json"), + "layout_intent": optional("layout_intent.json"), + "style_plan": optional("style_plan.json"), + } + + +def _text(item: dict, fallback: str = "") -> str: + for key in ("visible_label", "name", "text", "statement", "label", "id"): + value = str(item.get(key) or "").strip() + if value: + return value + return fallback + + +def _normalized_text(value: Any) -> str: + return re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "", str(value or "").casefold()) + + +def _specification(contract: dict) -> dict: + return contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + + +def _panel_graph_entities(spec: dict) -> list[dict]: + entities: list[dict] = [] + seen_ids: set[str] = set() + for panel_index, panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + continue + panel_id = str(panel.get("id") or f"panel_{panel_index + 1}").strip() + panel_label = str(panel.get("label") or panel.get("description") or panel_id).strip() + for node_index, raw in enumerate(panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []): + if not isinstance(raw, dict): + continue + role = str(raw.get("role") or "module").strip() + if role.casefold() in {"group", "container", "panel", "stage"}: + continue + label = _text(raw) + entity_id = str(raw.get("instance_id") or raw.get("id") or f"{panel_id}_node_{node_index + 1:02d}").strip() + if not label or not entity_id or entity_id in seen_ids: + continue + seen_ids.add(entity_id) + entities.append({ + "id": entity_id, + "semantic_id": str(raw.get("semantic_id") or "").strip() or None, + "label": label, + "role": role, + "panel_id": panel_id, + "panel_label": panel_label, + "bbox_percent": raw.get("bbox_percent"), + "evidence_ids": [str(value) for value in raw.get("evidence_ids", []) if value], + "source": raw, + }) + return entities + + +def _entities(contract: dict) -> list[dict]: + spec = _specification(contract) + panel_entities = _panel_graph_entities(spec) + if panel_entities: + return panel_entities + required_labels = { + re.sub(r"[^a-z0-9]+", "", str(value).casefold()) + for value in spec.get("required_labels", []) + if str(value).strip() + } + entities: list[dict] = [] + for field, role in (("inputs", "input"), ("modules", "module"), ("outputs", "output"), ("innovations", "innovation")): + values = spec.get(field) if isinstance(spec.get(field), list) else [] + for index, raw in enumerate(values, start=1): + if not isinstance(raw, dict): + continue + label = _text(raw) + if not label: + continue + if role == "innovation": + visible_required = bool(raw.get("visible_label_required") or raw.get("must_appear_in_figure")) + normalized_label = re.sub(r"[^a-z0-9]+", "", label.casefold()) + if not visible_required and normalized_label not in required_labels: + continue + entity_id = str(raw.get("id") or raw.get("name") or f"{role}_{index:02d}").strip() + entities.append({ + "id": entity_id, + "label": label, + "role": role, + "evidence_ids": list(raw.get("evidence_ids") or []), + "source": raw, + }) + return entities + + +def _relations(contract: dict) -> tuple[list[dict], str]: + spec = _specification(contract) + panel_entities = _panel_graph_entities(spec) + if panel_entities: + entity_ids = {item["id"] for item in panel_entities} + relations: list[dict] = [] + for panel_index, panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + continue + panel_id = str(panel.get("id") or f"panel_{panel_index + 1}").strip() + for relation_index, raw in enumerate(panel.get("relations", []) if isinstance(panel.get("relations"), list) else []): + if not isinstance(raw, dict): + continue + source = str(raw.get("source") or raw.get("source_id") or "").strip() + target = str(raw.get("target") or raw.get("target_id") or "").strip() + if not source or not target or source == target or source not in entity_ids or target not in entity_ids: + continue + relations.append({ + **raw, + "id": str(raw.get("id") or f"{panel_id}_relation_{relation_index + 1:02d}"), + "source": source, + "target": target, + "panel_id": panel_id, + }) + return relations, "panel_graphs" + return [item for item in spec.get("relations", []) if isinstance(item, dict)] if isinstance(spec.get("relations"), list) else [], "global_graph" + + +def _composition_type(role: str, label: str) -> str: + low = f"{role} {label}".casefold() + if any(token in low for token in ("input", "dataset", "sample", "image", "text", "audio", "video")): + return "scene_thumbnail" + if any(token in low for token in ("output", "prediction", "loss", "objective", "score", "distribution")): + return "full_frame_icon" + return "full_bleed_card" + + +def compile_paper_brief_from_semantic_contract( + contract: dict, + out_dir: str | Path | None = None, + *, + source_path: str | Path | None = None, +) -> dict[str, Any]: + """Compile the fast paper contract into the legacy brief shape without rereading the paper.""" + + spec = _specification(contract) + entities = _entities(contract) + relations, relation_source = _relations(contract) + panels = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict)] if isinstance(spec.get("panel_graphs"), list) else [] + modules = [str(item.get("label") or item.get("description") or item.get("id") or "").strip() for item in panels] + modules = [item for item in modules if item] + if not modules: + modules = list(dict.fromkeys(item["panel_label"] for item in entities if item.get("panel_label"))) + if not modules: + modules = list(dict.fromkeys(item["label"] for item in entities if item["role"].casefold() in {"module", "stage"})) + if not modules: + modules = ["Paper-grounded method"] + + outgoing: dict[str, list[str]] = {} + incoming: dict[str, list[str]] = {} + labels_by_id = {item["id"]: item["label"] for item in entities} + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + outgoing.setdefault(source, []).append(labels_by_id.get(target, target)) + incoming.setdefault(target, []).append(labels_by_id.get(source, source)) + + suggestions = [] + for entity in entities: + panel = entity.get("panel_label") or modules[0] + neighbor_cues = [*incoming.get(entity["id"], []), *outgoing.get(entity["id"], [])] + must_show = [f"paper-grounded visual object for {entity['label']}", "visible mechanism supported by cited paper evidence"] + if neighbor_cues: + must_show.append("declared local context: " + ", ".join(list(dict.fromkeys(neighbor_cues))[:3])) + suggestions.append({ + "id": entity["id"], + "macro_panel": panel, + "paper_concept": entity["label"], + "composition_type": _composition_type(entity["role"], entity["label"]), + "visual_metaphor": f"paper-grounded scientific visual for the paper term {entity['label']}", + "must_show": must_show, + "avoid_showing": ["unmentioned architecture modules", "fake formulas", "fake numeric chart text", "critical text baked into the image"], + "semantic_instance_id": entity["id"], + "semantic_id": entity.get("semantic_id"), + "semantic_panel_id": entity.get("panel_id"), + "semantic_role": entity["role"], + "evidence_ids": entity["evidence_ids"], + "contract_source": relation_source, + }) + + review = contract.get("paper_review") if isinstance(contract.get("paper_review"), dict) else {} + summary = contract.get("paper_summary") if isinstance(contract.get("paper_summary"), dict) else {} + title = str(review.get("title") or review.get("paper_title") or summary.get("title") or "Paper-grounded framework").strip() + goal = str(spec.get("figure_goal") or "").strip() + if not goal: + central_claim = spec.get("central_claim") + goal = _text(central_claim) if isinstance(central_claim, dict) else str(central_claim or "").strip() + if not goal: + goal = "Visualize only the modules and local relations declared in the paper semantic contract." + + brief = { + "summary": "Paper brief compiled directly from the fast evidence-grounded semantic contract.", + "planner": "semantic_contract", + "semantic_authority": relation_source, + "title_guess": title, + "figure_kind": str(spec.get("topology") or "paper_driven_generic"), + "figure_goal": goal, + "modules": modules[:8], + "concepts": [item["label"] for item in entities], + "variables": [str(item) for item in spec.get("required_labels", []) if str(item).strip()], + "slot_suggestions": suggestions, + "semantic_relations": relations, + "semantic_entity_count": len(entities), + "semantic_relation_count": len(relations), + "source_path": str(source_path or contract.get("source_run_dir") or ""), + "source_loader": "semantic_contract", + "char_count": 0, + "warnings": [] if relation_source == "panel_graphs" else ["panel_graphs unavailable; using the compatibility global graph"], + } + if out_dir is not None: + out = Path(out_dir) + write_json(out / "paper_brief.json", brief) + lines = [ + "# Summary", + "Paper brief compiled directly from the fast evidence-grounded semantic contract.", + "", + "## Source", + f"- Contract authority: {relation_source}", + f"- Source: `{brief['source_path']}`", + f"- Title: {title}", + "", + "## Figure Goal", + goal, + "", + "## Panels", + *[f"- {item}" for item in modules], + "", + "## Contract Entities", + *[f"- {item['id']}: {item['label']} ({item['role']})" for item in entities], + "", + "## Contract Relations", + *[f"- {item.get('source')} -> {item.get('target')} ({item.get('type') or 'data_flow'})" for item in relations], + ] + write_text(out / "paper_brief.md", "\n".join(lines) + "\n") + return brief + + +def _tokens(value: object) -> set[str]: + return {token for token in re.findall(r"[a-z0-9]+", str(value or "").lower()) if len(token) > 1} + + +def _object_text(obj: dict) -> str: + return " ".join( + str(obj.get(key) or "") + for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt", "prompt_subject", "display_label", "paper_concept") + ) + + +def _center(obj: dict) -> tuple[float, float]: + box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + return float(box.get("x") or 0) + float(box.get("w") or 0) / 2, float(box.get("y") or 0) + float(box.get("h") or 0) / 2 + + +def _semantic_panels(contract: dict) -> list[dict]: + spec = _specification(contract) + panels = [] + for index, raw in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(raw, dict): + continue + panel_id = str(raw.get("id") or f"panel_{index + 1}").strip() + label = str(raw.get("label") or raw.get("description") or panel_id).strip() + if panel_id: + panels.append({"id": panel_id, "label": label, "order": index}) + return panels + + +def _visual_panel_id(obj: dict, panels: list[dict]) -> str | None: + direct = str(obj.get("panel_id") or obj.get("parent_panel") or "").strip() + if direct: + return direct + object_id = str(obj.get("id") or "").strip() + if any(str(panel.get("id") or "") == object_id for panel in panels): + return object_id + box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + containing = [ + panel for panel in panels + if isinstance(panel.get("bbox_percent"), dict) and _inside(box, panel["bbox_percent"]) + ] + if not containing: + return None + containing.sort(key=lambda panel: float(panel["bbox_percent"].get("w") or 0) * float(panel["bbox_percent"].get("h") or 0)) + return str(containing[0].get("id") or "").strip() or None + + +def _resolve_panel_mapping(semantic_panels: list[dict], program_panels: list[dict]) -> tuple[dict[str, str], list[dict]]: + visual_panels = sorted(program_panels, key=lambda item: (_center(item)[0], _center(item)[1], str(item.get("id") or ""))) + available = list(visual_panels) + mapping: dict[str, str] = {} + records: list[dict] = [] + for semantic_index, semantic in enumerate(semantic_panels): + semantic_id = str(semantic.get("id") or "").strip() + semantic_label = str(semantic.get("label") or semantic_id).strip() + semantic_tokens = _tokens(semantic_label) | _tokens(semantic_id) + semantic_compact = _normalized_text(semantic_label) + scored = [] + for visual in available: + visual_id = str(visual.get("id") or "").strip() + searchable = _object_text(visual) + visual_tokens = _tokens(searchable) + score = len(semantic_tokens & visual_tokens) / max(1, len(semantic_tokens | visual_tokens)) + if semantic_id and semantic_id == visual_id: + score += 10.0 + visual_compact = _normalized_text(searchable) + if semantic_compact and visual_compact: + if semantic_compact == visual_compact: + score += 4.0 + elif semantic_compact in visual_compact or visual_compact in semantic_compact: + score += 2.0 + scored.append((score, -_center(visual)[0], -_center(visual)[1], visual)) + if not scored: + records.append({ + "semantic_panel_id": semantic_id, + "semantic_panel_label": semantic_label, + "visual_panel_id": None, + "match_score": 0.0, + "status": "unmapped", + }) + continue + scored.sort(key=lambda value: (value[0], value[1], value[2]), reverse=True) + best_score, _, _, chosen = scored[0] + status = "mapped" + if best_score <= 0 and semantic_index < len(visual_panels): + ordered_candidate = visual_panels[semantic_index] + if ordered_candidate in available: + chosen = ordered_candidate + status = "mapped_spatial_fallback" + elif best_score <= 0: + records.append({ + "semantic_panel_id": semantic_id, + "semantic_panel_label": semantic_label, + "visual_panel_id": None, + "match_score": 0.0, + "status": "unmapped", + }) + continue + available.remove(chosen) + visual_id = str(chosen.get("id") or "").strip() + mapping[semantic_id] = visual_id + records.append({ + "semantic_panel_id": semantic_id, + "semantic_panel_label": semantic_label, + "visual_panel_id": visual_id, + "visual_panel_title": str(chosen.get("title") or chosen.get("label") or visual_id), + "match_score": round(max(0.0, best_score), 4), + "status": status, + }) + return mapping, records + + +def _assign_entities( + entities: list[dict], + program: dict, + semantic_panels: list[dict] | None = None, +) -> tuple[dict[str, dict], list[dict], dict[str, str], list[dict]]: + slots = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + cards = [item for item in program.get("cards", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + panels = [item for item in program.get("panels", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + primary_objects = [*cards, *slots] + seen: set[str] = set() + primary_objects = [item for item in primary_objects if not (str(item.get("id")) in seen or seen.add(str(item.get("id"))))] + text_items = [ + item for item in (program.get("text_program", {}).get("items", []) if isinstance(program.get("text_program"), dict) else []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) + and not str(item.get("id") or "").startswith("semantic_text_") + ] + text_objects = [ + { + "id": f"semantic_text_anchor_{str(item.get('id') or index)}", + "title": str(item.get("text") or ""), + "bbox_percent": dict(item["bbox_percent"]), + "panel_id": _visual_panel_id(item, panels), + "text_item_id": str(item.get("id") or ""), + "semantic_instance_id": str(item.get("semantic_entity_id") or "") or None, + "_semantic_text_anchor": True, + } + for index, item in enumerate(text_items, start=1) + if str(item.get("text") or "").strip() + ] + + def object_search_text(obj: dict) -> str: + if obj.get("_semantic_text_anchor"): + return str(obj.get("title") or "") + if obj in panels: + return _object_text(obj) + values = [_object_text(obj)] + box = obj.get("bbox_percent") or {} + for text_item in text_items: + if str(text_item.get("target_id") or "") == str(obj.get("id") or "") or _inside(text_item.get("bbox_percent") or {}, box): + values.append(str(text_item.get("text") or "")) + return " ".join(values) + + def object_priority(obj: dict) -> int: + if obj in cards: + return 3 + if obj in slots: + return 2 + if obj.get("_semantic_text_anchor"): + return 1 + return 0 + + def score_candidates(entity: dict, candidates: list[dict]) -> list[tuple]: + entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) + semantic_panel_id = str(entity.get("panel_id") or "").strip() + scored = [] + for obj in candidates: + searchable = object_search_text(obj) + obj_tokens = _tokens(searchable) + score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) + semantic_instance_id = str(obj.get("semantic_instance_id") or "").strip() + if semantic_instance_id and semantic_instance_id == entity["id"]: + score += 10.0 + object_semantic_panel_id = str(obj.get("semantic_panel_id") or "").strip() + if object_semantic_panel_id and object_semantic_panel_id == semantic_panel_id: + score += 0.5 + entity_compact = _normalized_text(entity.get("label")) + object_compact = _normalized_text(searchable) + if entity_compact and entity_compact == object_compact: + score += 2.0 + elif entity_compact and entity_compact in object_compact: + score += 1.0 + scored.append((score, object_priority(obj), -_center(obj)[1], -_center(obj)[0], obj)) + scored.sort(key=lambda value: (value[0], value[1], value[2], value[3]), reverse=True) + return scored + + def strong_primary_match(entity: dict, scored: list[tuple]) -> bool: + if not scored or scored[0][0] <= 0: + return False + entity_compact = _normalized_text(entity.get("label")) + return bool(entity_compact and entity_compact in _normalized_text(object_search_text(scored[0][4]))) + + def exact_text_match(entity: dict, scored: list[tuple]) -> bool: + if not scored or scored[0][0] <= 0: + return False + return _normalized_text(entity.get("label")) == _normalized_text(object_search_text(scored[0][4])) + + def composite_text_candidates(entity: dict, candidates: list[dict]) -> list[dict]: + """Build exact semantic anchors from adjacent OCR line fragments. + + OCR commonly returns a visible label such as Position Embedding as two + stacked regions. These composites are entity-specific and require an + exact normalized concatenation, so a short label such as Class still + cannot claim the Class Token region. + """ + + target = _normalized_text(entity.get("label")) + if not target or len(candidates) < 2: + return [] + + def local_sequence(items: tuple[dict, ...]) -> bool: + ordered = sorted(items, key=lambda item: (_center(item)[1], _center(item)[0])) + for previous, current in zip(ordered, ordered[1:]): + prev_box = previous["bbox_percent"] + cur_box = current["bbox_percent"] + prev_left = float(prev_box.get("x") or 0) + prev_top = float(prev_box.get("y") or 0) + prev_right = prev_left + float(prev_box.get("w") or 0) + prev_bottom = prev_top + float(prev_box.get("h") or 0) + cur_left = float(cur_box.get("x") or 0) + cur_top = float(cur_box.get("y") or 0) + cur_right = cur_left + float(cur_box.get("w") or 0) + cur_bottom = cur_top + float(cur_box.get("h") or 0) + max_height = max(float(prev_box.get("h") or 0), float(cur_box.get("h") or 0), 0.001) + vertical_gap = max(0.0, cur_top - prev_bottom) + horizontal_gap = max(0.0, cur_left - prev_right, prev_left - cur_right) + horizontal_overlap = min(prev_right, cur_right) - max(prev_left, cur_left) + vertical_overlap = min(prev_bottom, cur_bottom) - max(prev_top, cur_top) + stacked = horizontal_overlap >= -0.012 and vertical_gap <= max(0.025, max_height * 1.35) + inline = vertical_overlap >= -max_height * 0.35 and horizontal_gap <= max(0.035, max_height * 2.0) + if not (stacked or inline): + return False + return True + + results = [] + max_members = min(4, len(candidates)) + for member_count in range(2, max_members + 1): + for members in combinations(candidates, member_count): + panel_ids = {str(item.get("panel_id") or "") for item in members} + if len(panel_ids) > 1 or not local_sequence(members): + continue + ordered = sorted(members, key=lambda item: (_center(item)[1], _center(item)[0])) + if "".join(_normalized_text(item.get("title")) for item in ordered) != target: + continue + boxes = [item["bbox_percent"] for item in ordered] + left = min(float(box.get("x") or 0) for box in boxes) + top = min(float(box.get("y") or 0) for box in boxes) + right = max(float(box.get("x") or 0) + float(box.get("w") or 0) for box in boxes) + bottom = max(float(box.get("y") or 0) + float(box.get("h") or 0) for box in boxes) + member_ids = [str(item.get("id") or "") for item in ordered] + results.append({ + "id": "semantic_text_anchor_composite_" + "__".join(member_ids), + "title": " ".join(str(item.get("title") or "").strip() for item in ordered), + "bbox_percent": { + "x": round(left, 4), + "y": round(top, 4), + "w": round(right - left, 4), + "h": round(bottom - top, 4), + }, + "panel_id": str(ordered[0].get("panel_id") or "") or None, + "_semantic_text_anchor": True, + "_semantic_text_anchor_member_ids": member_ids, + }) + results.sort(key=lambda item: (len(item["_semantic_text_anchor_member_ids"]), _center(item)[1], _center(item)[0])) + return results + + panel_mapping, panel_records = _resolve_panel_mapping(semantic_panels or [], panels) + primary_objects.sort(key=lambda item: (_center(item)[1], _center(item)[0], str(item.get("id") or ""))) + available_primary = list(primary_objects) + available_text = list(text_objects) + available_panels = list(panels) + mapping: dict[str, dict] = {} + records: list[dict] = [] + for entity in entities: + semantic_panel_id = str(entity.get("panel_id") or "").strip() + expected_visual_panel_id = panel_mapping.get(semantic_panel_id) + eligible_primary = [ + obj for obj in available_primary + if not expected_visual_panel_id or _visual_panel_id(obj, panels) == expected_visual_panel_id + ] + eligible_text = [ + obj for obj in available_text + if not expected_visual_panel_id or _visual_panel_id(obj, panels) == expected_visual_panel_id + ] + eligible_panels = [ + obj for obj in available_panels + if not expected_visual_panel_id or _visual_panel_id(obj, panels) == expected_visual_panel_id + ] + if expected_visual_panel_id and not (eligible_primary or eligible_text or eligible_panels): + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "semantic_panel_id": semantic_panel_id, + "expected_visual_panel_id": expected_visual_panel_id, + "object_id": None, + "object_panel_id": None, + "panel_match": False, + "match_score": 0.0, + "status": "blocked_panel_constraint", + }) + continue + + scored_primary = score_candidates(entity, eligible_primary) + eligible_text_with_composites = [*eligible_text, *composite_text_candidates(entity, eligible_text)] + scored_text = score_candidates(entity, eligible_text_with_composites) + scored_panels = score_candidates(entity, eligible_panels) + scored: list[tuple] = [] + chosen_pool: list[dict] | None = None + if strong_primary_match(entity, scored_primary): + scored, chosen_pool = scored_primary, available_primary + elif exact_text_match(entity, scored_text): + scored, chosen_pool = scored_text, available_text + elif scored_primary and scored_primary[0][0] > 0: + scored, chosen_pool = scored_primary, available_primary + elif scored_panels and scored_panels[0][0] > 0: + scored, chosen_pool = scored_panels, available_panels + elif expected_visual_panel_id: + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "semantic_panel_id": semantic_panel_id, + "expected_visual_panel_id": expected_visual_panel_id, + "object_id": None, + "object_panel_id": None, + "panel_match": False, + "match_score": 0.0, + "status": "blocked_panel_constraint", + }) + continue + elif eligible_primary: + scored, chosen_pool = scored_primary, available_primary + elif eligible_panels: + scored, chosen_pool = scored_panels, available_panels + + if not scored or chosen_pool is None: + records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) + continue + + chosen = scored[0][4] + composite_member_ids = set(chosen.get("_semantic_text_anchor_member_ids") or []) + if composite_member_ids: + available_text[:] = [ + item for item in available_text + if str(item.get("id") or "") not in composite_member_ids + ] + else: + chosen_pool.remove(chosen) + mapping[entity["id"]] = chosen + object_panel_id = _visual_panel_id(chosen, panels) + panel_match = not expected_visual_panel_id or object_panel_id == expected_visual_panel_id + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "object_id": chosen.get("id"), + "semantic_panel_id": semantic_panel_id or None, + "expected_visual_panel_id": expected_visual_panel_id, + "object_panel_id": object_panel_id, + "object_kind": "text_anchor" if chosen.get("_semantic_text_anchor") else ("panel" if chosen in panels else ("card" if chosen in cards else "slot")), + "text_anchor_member_ids": list(chosen.get("_semantic_text_anchor_member_ids") or []), + "panel_match": panel_match, + "match_score": round(max(0.0, scored[0][0]), 4), + "status": ("mapped" if scored[0][0] > 0 else "mapped_spatial_fallback") if panel_match else "blocked_cross_panel", + }) + return mapping, records, panel_mapping, panel_records + + +def _inside(inner: dict, outer: dict) -> bool: + cx = float(inner.get("x") or 0) + float(inner.get("w") or 0) / 2 + cy = float(inner.get("y") or 0) + float(inner.get("h") or 0) / 2 + return float(outer.get("x") or 0) <= cx <= float(outer.get("x") or 0) + float(outer.get("w") or 0) and float(outer.get("y") or 0) <= cy <= float(outer.get("y") or 0) + float(outer.get("h") or 0) + + +def _label_bbox(obj: dict) -> dict[str, float]: + box = obj["bbox_percent"] + x, y, w, h = (float(box[key]) for key in ("x", "y", "w", "h")) + if "paper_native" in str(obj.get("reference_crop_role") or "").casefold(): + label_h = min(0.05, max(0.035, h * 0.24)) + return { + "x": round(x, 4), + "y": round(max(0.0, y - label_h - 0.008), 4), + "w": round(w, 4), + "h": round(label_h, 4), + } + label_h = min(0.055, max(0.026, h * 0.18)) + return {"x": round(x, 4), "y": round(max(0.0, y), 4), "w": round(w, 4), "h": round(label_h, 4)} + + +def _anchor(source: dict, target: dict) -> list[list[float]]: + sx, sy = _center(source) + tx, ty = _center(target) + sb, tb = source["bbox_percent"], target["bbox_percent"] + if abs(tx - sx) >= abs(ty - sy): + start = [float(sb["x"]) + (float(sb["w"]) if tx >= sx else 0.0), sy] + end = [float(tb["x"]) + (0.0 if tx >= sx else float(tb["w"])), ty] + mid = (start[0] + end[0]) / 2 + points = [start, [mid, start[1]], [mid, end[1]], end] + else: + start = [sx, float(sb["y"]) + (float(sb["h"]) if ty >= sy else 0.0)] + end = [tx, float(tb["y"]) + (0.0 if ty >= sy else float(tb["h"]))] + mid = (start[1] + end[1]) / 2 + points = [start, [start[0], mid], [end[0], mid], end] + clean: list[list[float]] = [] + for point in points: + rounded = [round(max(0.0, min(1.0, point[0])), 4), round(max(0.0, min(1.0, point[1])), 4)] + if not clean or clean[-1] != rounded: + clean.append(rounded) + return clean + + +def _segment_hits_box(start: list[float], end: list[float], box: dict, margin: float = 0.004) -> bool: + left = float(box.get("x") or 0) + margin + top = float(box.get("y") or 0) + margin + right = float(box.get("x") or 0) + float(box.get("w") or 0) - margin + bottom = float(box.get("y") or 0) + float(box.get("h") or 0) - margin + if abs(start[0] - end[0]) < 1e-6: + x = start[0] + low, high = sorted((start[1], end[1])) + return left < x < right and max(low, top) < min(high, bottom) + if abs(start[1] - end[1]) < 1e-6: + y = start[1] + low, high = sorted((start[0], end[0])) + return top < y < bottom and max(low, left) < min(high, right) + return False + + +def _route_around_cards(source: dict, target: dict, cards: list[dict], panel_box: dict | None = None) -> tuple[list[list[float]], str]: + base = _anchor(source, target) + source_id = str(source.get("id") or "") + target_id = str(target.get("id") or "") + obstacles = [ + card for card in cards + if str(card.get("id") or "") not in {source_id, target_id} + and isinstance(card.get("bbox_percent"), dict) + and (not panel_box or _inside(card.get("bbox_percent") or {}, panel_box)) + ] + + def route_hits(path: list[list[float]]) -> bool: + return any( + _segment_hits_box(start, end, card["bbox_percent"]) + for start, end in zip(path, path[1:]) + for card in obstacles + ) + + if not route_hits(base): + return base, "direct_orthogonal" + start, end = base[0], base[-1] + vertical = abs(end[1] - start[1]) >= abs(end[0] - start[0]) + candidates: list[list[list[float]]] = [] + if vertical: + obstacle_left = min(float(card["bbox_percent"].get("x") or 0) for card in obstacles) + obstacle_right = max(float(card["bbox_percent"].get("x") or 0) + float(card["bbox_percent"].get("w") or 0) for card in obstacles) + lanes = [obstacle_left - 0.012, obstacle_right + 0.012] + if panel_box: + panel_left = float(panel_box.get("x") or 0) + 0.008 + panel_right = float(panel_box.get("x") or 0) + float(panel_box.get("w") or 0) - 0.008 + lanes = [lane for lane in lanes if panel_left <= lane <= panel_right] + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_y = float(source_box.get("y") or 0) + float(source_box.get("h") or 0) / 2 + target_center_y = float(target_box.get("y") or 0) + float(target_box.get("h") or 0) / 2 + for lane in lanes: + use_right = lane >= (start[0] + end[0]) / 2 + source_side = float(source_box.get("x") or 0) + (float(source_box.get("w") or 0) if use_right else 0.0) + target_side = float(target_box.get("x") or 0) + (float(target_box.get("w") or 0) if use_right else 0.0) + candidates.append([ + [round(source_side, 4), round(source_center_y, 4)], + [round(lane, 4), round(source_center_y, 4)], + [round(lane, 4), round(target_center_y, 4)], + [round(target_side, 4), round(target_center_y, 4)], + ]) + else: + obstacle_top = min(float(card["bbox_percent"].get("y") or 0) for card in obstacles) + obstacle_bottom = max(float(card["bbox_percent"].get("y") or 0) + float(card["bbox_percent"].get("h") or 0) for card in obstacles) + lanes = [obstacle_top - 0.012, obstacle_bottom + 0.012] + if panel_box: + panel_top = float(panel_box.get("y") or 0) + 0.008 + panel_bottom = float(panel_box.get("y") or 0) + float(panel_box.get("h") or 0) - 0.008 + lanes = [lane for lane in lanes if panel_top <= lane <= panel_bottom] + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_x = float(source_box.get("x") or 0) + float(source_box.get("w") or 0) / 2 + target_center_x = float(target_box.get("x") or 0) + float(target_box.get("w") or 0) / 2 + for lane in lanes: + use_bottom = lane >= (start[1] + end[1]) / 2 + source_side = float(source_box.get("y") or 0) + (float(source_box.get("h") or 0) if use_bottom else 0.0) + target_side = float(target_box.get("y") or 0) + (float(target_box.get("h") or 0) if use_bottom else 0.0) + candidates.append([ + [round(source_center_x, 4), round(source_side, 4)], + [round(source_center_x, 4), round(lane, 4)], + [round(target_center_x, 4), round(lane, 4)], + [round(target_center_x, 4), round(target_side, 4)], + ]) + valid = [path for path in candidates if not route_hits(path)] + if not valid: + return base, "direct_orthogonal_obstacle_unresolved" + valid.sort(key=lambda path: sum(abs(b[0] - a[0]) + abs(b[1] - a[1]) for a, b in zip(path, path[1:]))) + return valid[0], "obstacle_avoiding_orthogonal" + + +def _preserve_local_reference_arrows(program: dict, mapping: dict[str, dict]) -> tuple[list[dict], list[str]]: + mapped_objects = list(mapping.values()) + all_objects = { + str(item.get("id")): item + for field in ("slots", "cards", "panels") + for item in program.get(field, []) + if isinstance(item, dict) and item.get("id") and isinstance(item.get("bbox_percent"), dict) + } + + def mapped_container(obj: dict | None) -> str | None: + if not obj: + return None + center_box = obj.get("bbox_percent") or {} + for mapped in mapped_objects: + if str(mapped.get("id")) == str(obj.get("id")) or _inside(center_box, mapped.get("bbox_percent") or {}): + return str(mapped.get("id")) + return None + + preserved: list[dict] = [] + replaced: list[str] = [] + for arrow in program.get("arrows", []) if isinstance(program.get("arrows"), list) else []: + if not isinstance(arrow, dict): + continue + source_id = str(arrow.get("source_id") or arrow.get("source") or "") + target_id = str(arrow.get("target_id") or arrow.get("target") or "") + arrow_id = str(arrow.get("id") or "reference_arrow") + if not source_id or not target_id: + replaced.append(arrow_id) + continue + source_container = mapped_container(all_objects.get(source_id)) + target_container = mapped_container(all_objects.get(target_id)) + if source_container and target_container and source_container != target_container: + replaced.append(arrow_id) + continue + preserved.append(arrow) + return preserved, replaced + + +def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: + out = Path(out_dir) + entities = _entities(contract) + semantic_panels = _semantic_panels(contract) + relations, relation_source = _relations(contract) + mapping, mapping_records, panel_mapping, panel_mapping_records = _assign_entities(entities, program, semantic_panels) + mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} + existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] + preserved = [dict(item) for item in existing if not str(item.get("id") or "").startswith("semantic_text_")] + semantic_text = [] + semantic_reference_regions = [] + + def exact_local_fragment_sequence(items: tuple[dict, ...], expected_text: str) -> bool: + ordered = sorted(items, key=lambda item: (_center(item)[1], _center(item)[0])) + if "".join(_normalized_text(item.get("text")) for item in ordered) != expected_text: + return False + for previous, current in zip(ordered, ordered[1:]): + prev_box = previous.get("bbox_percent") or {} + cur_box = current.get("bbox_percent") or {} + prev_left = float(prev_box.get("x") or 0) + prev_top = float(prev_box.get("y") or 0) + prev_right = prev_left + float(prev_box.get("w") or 0) + prev_bottom = prev_top + float(prev_box.get("h") or 0) + cur_left = float(cur_box.get("x") or 0) + cur_top = float(cur_box.get("y") or 0) + cur_right = cur_left + float(cur_box.get("w") or 0) + cur_bottom = cur_top + float(cur_box.get("h") or 0) + max_height = max(float(prev_box.get("h") or 0), float(cur_box.get("h") or 0), 0.001) + vertical_gap = max(0.0, cur_top - prev_bottom) + horizontal_gap = max(0.0, cur_left - prev_right, prev_left - cur_right) + horizontal_overlap = min(prev_right, cur_right) - max(prev_left, cur_left) + vertical_overlap = min(prev_bottom, cur_bottom) - max(prev_top, cur_top) + stacked = horizontal_overlap >= -0.012 and vertical_gap <= max(0.025, max_height * 1.35) + inline = vertical_overlap >= -max_height * 0.35 and horizontal_gap <= max(0.035, max_height * 2.0) + if not (stacked or inline): + return False + return True + + for entity in entities: + obj = mapping.get(entity["id"]) + if not obj: + continue + box = _label_bbox(obj) + object_box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + target_ids = {str(obj.get("id") or ""), str(obj.get("panel_id") or "")} + target_ids.discard("") + entity_norm = _normalized_text(entity["label"]) + spatial_candidates = [ + item for item in preserved + if ( + str(item.get("target_id") or "") in target_ids + or _inside(item.get("bbox_percent") or {}, object_box) + ) + ] + reference_matches = [ + item for item in spatial_candidates + if entity_norm and _normalized_text(item.get("text")) + and (entity_norm in _normalized_text(item.get("text")) or _normalized_text(item.get("text")) in entity_norm) + ] + if reference_matches: + best_match = max( + reference_matches, + key=lambda item: ( + _normalized_text(item.get("text")) == entity_norm, + str(item.get("target_id") or "") == str(obj.get("id") or ""), + -abs(len(_normalized_text(item.get("text"))) - len(entity_norm)), + ), + ) + original_text = str(best_match.get("text") or "") + geometry_matches = [best_match] + if _normalized_text(best_match.get("text")) != entity_norm: + fragment_sequences = [] + max_members = min(4, len(reference_matches)) + for member_count in range(2, max_members + 1): + for members in combinations(reference_matches, member_count): + if best_match not in members: + continue + if exact_local_fragment_sequence(members, entity_norm): + fragment_sequences.append(members) + if fragment_sequences: + break + if fragment_sequences: + fragment_sequences.sort(key=lambda members: ( + len(members), + max(_center(item)[1] for item in members) - min(_center(item)[1] for item in members), + )) + geometry_matches = list(fragment_sequences[0]) + if len(geometry_matches) > 1: + reference_boxes = [item["bbox_percent"] for item in geometry_matches] + left = min(float(item.get("x") or 0) for item in reference_boxes) + top = min(float(item.get("y") or 0) for item in reference_boxes) + right = max(float(item.get("x") or 0) + float(item.get("w") or 0) for item in reference_boxes) + bottom = max(float(item.get("y") or 0) + float(item.get("h") or 0) for item in reference_boxes) + merged_box = { + "x": round(left, 4), + "y": round(top, 4), + "w": round(right - left, 4), + "h": round(bottom - top, 4), + } + best_match.update({ + "bbox_percent": merged_box, + "center_percent": { + "x": round(merged_box["x"] + merged_box["w"] / 2, 4), + "y": round(merged_box["y"] + merged_box["h"] / 2, 4), + }, + "width_percent": merged_box["w"], + "height_percent": merged_box["h"], + "semantic_merged_reference_text_ids": [ + str(item.get("id") or "") for item in geometry_matches + ], + }) + display_text = entity["label"] + if len(geometry_matches) > 1: + ordered_geometry_matches = sorted( + geometry_matches, + key=lambda item: (_center(item)[1], _center(item)[0]), + ) + display_text = "\n".join( + str(item.get("text") or "").strip() + for item in ordered_geometry_matches + if str(item.get("text") or "").strip() + ) + best_match.update({ + "text": display_text, + "semantic_contract_text": entity["label"], + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "semantic_evidence_ids": entity["evidence_ids"], + "semantic_original_ocr_text": original_text, + "reference_binding": "paper_semantic_contract_reference_text", + "fit_strategy": best_match.get("fit_strategy") or "reference_text_geometry_exact", + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + best_id = str(best_match.get("id") or "") + duplicate_ids = { + str(item.get("id") or "") for item in geometry_matches + if str(item.get("id") or "") != best_id + } + if duplicate_ids: + preserved = [item for item in preserved if str(item.get("id") or "") not in duplicate_ids] + continue + if str(obj.get("id") or "") not in {str(panel.get("id") or "") for panel in program.get("panels", []) if isinstance(panel, dict)}: + preserved = [ + item for item in preserved + if not _inside(item.get("bbox_percent") or {}, object_box) + ] + reference_text_id = f"semantic_ref_text_{entity['id']}" + center = {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)} + estimated_font_ratio = round(max(0.006, box["h"] * 0.55), 6) + semantic_text.append({ + "id": f"semantic_text_{entity['id']}", + "text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "semantic_evidence_ids": entity["evidence_ids"], + "source_reference_text_id": reference_text_id, + "reference_binding": "reference_slot_semantic_overlay_geometry", + "bbox_percent": box, + "center_percent": center, + "width_percent": box["w"], + "height_percent": box["h"], + "estimated_font_ratio": estimated_font_ratio, + "font_size_pt": 10.0, + "color_hex": "#263747", + "bold": entity["role"] in {"module", "innovation"}, + "align": "center", + "font_family_guess": "Arial", + "font_weight_guess": "bold" if entity["role"] in {"module", "innovation"} else "regular", + "fit_strategy": "paper_semantic_contract_exact_label", + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + semantic_reference_regions.append({ + "id": reference_text_id, + "text": entity["label"], + "raw_text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "evidence_ids": entity["evidence_ids"], + "bbox_percent": box, + "center_percent": center, + "width_percent": box["w"], + "height_percent": box["h"], + "estimated_font_ratio": estimated_font_ratio, + "color_hex": "#263747", + "font_family_guess": "Arial", + "font_weight_guess": "bold" if entity["role"] in {"module", "innovation"} else "regular", + "source": "reference_slot_semantic_overlay_geometry", + "editable_in": "pptx", + }) + + panels_by_id = { + str(item.get("id") or ""): item + for item in program.get("panels", []) + if isinstance(item, dict) and item.get("id") and isinstance(item.get("bbox_percent"), dict) + } + panel_title_records = [] + for semantic_panel in semantic_panels: + semantic_panel_id = str(semantic_panel.get("id") or "") + visual_panel_id = panel_mapping.get(semantic_panel_id) + visual_panel = panels_by_id.get(str(visual_panel_id or "")) + if not visual_panel: + panel_title_records.append({ + "semantic_panel_id": semantic_panel_id, + "visual_panel_id": visual_panel_id, + "status": "unmapped", + }) + continue + label = str(semantic_panel.get("label") or semantic_panel_id) + panel_box = visual_panel["bbox_percent"] + header_limit = float(panel_box.get("y") or 0) + min(0.09, max(0.045, float(panel_box.get("h") or 0) * 0.08)) + candidates = [ + item for item in preserved + if str(item.get("target_id") or "") == visual_panel_id + and str(item.get("role") or "").casefold() == "panel_title" + and not item.get("semantic_entity_id") + and _center(item)[1] <= header_limit + ] + label_norm = _normalized_text(label) + fragments = [ + item for item in candidates + if _normalized_text(item.get("text")) and _normalized_text(item.get("text")) in label_norm + ] + best = max(fragments or candidates, key=lambda item: float((item.get("bbox_percent") or {}).get("w") or 0), default=None) + if best: + original_text = str(best.get("text") or "") + panel_title_box = { + "x": round(float(panel_box.get("x") or 0) + 0.008, 4), + "y": round(float(panel_box.get("y") or 0) + 0.004, 4), + "w": round(max(0.02, float(panel_box.get("w") or 0) - 0.016), 4), + "h": round(min(0.06, max(0.038, float(panel_box.get("h") or 0) * 0.07)), 4), + } + display_label = label + if len(label) > 34 and panel_title_box["w"] < 0.30 and " & " in label: + display_label = label.replace(" & ", " &\n", 1) + best.update({ + "text": display_label, + "target_id": visual_panel_id, + "semantic_panel_id": semantic_panel_id, + "semantic_panel_title": True, + "semantic_contract_text": label, + "semantic_original_ocr_text": original_text, + "reference_binding": "paper_semantic_contract_panel_title", + "bbox_percent": panel_title_box, + "center_percent": { + "x": round(panel_title_box["x"] + panel_title_box["w"] / 2, 4), + "y": round(panel_title_box["y"] + panel_title_box["h"] / 2, 4), + }, + "width_percent": panel_title_box["w"], + "height_percent": panel_title_box["h"], + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + best_id = str(best.get("id") or "") + duplicate_ids = { + str(item.get("id") or "") for item in fragments + if str(item.get("id") or "") != best_id + } + if duplicate_ids: + preserved = [item for item in preserved if str(item.get("id") or "") not in duplicate_ids] + panel_title_records.append({ + "semantic_panel_id": semantic_panel_id, + "visual_panel_id": visual_panel_id, + "text_id": best_id, + "status": "replaced_exact", + }) + else: + panel_title_records.append({ + "semantic_panel_id": semantic_panel_id, + "visual_panel_id": visual_panel_id, + "status": "missing_reference_title", + }) + text_program = { + **text_program, + "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", + "semantic_authority": "paper_ground_truth", + "items": preserved + semantic_text, + } + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + program["reference_text_geometry_path"] = "reference_text_geometry.json" + program["text_alignment_report_path"] = "text_alignment_report.json" + + geometry_path = out / "reference_text_geometry.json" + if geometry_path.exists(): + reference_geometry = read_json(geometry_path) + else: + reference_geometry = { + "summary": "Reference-first text geometry with semantic overlay bindings.", + "policy": "reference_image_is_highest_authority_for_text_size_position_color_and_hierarchy", + "detection_mode": "reference_geometry_and_semantic_overlay", + "text_regions": [], + } + geometry_regions = [ + item for item in reference_geometry.get("text_regions", []) + if isinstance(item, dict) and not str(item.get("id") or "").startswith("semantic_ref_text_") + ] + exact_item_by_reference_id = { + str(item.get("source_reference_text_id") or ""): item + for item in text_program["items"] + if item.get("source_reference_text_id") and (item.get("semantic_entity_id") or item.get("semantic_panel_title")) + } + for region in geometry_regions: + region_id = str(region.get("id") or "") + if region_id in exact_item_by_reference_id: + exact_item = exact_item_by_reference_id[region_id] + region["semantic_original_reference_text"] = str(region.get("text") or region.get("raw_text") or "") + region["text"] = str(exact_item.get("text") or "") + region["raw_text"] = str(exact_item.get("text") or "") + for key in ("bbox_percent", "center_percent", "width_percent", "height_percent"): + if key in exact_item: + region[key] = exact_item[key] + region["semantic_text_replaced"] = True + reference_geometry["text_regions"] = geometry_regions + semantic_reference_regions + reference_geometry["semantic_overlay_policy"] = "exact paper terminology bound to reference-derived slot geometry" + + alignment_items = [] + for item in text_program["items"]: + alignment_items.append({ + "label_id": item.get("id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "target_id": item.get("target_id"), + "center_delta_percent": 0.0, + "width_delta_percent": 0.0, + "height_delta_percent": 0.0, + "font_ratio_delta": 0.0, + "color_match": True, + "editable_in": "pptx", + "status": "pass", + }) + alignment_report = { + "summary": "Reference alignment report after applying exact paper-semantic overlay labels.", + "policy": "reference_alignment_only; semantic labels inherit their mapped reference slot geometry", + "status": "pass", + "max_center_delta_percent": 0.0, + "max_width_delta_percent": 0.0, + "max_height_delta_percent": 0.0, + "items": alignment_items, + } + + semantic_arrows = [] + skipped_relations = [] + blueprint_routes = { + ( + str(item.get("semantic_source_id") or item.get("source_id") or ""), + str(item.get("semantic_target_id") or item.get("target_id") or ""), + ): item + for item in program.get("arrows", []) if isinstance(item, dict) + if str(item.get("binding_source") or "") == "generation_semantic_blueprint" + } + cards = [item for item in program.get("cards", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + panels_by_visual_id = { + str(item.get("id") or ""): item + for item in program.get("panels", []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) + } + for index, relation in enumerate(relations, start=1): + source_id = str(relation.get("source") or relation.get("source_id") or "") + target_id = str(relation.get("target") or relation.get("target_id") or "") + source, target = mapping.get(source_id), mapping.get(target_id) + if not source or not target: + skipped_relations.append({"source": source_id, "target": target_id, "reason": "unmapped_endpoint"}) + continue + source_panel_id = _visual_panel_id(source, list(panels_by_visual_id.values())) + target_panel_id = _visual_panel_id(target, list(panels_by_visual_id.values())) + panel_box = panels_by_visual_id.get(str(source_panel_id or ""), {}).get("bbox_percent") if source_panel_id == target_panel_id else None + blueprint_route = blueprint_routes.get((source_id, target_id)) + if blueprint_route and isinstance(blueprint_route.get("path_percent"), list) and len(blueprint_route["path_percent"]) >= 2: + path_percent = blueprint_route["path_percent"] + route_generation_status = "generation_blueprint_preserved" + routing_algorithm = str(blueprint_route.get("routing_algorithm") or "generation_semantic_blueprint") + else: + path_percent, route_generation_status = _route_around_cards(source, target, cards, panel_box) + routing_algorithm = "panel_card_obstacle_aware_orthogonal" + semantic_arrows.append({ + "id": str(relation.get("id") or f"semantic_relation_{index:02d}"), + "source_id": str(source.get("id")), + "target_id": str(target.get("id")), + "semantic_source_id": source_id, + "semantic_target_id": target_id, + "semantic_panel_id": relation.get("panel_id"), + "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), + "label": str(relation.get("label") or relation.get("statement") or ""), + "evidence_ids": list(relation.get("evidence_ids") or []), + "path_percent": path_percent, + "source_anchor": "auto", + "target_anchor": "auto", + "control_kind": "elbow_connector", + "render_style": "elbow_connector", + "route_intent": "paper_semantic_relation", + "routing_algorithm": routing_algorithm, + "route_generation_status": route_generation_status, + "binding_source": "generation_semantic_blueprint" if blueprint_route else "paper_semantic_contract", + "reference_locked": bool(blueprint_route), + "reference_path_preserved": bool(blueprint_route), + "stroke_color": str((blueprint_route or {}).get("stroke_color") or "#3F5063"), + "stroke_width_pt": float((blueprint_route or {}).get("stroke_width_pt") or 1.8), + "arrowhead": str((blueprint_route or {}).get("arrowhead") or "triangle"), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + }) + paper_native = any( + "paper_native" in str(item.get("reference_crop_role") or "").casefold() + for item in program.get("slots", []) if isinstance(item, dict) + ) + preserved_reference_arrows, replaced_reference_arrow_ids = _preserve_local_reference_arrows(program, mapping) + if paper_native: + replaced_reference_arrow_ids.extend(str(item.get("id") or "seed_control") for item in preserved_reference_arrows) + preserved_reference_arrows = [] + if relation_source == "panel_graphs": + heuristic_defaults = [ + item for item in preserved_reference_arrows + if str(item.get("binding_source") or "").startswith("heuristic_default_") + ] + if heuristic_defaults: + replaced_reference_arrow_ids.extend(str(item.get("id") or "heuristic_default_arrow") for item in heuristic_defaults) + preserved_reference_arrows = [ + item for item in preserved_reference_arrows + if not str(item.get("binding_source") or "").startswith("heuristic_default_") + ] + if mapping: + program["arrows"] = [*preserved_reference_arrows, *semantic_arrows] + + low_confidence_mappings = [ + item for item in mapping_records + if float(item.get("match_score") or 0) <= 0 and str(item.get("status") or "") != "blocked_panel_constraint" + ] + cross_panel_mappings = [item for item in mapping_records if item.get("panel_match") is False and item.get("object_id")] + panel_constraint_failures = [item for item in mapping_records if item.get("status") == "blocked_panel_constraint"] + unmapped_panels = [item for item in panel_mapping_records if item.get("status") == "unmapped"] + blocked = bool(cross_panel_mappings or panel_constraint_failures) + complete = len(mapping) == len(entities) and not skipped_relations and not low_confidence_mappings and not unmapped_panels + report = { + "summary": "Paper semantic contract application report.", + "status": "blocked" if blocked else ("pass" if complete else "warning"), + "semantic_authority": "paper_ground_truth", + "contract_graph_source": relation_source, + "visual_authority": "paper_native_slot_assets" if paper_native else "generated_reference_image", + "entity_count": len(entities), + "mapped_entity_count": len(mapping), + "relation_count": len(relations), + "mapped_relation_count": len(semantic_arrows), + "preserved_reference_arrow_count": len(preserved_reference_arrows), + "replaced_reference_arrow_ids": replaced_reference_arrow_ids, + "mappings": mapping_records, + "panel_mappings": panel_mapping_records, + "panel_title_bindings": panel_title_records, + "cross_panel_mappings": cross_panel_mappings, + "panel_constraint_failures": panel_constraint_failures, + "unmapped_panels": unmapped_panels, + "low_confidence_mappings": low_confidence_mappings, + "skipped_relations": skipped_relations, + "policy": ( + "exact labels, scientific objects, and relations come from the paper contract; the generated seed supplies geometry and style only" + if paper_native + else "exact labels and scientific relations come from the paper contract; the image supplies layout and style only; panel identity is a hard constraint for repeated labels" + ), + } + write_json(out / "paper_semantic_contract.json", contract) + write_json(out / "semantic_binding_report.json", report) + write_json(out / "reference_text_geometry.json", reference_geometry) + write_json(out / "text_program.json", text_program) + write_json(out / "text_alignment_report.json", alignment_report) + write_json(out / "figure_program.json", program) + return program, report diff --git a/rfs/control_localizer.py b/rfs/control_localizer.py index 27761e3..77ca9e2 100644 --- a/rfs/control_localizer.py +++ b/rfs/control_localizer.py @@ -1,11 +1,27 @@ from __future__ import annotations import math +import inspect from pathlib import Path from typing import Callable from PIL import Image, ImageDraw +from .utils import read_json, write_json +from .cv_compat import hough_line_coordinates + +VALID_RENDER_STYLES = { + "filled_block_arrow", + "line_connector", + "elbow_connector", + "branch_line_connector", + "dashed_loop_connector", +} +VALID_ROUTE_INTENTS = {"straight", "orthogonal", "branch", "loop"} +VALID_VISUAL_WEIGHTS = {"chunky", "normal", "thin"} +VALID_PREFERRED_AXES = {"horizontal", "vertical", "horizontal_first", "vertical_first"} +VALID_LINE_PATTERNS = {"solid", "dash", "dashed"} + def _center(box: dict) -> tuple[float, float]: return float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2 @@ -18,6 +34,136 @@ def _nearest_slot(point: tuple[float, float], slots: list[dict]) -> dict | None: return min(slots, key=lambda slot: (_center(slot["bbox_percent"])[0] - px) ** 2 + (_center(slot["bbox_percent"])[1] - py) ** 2) +def _distance_to_bbox(point: tuple[float, float], obj: dict | None) -> float: + if not obj or not isinstance(obj.get("bbox_percent"), dict): + return 1.0 + box = obj["bbox_percent"] + px, py = point + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + dx = max(x - px, 0.0, px - (x + w)) + dy = max(y - py, 0.0, py - (y + h)) + return math.hypot(dx, dy) + + +def _coerce_choice(value, allowed: set[str], default: str = "") -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "block_arrow": "filled_block_arrow", + "chunky_arrow": "filled_block_arrow", + "thick_arrow": "filled_block_arrow", + "straight_arrow": "line_connector", + "elbow": "elbow_connector", + "orthogonal_connector": "elbow_connector", + "branch_connector": "branch_line_connector", + "dashed_loop": "dashed_loop_connector", + "loop_connector": "dashed_loop_connector", + "dashed": "dash", + } + text = aliases.get(text, text) + return text if text in allowed else default + + +def _slot_by_id(slots: list[dict]) -> dict[str, dict]: + return {str(slot.get("id")): slot for slot in slots if isinstance(slot, dict) and slot.get("id")} + + +def _path_bbox(points: list[list[float]]) -> dict[str, float]: + xs = [float(point[0]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + ys = [float(point[1]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + if not xs or not ys: + return {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0} + x0 = min(xs) + y0 = min(ys) + return {"x": x0, "y": y0, "w": max(xs) - x0, "h": max(ys) - y0} + + +def _bbox_iou(a: dict, b: dict) -> float: + ax0, ay0 = float(a.get("x", 0.0)), float(a.get("y", 0.0)) + ax1, ay1 = ax0 + float(a.get("w", 0.0)), ay0 + float(a.get("h", 0.0)) + bx0, by0 = float(b.get("x", 0.0)), float(b.get("y", 0.0)) + bx1, by1 = bx0 + float(b.get("w", 0.0)), by0 + float(b.get("h", 0.0)) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + inter = ix * iy + union = float(a.get("w", 0.0)) * float(a.get("h", 0.0)) + float(b.get("w", 0.0)) * float(b.get("h", 0.0)) - inter + return inter / max(union, 0.000001) + + +def _point_distance(a: list[float], b: list[float]) -> float: + return math.hypot(float(a[0]) - float(b[0]), float(a[1]) - float(b[1])) + + +def _path_distance(a: list[list[float]], b: list[list[float]]) -> float: + if len(a) < 2 or len(b) < 2: + return 1.0 + return (_point_distance(a[0], b[0]) + _point_distance(a[-1], b[-1])) / 2 + + +def _control_signature(control: dict) -> tuple[str, str]: + return str(control.get("source_id") or control.get("source") or ""), str(control.get("target_id") or control.get("target") or "") + + +def _controls_are_duplicate(a: dict, b: dict) -> bool: + ap = a.get("path_percent") or [] + bp = b.get("path_percent") or [] + same_binding = _control_signature(a) == _control_signature(b) + if same_binding and _path_distance(ap, bp) <= 0.08: + return True + return _bbox_iou(_path_bbox(ap), _path_bbox(bp)) >= 0.55 and _path_distance(ap, bp) <= 0.12 + + +def _controls_match_for_disagreement(heuristic: dict, vlm: dict) -> bool: + if _controls_are_duplicate(heuristic, vlm): + return True + hp = heuristic.get("path_percent") or [] + vp = vlm.get("path_percent") or [] + return _control_signature(heuristic) == _control_signature(vlm) or _path_distance(hp, vp) <= 0.08 or _bbox_iou(_path_bbox(hp), _path_bbox(vp)) >= 0.35 + + +def _bindable_objects(out_dir: Path, slots: list[dict]) -> list[dict]: + bindable = [dict(slot) for slot in slots if isinstance(slot, dict) and isinstance(slot.get("bbox_percent"), dict)] + seen = {str(item.get("id")) for item in bindable} + try: + geometry = read_json(out_dir / "reference_geometry.json") + except Exception: + geometry = {} + for card in geometry.get("cards", []) if isinstance(geometry, dict) else []: + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + card_id = str(card.get("id") or "") + if not card_id or card_id in seen: + continue + enriched = dict(card) + enriched.setdefault("display_label", card.get("title") or card_id) + enriched.setdefault("paper_concept", card.get("title") or card_id) + enriched.setdefault("semantic_role", "reference_card") + enriched.setdefault("asset_type", "editable_card") + bindable.append(enriched) + seen.add(card_id) + return bindable + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _center(sbox) + tx, ty = _center(tbox) + if abs(tx - sx) >= abs(ty - sy): + start = (float(sbox["x"]) + (float(sbox["w"]) if tx >= sx else 0.0), sy) + end = (float(tbox["x"]) + (0.0 if tx >= sx else float(tbox["w"])), ty) + else: + start = (sx, float(sbox["y"]) + (float(sbox["h"]) if ty >= sy else 0.0)) + end = (tx, float(tbox["y"]) + (0.0 if ty >= sy else float(tbox["h"]))) + return [[round(start[0], 4), round(start[1], 4)], [round(end[0], 4), round(end[1], 4)]] + + def _sequence_controls(slots: list[dict], palette: list[str]) -> list[dict]: ordered = sorted(slots, key=lambda slot: (float(slot["bbox_percent"]["y"]), float(slot["bbox_percent"]["x"]))) arrows = [] @@ -67,7 +213,13 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str return [] candidates = [] for line in lines[:80]: - x1, y1, x2, y2 = [int(v) for v in line[0]] + coordinates = hough_line_coordinates(line) + if coordinates is None: + values = line.reshape(-1) if hasattr(line, "reshape") else line + if len(values) < 4: + continue + coordinates = [int(v) for v in values[:4]] + x1, y1, x2, y2 = coordinates length = math.hypot(x2 - x1, y2 - y1) if length < min(width, height) * 0.06: continue @@ -77,6 +229,8 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str target = _nearest_slot(p2, slots) if not source or not target or source["id"] == target["id"]: continue + if _distance_to_bbox(p1, source) > 0.075 or _distance_to_bbox(p2, target) > 0.075: + continue direction = math.degrees(math.atan2(p2[1] - p1[1], p2[0] - p1[0])) candidates.append({ "line": [p1, p2], @@ -121,6 +275,7 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str def _normalize_controls(raw: list[dict], slots: list[dict], palette: list[str]) -> list[dict]: slot_ids = {str(slot.get("id")) for slot in slots} + objects = _slot_by_id(slots) normalized = [] for idx, item in enumerate(raw or [], start=1): if not isinstance(item, dict): @@ -128,40 +283,359 @@ def _normalize_controls(raw: list[dict], slots: list[dict], palette: list[str]) source = str(item.get("source_id") or item.get("source") or "") target = str(item.get("target_id") or item.get("target") or "") path = item.get("path_percent") - if source not in slot_ids or target not in slot_ids or not isinstance(path, list) or len(path) < 2: + if source not in slot_ids or target not in slot_ids: continue points = [] - for point in path: - if not isinstance(point, (list, tuple)) or len(point) < 2: - continue - points.append([round(max(0.0, min(1.0, float(point[0]))), 4), round(max(0.0, min(1.0, float(point[1]))), 4)]) + if isinstance(path, list): + for point in path: + if not isinstance(point, (list, tuple)) or len(point) < 2: + continue + points.append([round(max(0.0, min(1.0, float(point[0]))), 4), round(max(0.0, min(1.0, float(point[1]))), 4)]) + has_reference_path = len(points) >= 2 + if len(points) < 2: + points = _edge_path(objects.get(source), objects.get(target)) if len(points) < 2: continue dx = points[-1][0] - points[0][0] dy = points[-1][1] - points[0][1] dash_style = str(item.get("dash_style") or item.get("line_pattern") or "solid") - normalized.append({ + line_pattern = _coerce_choice(dash_style, VALID_LINE_PATTERNS, "solid") + render_style = _coerce_choice(item.get("render_style"), VALID_RENDER_STYLES, "") + route_intent = _coerce_choice(item.get("route_intent"), VALID_ROUTE_INTENTS, "") + visual_weight = _coerce_choice(item.get("visual_weight"), VALID_VISUAL_WEIGHTS, "") + preferred_axis = _coerce_choice(item.get("preferred_axis"), VALID_PREFERRED_AXES, "") + explicit_route_policy = str(item.get("route_policy") or "").strip() + route_policy = explicit_route_policy or ("preserve_reference_path" if has_reference_path else "bbox_route_from_hint") + route_style = str(item.get("route_style") or ("dashed_loop" if str(item.get("control_kind") or "").lower() == "dashed_loop" else "")) + if line_pattern in {"dash", "dashed"} and len(points) >= 3 and not route_style: + route_style = "dashed_spline_like" + record = { "id": str(item.get("id") or f"arrow_{idx:02d}"), "source": source, "target": target, "source_id": source, "target_id": target, + "target_ids": [str(target_id) for target_id in item.get("target_ids", []) if str(target_id) in slot_ids] if isinstance(item.get("target_ids"), list) else [], "control_kind": str(item.get("control_kind") or "straight_arrow"), "path_percent": points, "arrowhead_direction": float(item.get("arrowhead_direction") if item.get("arrowhead_direction") is not None else math.degrees(math.atan2(dy, dx))), "stroke_color": str(item.get("stroke_color") or (palette[2] if len(palette) > 2 else "#4A6FA5")), "stroke_width_pt": float(item.get("stroke_width_pt") or 1.5), "dash_style": dash_style, - "line_pattern": "dash" if dash_style.lower() in {"dash", "dashed"} else "solid", - "route_style": str(item.get("route_style") or ("dashed_loop" if str(item.get("control_kind") or "").lower() == "dashed_loop" else "")), + "line_pattern": "dash" if line_pattern in {"dash", "dashed"} else "solid", + "route_style": route_style, + "render_style": render_style, + "route_intent": route_intent, + "visual_weight": visual_weight, + "preferred_axis": preferred_axis, + "bend_side": str(item.get("bend_side") or ""), + "fill_color": str(item.get("fill_color") or ""), + "outline_color": str(item.get("outline_color") or ""), + "outline_width_pt": item.get("outline_width_pt"), + "block_arrow_thickness_percent": item.get("block_arrow_thickness_percent"), + "line_cap": str(item.get("line_cap") or ""), + "arrowhead_size": str(item.get("arrowhead_size") or ""), + "binding_source": str(item.get("binding_source") or ("heuristic" if str(item.get("detected_by") or "").startswith(("cv_", "sequence_")) else "vlm")), + "route_policy": route_policy, "confidence": float(item.get("confidence") or 0.8), "detected_by": str(item.get("detected_by") or "vlm"), "editable_in": "pptx", "render_policy": "ppt_shape_not_image_asset", - }) + } + normalized.append({key: value for key, value in record.items() if value not in ("", [], None)}) return normalized +def _disagreement_brief(item: dict | None) -> dict | None: + if not isinstance(item, dict): + return None + return { + key: item.get(key) + for key in ( + "id", + "source_id", + "target_id", + "control_kind", + "render_style", + "route_intent", + "visual_weight", + "path_percent", + "stroke_color", + "stroke_width_pt", + "dash_style", + "line_pattern", + "confidence", + "detected_by", + ) + if key in item + } + + +def _control_disagreements(heuristic: list[dict], vlm: list[dict]) -> dict: + matched_heuristic: set[int] = set() + matched_vlm: set[int] = set() + disagreements: list[dict] = [] + duplicates: list[dict] = [] + for v_idx, vlm_item in enumerate(vlm): + best_idx = None + best_score = -1.0 + for h_idx, heuristic_item in enumerate(heuristic): + if h_idx in matched_heuristic: + continue + if not _controls_match_for_disagreement(heuristic_item, vlm_item): + continue + score = _bbox_iou(_path_bbox(heuristic_item.get("path_percent") or []), _path_bbox(vlm_item.get("path_percent") or [])) + if _control_signature(heuristic_item) == _control_signature(vlm_item): + score += 1.0 + score -= _path_distance(heuristic_item.get("path_percent") or [], vlm_item.get("path_percent") or []) + if score > best_score: + best_idx = h_idx + best_score = score + if best_idx is None: + continue + heuristic_item = heuristic[best_idx] + matched_heuristic.add(best_idx) + matched_vlm.add(v_idx) + same_style = ( + str(heuristic_item.get("render_style") or "") == str(vlm_item.get("render_style") or "") + or not heuristic_item.get("render_style") + or not vlm_item.get("render_style") + ) + same_route = ( + str(heuristic_item.get("route_intent") or "") == str(vlm_item.get("route_intent") or "") + or not heuristic_item.get("route_intent") + or not vlm_item.get("route_intent") + ) + if _controls_are_duplicate(heuristic_item, vlm_item) and same_style and same_route: + duplicates.append({ + "id": f"duplicate_{len(duplicates)+1:02d}", + "kind": "duplicate", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + else: + disagreements.append({ + "id": f"control_mismatch_{len(disagreements)+1:02d}", + "kind": "mismatch", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + for idx, item in enumerate(heuristic): + if idx not in matched_heuristic: + disagreements.append({ + "id": f"heuristic_only_{len(disagreements)+1:02d}", + "kind": "heuristic_only", + "heuristic": _disagreement_brief(item), + "vlm": None, + "default_decision": "keep_heuristic", + }) + for idx, item in enumerate(vlm): + if idx not in matched_vlm: + disagreements.append({ + "id": f"vlm_only_{len(disagreements)+1:02d}", + "kind": "vlm_only", + "heuristic": None, + "vlm": _disagreement_brief(item), + "default_decision": "keep_vlm", + }) + return { + "summary": "Control disagreements between heuristic/CV and VLM arrow detection.", + "heuristic_count": len(heuristic), + "vlm_count": len(vlm), + "duplicate_count": len(duplicates), + "disagreement_count": len(disagreements), + "duplicates": duplicates, + "disagreements": disagreements, + } + + +def _call_adjudicator(adapter: Callable, reference: Path, bindable: list[dict], disagreements: dict, flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, disagreements, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, disagreements) + + +def _decision_by_id(raw: dict | list | None) -> dict[str, dict]: + decisions = raw.get("decisions", []) if isinstance(raw, dict) else raw + result = {} + for item in decisions or []: + if not isinstance(item, dict): + continue + disagreement_id = str(item.get("disagreement_id") or item.get("id") or "") + if disagreement_id: + result[disagreement_id] = item + return result + + +def _append_unique(arrows: list[dict], candidate: dict) -> bool: + if any(_controls_are_duplicate(existing, candidate) for existing in arrows): + return False + candidate_id = str(candidate.get("id") or f"arrow_{len(arrows)+1:02d}") + used_ids = {str(item.get("id") or "") for item in arrows} + if candidate_id in used_ids: + stem = candidate_id + suffix = 2 + while f"{stem}_{suffix}" in used_ids: + suffix += 1 + candidate = dict(candidate) + candidate["id"] = f"{stem}_{suffix}" + arrows.append(candidate) + return True + + +def _merge_with_adjudication( + heuristic: list[dict], + vlm: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + adjudicator: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict, dict]: + disagreements = _control_disagreements(heuristic, vlm) + write_json(out / "reference_control_disagreements.json", disagreements) + if mode == "off": + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(vlm), + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return vlm, disagreements, adjudication_doc + raw_adjudication: dict | list | None = None + decisions: dict[str, dict] = {} + status = "skipped" + if mode == "vlm" and adjudicator and disagreements["disagreement_count"]: + try: + raw_adjudication = _call_adjudicator(adjudicator, reference, bindable, disagreements, flow_graph) + decisions = _decision_by_id(raw_adjudication) + status = "used" + except Exception as exc: + status = "fallback" + warnings.append(f"control_adjudication_failed:{exc}") + elif mode == "vlm" and disagreements["disagreement_count"]: + status = "unavailable" + merged: list[dict] = [] + for duplicate in disagreements["duplicates"]: + vlm_item = duplicate.get("vlm") + normalized = _normalize_controls([vlm_item], bindable, palette) + if normalized: + _append_unique(merged, normalized[0]) + for item in disagreements["disagreements"]: + decision = decisions.get(str(item.get("id") or ""), {}) + action = str(decision.get("decision") or item.get("default_decision") or "").lower() + selected: list[dict] = [] + if action == "keep_heuristic": + if item.get("heuristic"): + selected = [item["heuristic"]] + elif action == "keep_both": + selected = [candidate for candidate in [item.get("vlm"), item.get("heuristic")] if candidate] + elif action == "drop_both": + selected = [] + elif action == "merge_or_adjust": + selected = [decision.get("merged_control") or decision.get("control") or item.get("vlm") or item.get("heuristic")] + else: + selected = [item.get("vlm") or item.get("heuristic")] + normalized = _normalize_controls(selected, bindable, palette) + if len(normalized) < len([candidate for candidate in selected if candidate]): + warnings.append(f"invalid_control_adjudication_item_dropped:{item.get('id')}") + for candidate in normalized: + _append_unique(merged, candidate) + if not merged: + merged = vlm or heuristic + adjudication_doc = { + "summary": "Control disagreement adjudication report.", + "mode": mode, + "status": status, + "decision_count": len(decisions), + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(merged), + "raw_adjudication": raw_adjudication, + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return merged, disagreements, adjudication_doc + + +def _call_coverage_audit(adapter: Callable, reference: Path, bindable: list[dict], arrows: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, arrows, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, arrows) + + +def _apply_coverage_audit( + arrows: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + coverage_audit_adapter: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict]: + audit_doc: dict = { + "summary": "Control coverage audit report.", + "mode": mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } + if mode != "vlm": + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + if not coverage_audit_adapter: + audit_doc["status"] = "unavailable" + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + try: + raw = _call_coverage_audit(coverage_audit_adapter, reference, bindable, arrows, flow_graph) + raw_items = raw.get("missing_controls") if isinstance(raw, dict) else raw + if raw_items is None and isinstance(raw, dict): + raw_items = raw.get("arrows", []) + normalized = _normalize_controls(raw_items or [], bindable, palette) + appended = 0 + for candidate in normalized: + candidate = dict(candidate) + candidate["detected_by"] = "coverage_audit_vlm" + candidate["binding_source"] = "coverage_audit_vlm" + if _append_unique(arrows, candidate): + appended += 1 + audit_doc.update({ + "status": "used", + "raw_audit": raw, + "missing_control_count": len(raw_items or []), + "valid_missing_control_count": len(normalized), + "appended_control_count": appended, + }) + invalid_count = max(0, len(raw_items or []) - len(normalized)) + if invalid_count: + warnings.append(f"{invalid_count}_invalid_coverage_audit_control(s)_dropped") + except Exception as exc: + audit_doc["status"] = "fallback" + audit_doc["fallback_reason"] = f"control_coverage_audit_failed:{exc}" + warnings.append(str(audit_doc["fallback_reason"])) + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + + def _draw_overlay(reference_path: Path, out_path: Path, arrows: list[dict]) -> None: with Image.open(reference_path).convert("RGB") as image: draw = ImageDraw.Draw(image) @@ -185,6 +659,39 @@ def _draw_overlay(reference_path: Path, out_path: Path, arrows: list[dict]) -> N image.save(out_path) +def _flow_edges(flow_graph: dict | None) -> list[dict]: + if not isinstance(flow_graph, dict): + return [] + return [item for item in flow_graph.get("edges", []) or [] if isinstance(item, dict)] + + +def _flow_graph_stats(flow_graph: dict | None, objects: list[dict]) -> dict: + object_ids = {str(item.get("id") or "") for item in objects if isinstance(item, dict)} + edges = _flow_edges(flow_graph) + invalid = [] + for edge in edges: + source = str(edge.get("source_id") or edge.get("source") or "") + target = str(edge.get("target_id") or edge.get("target") or "") + if source not in object_ids or target not in object_ids: + invalid.append(str(edge.get("id") or f"{source}->{target}")) + return { + "flow_graph_used": bool(edges), + "flow_graph_edge_count": len(edges), + "invalid_flow_edge_count": len(invalid), + "invalid_flow_edge_ids": invalid, + } + + +def _call_control_adapter(adapter: Callable, reference: Path, bindable: list[dict], heuristic: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, heuristic, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, heuristic) + + def localize_reference_controls( reference_path: str | Path, slots: list[dict], @@ -192,46 +699,134 @@ def localize_reference_controls( out_dir: str | Path, mode: str = "hybrid", control_adapter: Callable[[str | Path, list[dict], list[dict]], dict | list[dict]] | None = None, + flow_graph: dict | None = None, + control_adjudication_mode: str = "off", + control_adjudicator: Callable | None = None, + control_coverage_audit_mode: str = "off", + coverage_audit_adapter: Callable | None = None, ) -> dict: reference = Path(reference_path) out = Path(out_dir) warnings = [] - heuristic = _cv_line_controls(reference, slots, palette) or _sequence_controls(slots, palette) + bindable = _bindable_objects(out, slots) + flow_stats = _flow_graph_stats(flow_graph, bindable) + if flow_stats["invalid_flow_edge_count"]: + warnings.append(f"{flow_stats['invalid_flow_edge_count']}_invalid_flow_graph_edge(s)") + cv_heuristic = _cv_line_controls(reference, bindable, palette) + heuristic = cv_heuristic or _sequence_controls(slots, palette) _draw_overlay(reference, out / "reference_controls_candidates_overlay.png", heuristic) arrows = heuristic vlm_status = "not_requested" + normalized: list[dict] = [] + raw: dict | list | None = None + disagreements = { + "summary": "Control disagreement detection skipped.", + "heuristic_count": len(heuristic), + "vlm_count": 0, + "duplicate_count": 0, + "disagreement_count": 0, + "duplicates": [], + "disagreements": [], + } + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": control_adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": 0, + "input_duplicate_count": 0, + "merged_arrow_count": len(arrows), + } + audit_doc = { + "summary": "Control coverage audit skipped.", + "mode": control_coverage_audit_mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } if mode in {"vlm", "hybrid"}: if control_adapter: try: - raw = control_adapter(reference, slots, heuristic) + raw = _call_control_adapter(control_adapter, reference, bindable, heuristic, flow_graph) raw_arrows = raw.get("arrows", []) if isinstance(raw, dict) else raw - normalized = _normalize_controls(raw_arrows, slots, palette) + normalized = _normalize_controls(raw_arrows, bindable, palette) if normalized: - arrows = normalized vlm_status = "used" + if mode == "hybrid": + arrows, disagreements, adjudication_doc = _merge_with_adjudication( + heuristic, + normalized, + bindable, + palette, + reference, + out, + flow_graph, + control_adjudication_mode, + control_adjudicator, + warnings, + ) + else: + arrows = normalized + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) invalid_count = max(0, len(raw_arrows or []) - len(normalized)) if invalid_count: warnings.append(f"{invalid_count}_invalid_vlm_control_arrow(s)_dropped") else: vlm_status = "fallback" warnings.append("control_adapter_returned_no_valid_arrows") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) except Exception as exc: vlm_status = "fallback" warnings.append(f"vlm_control_failed:{exc}") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) else: vlm_status = "unavailable_fallback_to_heuristic" if mode == "vlm": warnings.append("control_mode_vlm_requested_without_adapter") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + else: + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) if mode == "manual": arrows = [] vlm_status = "manual_empty" _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) + if mode in {"vlm", "hybrid"}: + arrows, audit_doc = _apply_coverage_audit( + arrows, + bindable, + palette, + reference, + out, + flow_graph, + control_coverage_audit_mode, + coverage_audit_adapter, + warnings, + ) + _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) return { "summary": "Reference control localization report.", "mode": mode, "status": "ok", "vlm_status": vlm_status, - "vlm_model": str(raw.get("_vlm_model")) if mode in {"vlm", "hybrid"} and isinstance(locals().get("raw"), dict) and raw.get("_vlm_model") else None, + "vlm_model": str(raw.get("_vlm_model")) if mode in {"vlm", "hybrid"} and isinstance(raw, dict) and raw.get("_vlm_model") else None, + "bindable_object_count": len(bindable), + "heuristic_arrow_count": len(heuristic), + "cv_heuristic_arrow_count": len(cv_heuristic), + "vlm_arrow_count": len(normalized), + "merged_arrow_count": len(arrows), + "control_adjudication_mode": control_adjudication_mode, + "control_adjudication_status": adjudication_doc.get("status"), + "control_disagreement_count": disagreements.get("disagreement_count", 0), + "control_duplicate_count": disagreements.get("duplicate_count", 0), + "control_coverage_audit_mode": control_coverage_audit_mode, + "control_coverage_audit_status": audit_doc.get("status"), + "control_coverage_appended_count": audit_doc.get("appended_control_count", 0), "warnings": warnings, "arrows": arrows, + **flow_stats, } diff --git a/rfs/cv_compat.py b/rfs/cv_compat.py new file mode 100644 index 0000000..53abbee --- /dev/null +++ b/rfs/cv_compat.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from typing import Any + + +def hough_line_coordinates(line: Any) -> tuple[int, int, int, int] | None: + """Normalize OpenCV HoughLinesP rows across array-shape variants.""" + try: + values = line.reshape(-1).tolist() if hasattr(line, "reshape") else line + while isinstance(values, (list, tuple)) and len(values) == 1 and isinstance(values[0], (list, tuple)): + values = values[0] + if not isinstance(values, (list, tuple)) or len(values) < 4: + return None + return tuple(int(value) for value in values[:4]) + except (TypeError, ValueError): + return None diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 54b571e..4106ea3 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -1,855 +1,14 @@ -from __future__ import annotations - -import base64 -import json -import mimetypes -import os -import platform -import shutil -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path -from typing import Callable - -from PIL import Image, ImageChops, ImageDraw - -from .control_localizer import localize_reference_controls -from .layout_planner import dominant_palette, estimate_background, plan_reference_layout -from .layout_semantic_planner import plan_slot_semantics -from .ppt_compiler import compile_ppt -from .rebuild_vlm_validation import build_rebuild_vlm_validation_report -from .text_layer import build_text_layer -from .utils import ensure_dir, write_json, write_text - - -ASSET_THRESHOLDS = { - "character": (0.80, 0.95), - "document_stack": (0.75, 0.95), - "chart_card": (0.75, 0.95), - "inspection": (0.70, 0.95), - "tool_icon": (0.80, 0.95), - "tool_combo": (0.70, 0.95), - "device": (0.70, 0.95), - "screenshot_card": (0.75, 0.95), - "legend_marker": (0.80, 0.95), - "thin_tool": (0.50, 0.95), - "generic": (0.70, 0.95), -} - - -def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: - x = max(0.0, min(0.995, x)) - y = max(0.0, min(0.995, y)) - w = max(0.001, min(w, 1.0 - x)) - h = max(0.001, min(h, 1.0 - y)) - return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} - - -def _canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: - ratio = width_px / max(height_px, 1) - width = max(10.0, min(15.6, 7.5 * ratio)) - return round(width, 3), round(width / ratio, 3) - - -def _rgb_hex(rgb: tuple[int, int, int]) -> str: - return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" - - -def _estimate_background(image: Image.Image) -> str: - rgb = image.convert("RGB") - w, h = rgb.size - samples = [] - for box in [(0, 0, w, max(1, h // 12)), (0, h - max(1, h // 12), w, h), (0, 0, max(1, w // 12), h), (w - max(1, w // 12), 0, w, h)]: - crop = rgb.crop(box).resize((1, 1)) - samples.append(crop.getpixel((0, 0))) - avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) - return _rgb_hex(avg) - - -def _dominant_palette(image: Image.Image, count: int = 6) -> list[str]: - small = image.convert("RGB").resize((96, 96)) - colors = small.quantize(colors=count).convert("RGB").getcolors(96 * 96) or [] - colors = sorted(colors, key=lambda item: item[0], reverse=True) - return [_rgb_hex(rgb) for _n, rgb in colors[:count]] - - -def _supported_aspect_ratio(width: float, height: float) -> str: - ratio = width / max(height, 0.001) - if ratio >= 1.55: - return "16:9" - if ratio >= 1.18: - return "4:3" - if ratio >= 0.85: - return "1:1" - if ratio >= 0.62: - return "3:4" - return "9:16" - - -def _slot_type_for_box(slot: dict) -> str: - semantic_type = str(slot.get("asset_type") or "").strip() - if semantic_type: - return semantic_type - ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) - if ratio >= 2.2: - return "thin_tool" - if ratio >= 1.35: - return "chart_card" - if ratio <= 0.78: - return "character" - return "generic" - - -def _threshold_for_type(slot_type: str) -> tuple[float, float]: - return ASSET_THRESHOLDS.get(slot_type, ASSET_THRESHOLDS["generic"]) - - -def _foreground_metrics(image_path: str | Path, background_hex: str | None = None) -> dict: - try: - image = Image.open(image_path).convert("RGB") - except Exception as exc: - return {"status": "error", "error": str(exc), "foreground_bbox_px": None, "foreground_bbox_fill_percent": 0.0} - w, h = image.size - bg = background_hex or _estimate_background(image) - bg_rgb = tuple(int(bg.strip("#")[i:i + 2], 16) for i in (0, 2, 4)) - bg_img = Image.new("RGB", image.size, bg_rgb) - diff = ImageChops.difference(image, bg_img).convert("L") - mask = diff.point(lambda p: 255 if p > 18 else 0) - box = mask.getbbox() - if not box: - return { - "status": "empty", - "foreground_bbox_px": None, - "foreground_bbox_fill_percent": 0.0, - "margin_left_percent": 0.5, - "margin_right_percent": 0.5, - "margin_top_percent": 0.5, - "margin_bottom_percent": 0.5, - } - left, top, right, bottom = box - fill = ((right - left) * (bottom - top)) / max(w * h, 1) - return { - "status": "ok", - "foreground_bbox_px": [left, top, right, bottom], - "foreground_bbox_fill_percent": round(fill, 4), - "margin_left_percent": round(left / w, 4), - "margin_right_percent": round((w - right) / w, 4), - "margin_top_percent": round(top / h, 4), - "margin_bottom_percent": round((h - bottom) / h, 4), - } - - -def economy_acceptance_decision(slot_type: str, fill_percent: float, strict: bool = False) -> dict: - low, high = (0.80, 0.95) if strict else _threshold_for_type(slot_type) - accepted = low <= float(fill_percent) <= high - if accepted: - reason = f"fill {fill_percent:.3f} within {slot_type} threshold {low:.2f}-{high:.2f}" - elif float(fill_percent) < low: - reason = f"fill {fill_percent:.3f} below {slot_type} threshold {low:.2f}" - else: - reason = f"fill {fill_percent:.3f} above tightness threshold {high:.2f}" - return {"accepted": accepted, "selected_reason": reason, "threshold_min": low, "threshold_max": high} - - -def _find_visual_components(reference_path: Path, canvas_w: int, canvas_h: int) -> list[dict]: - try: - import cv2 - import numpy as np - except Exception: - return [] - image = cv2.imread(str(reference_path)) - if image is None: - return [] - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - blur = cv2.GaussianBlur(gray, (5, 5), 0) - edges = cv2.Canny(blur, 45, 120) - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) - dilated = cv2.dilate(edges, kernel, iterations=2) - contours, _hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - boxes = [] - image_area = canvas_w * canvas_h - for contour in contours: - x, y, w, h = cv2.boundingRect(contour) - area = w * h - if area < image_area * 0.008 or area > image_area * 0.35: - continue - if w < 24 or h < 24: - continue - boxes.append((x, y, w, h)) - boxes.sort(key=lambda item: (item[1], item[0])) - kept: list[tuple[int, int, int, int]] = [] - for box in boxes: - x, y, w, h = box - duplicate = False - for kx, ky, kw, kh in kept: - ix = max(0, min(x + w, kx + kw) - max(x, kx)) - iy = max(0, min(y + h, ky + kh) - max(y, ky)) - overlap = ix * iy / max(w * h, 1) - if overlap > 0.55: - duplicate = True - break - if not duplicate: - kept.append(box) - if len(kept) >= 12: - break - slots = [] - for idx, (x, y, w, h) in enumerate(kept, start=1): - slots.append({ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": _bbox(x / canvas_w, y / canvas_h, w / canvas_w, h / canvas_h), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - }) - return slots - - -def _fallback_slots(count: int = 6) -> list[dict]: - boxes = [ - (0.08, 0.26, 0.16, 0.22), - (0.31, 0.26, 0.16, 0.22), - (0.54, 0.26, 0.16, 0.22), - (0.77, 0.26, 0.16, 0.22), - (0.25, 0.58, 0.18, 0.22), - (0.57, 0.58, 0.18, 0.22), - ][:count] - slots = [] - for idx, (x, y, w, h) in enumerate(boxes, start=1): - slots.append({ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": _bbox(x, y, w, h), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - }) - return slots - - -def _style_from_layout(layout: dict) -> dict: - palette = layout.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52", "#6B57C8", "#D44E5D"] - background = str(layout.get("canvas", {}).get("background") or "#FFFFFF") - style = { - "palette": palette, - "reference_palette": palette, - "color_tokens": [{"token_id": f"palette_{i:02d}", "hex": color} for i, color in enumerate(palette)], - "panel_styles": { - "reference_canvas": { - "fill_color": background, - "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", - "header_color": palette[1] if len(palette) > 1 else "#6C7A89", - } - }, - "arrow_weight_pt": 1.7, - } - for panel in layout.get("panels", []): - if panel.get("id") and panel["id"] != "reference_canvas": - style["panel_styles"].setdefault(str(panel["id"]), { - "fill_color": background, - "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", - "header_color": palette[1] if len(palette) > 1 else "#6C7A89", - }) - return style - - -def _build_program( - reference_path: Path, - out: Path, - layout_mode: str = "hybrid", - vlm_layout_adapter: Callable | None = None, -) -> tuple[dict, dict]: - layout = plan_reference_layout(reference_path, out, mode=layout_mode, vlm_adapter=vlm_layout_adapter) - style = _style_from_layout(layout) - program = { - "canvas": layout["canvas"], - "style": style, - "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, - "panels": layout.get("panels", []), - "cards": layout.get("cards", []), - "slots": layout.get("slots", []), - "assets": [], - "arrows": [], - "labels": [], - "groups": [], - "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}], - } - return program, layout - - -def _crop_reference(reference_path: Path, out_path: Path, bbox_percent: dict) -> Path: - with Image.open(reference_path).convert("RGB") as image: - w, h = image.size - left = int(float(bbox_percent["x"]) * w) - top = int(float(bbox_percent["y"]) * h) - right = int((float(bbox_percent["x"]) + float(bbox_percent["w"])) * w) - bottom = int((float(bbox_percent["y"]) + float(bbox_percent["h"])) * h) - crop = image.crop((max(0, left), max(0, top), min(w, right), min(h, bottom))) - out_path.parent.mkdir(parents=True, exist_ok=True) - crop.save(out_path) - return out_path +"""Compatibility alias for :mod:`rfs.workflows.rebuild_editable`. +The reference-image rebuild implementation now belongs to the workflows layer. +This alias preserves the historical import path and private monkeypatch targets. +""" -def _placeholder_asset(slot: dict, out_path: Path, background: str) -> None: - out_path.parent.mkdir(parents=True, exist_ok=True) - ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) - if ratio >= 1.55: - size = (512, 288) - elif ratio <= 0.62: - size = (288, 512) - else: - size = (384, 384) - image = Image.new("RGB", size, background) - draw = ImageDraw.Draw(image) - pad = int(min(size) * 0.12) - draw.rounded_rectangle((pad, pad, size[0] - pad, size[1] - pad), radius=18, fill="#FFFFFF", outline="#6C7A89", width=3) - draw.line((pad * 1.5, size[1] * 0.45, size[0] - pad * 1.5, size[1] * 0.45), fill="#4A6FA5", width=5) - draw.ellipse((size[0] * 0.42, size[1] * 0.22, size[0] * 0.58, size[1] * 0.38), fill="#E17721") - draw.text((pad * 1.3, size[1] * 0.58), slot["id"], fill="#263747") - image.save(out_path) - - -def _api_generate_asset(spec: dict, crop_path: Path, out_path: Path) -> dict: - import requests - - api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") - url = os.getenv("GEMINI_GEN_IMG_URL") - if not api_key or not url: - raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode api") - mime = mimetypes.guess_type(crop_path.name)[0] or "image/png" - image_b64 = base64.b64encode(crop_path.read_bytes()).decode("ascii") - prompt = spec["prompt"] - payload = { - "contents": [{ - "role": "user", - "parts": [ - {"text": prompt}, - {"inlineData": {"mimeType": mime, "data": image_b64}}, - ], - }], - "generationConfig": { - "responseModalities": ["IMAGE"], - "imageConfig": {"imageSize": "1K", "aspectRatio": spec["generation_aspect_ratio"]}, - }, - } - response = requests.post(url, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), timeout=240) - response.raise_for_status() - data = response.json() - for candidate in data.get("candidates", []): - for part in candidate.get("content", {}).get("parts", []): - inline = part.get("inlineData") or part.get("inline_data") - if inline and inline.get("data"): - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_bytes(base64.b64decode(inline["data"])) - return {"status": "generated", "api_requests_attempted": 1} - raise RuntimeError("API returned no image data") - - -def _fill_target_for_type(slot_type: str) -> tuple[float, float]: - return _threshold_for_type(slot_type) - - -def _asset_prompt(slot: dict, slot_type: str, background: str, aspect_ratio: str) -> str: - low, high = _fill_target_for_type(slot_type) - subject = str(slot.get("prompt_subject") or slot.get("paper_concept") or slot.get("id")) - nearby = slot.get("nearby_text") or [] - nearby_text = " | ".join(nearby) if isinstance(nearby, list) else str(nearby or "") - orientation = "horizontal" if aspect_ratio in {"16:9", "4:3"} else "vertical" if aspect_ratio in {"3:4", "9:16"} else "square" - return ( - "Use the provided reference crop only as guidance for object identity, pose, colors, and diagram style. " - "Generate one clean standalone visual asset for an editable PowerPoint reconstruction. " - f"Slot type: {slot_type}. Canvas aspect ratio: {aspect_ratio}. Composition orientation: {orientation}. " - f"Main subject: {subject}. " - f"Nearby reference text for semantics only: {nearby_text}. " - f"The main subject should occupy {int(low * 100)}%-{int(high * 100)}% of the image canvas, with only narrow uniform margins. " - "For horizontal slots, make the subject span the width; for vertical slots, make the subject span the height. " - "Avoid a tiny object floating in a large colored field. " - f"Use a flat background matching {background}. " - "Do not include readable text, captions, watermarks, arrows, panel borders, or the whole source diagram. " - "Do not crop off important details." - ) - - -def _load_accepted_assets(path: Path) -> dict: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - if isinstance(data, dict): - return data - return {} - - -def _make_asset_specs(program: dict, reference_path: Path, out: Path) -> list[dict]: - specs = [] - background = str(program["canvas"].get("background") or "#FFFFFF") - for slot in program["slots"]: - bbox = slot["bbox_percent"] - slot_type = _slot_type_for_box(slot) - fill_min, fill_max = _fill_target_for_type(slot_type) - slot_ratio = float(bbox["w"]) / max(float(bbox["h"]), 0.001) - generation_ratio = str(slot.get("generation_aspect_ratio") or _supported_aspect_ratio(float(bbox["w"]), float(bbox["h"]))) - slot_background = str(slot.get("background_color_hex") or background) - crop_path = out / "reference_slot_crops" / f"{slot['id']}.png" - _crop_reference(reference_path, crop_path, bbox) - spec = { - "slot_id": slot["id"], - "asset_id": slot["asset_id"], - "slot_type": slot_type, - "asset_type": slot_type, - "semantic_role": slot.get("semantic_role"), - "nearby_text": slot.get("nearby_text"), - "panel_context": slot.get("panel_context"), - "upstream_ids": slot.get("upstream_ids", []), - "downstream_ids": slot.get("downstream_ids", []), - "prompt_subject": slot.get("prompt_subject") or slot.get("paper_concept") or slot.get("id"), - "slot_bbox_percent": bbox, - "slot_aspect_ratio": round(slot_ratio, 4), - "generation_aspect_ratio": generation_ratio, - "background_color_hex": slot_background, - "reference_crop_path": str(crop_path), - "prompt": _asset_prompt(slot, slot_type, slot_background, generation_ratio), - "main_subject_fill_target": round((fill_min + fill_max) / 2, 3), - "internal_content_fill_target": round((fill_min + fill_max) / 2, 3), - "max_margin_percent": 0.10, - "forbidden_elements": ["readable text", "captions", "watermarks", "arrows", "panel borders", "full source diagram"], - "style_reference": "provided reference crop", - "no_text": True, - "no_arrows": True, - "no_full_diagram": True, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - } - specs.append(spec) - return specs - - -def _generate_assets( - specs: list[dict], - program: dict, - out: Path, - asset_mode: str, - asset_workers: int, - asset_retries: int, - economy_mode: bool, - regenerate_slots: set[str], - strict_asset_regeneration: bool, -) -> tuple[list[dict], dict]: - asset_dir = ensure_dir(out / "assets") - accepted = _load_accepted_assets(out / "accepted_assets.json") - reports: list[dict] = [] - api_requests = 0 - - def run_one(spec: dict) -> dict: - slot_id = spec["slot_id"] - out_path = asset_dir / f"{spec['asset_id']}.png" - locked = bool(accepted.get(slot_id, {}).get("accepted")) or bool(accepted.get(spec["asset_id"], {}).get("accepted")) - if out_path.exists() and economy_mode and slot_id not in regenerate_slots and not strict_asset_regeneration: - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) - if locked or decision["accepted"]: - return {**spec, **metrics, **decision, "status": "reused", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_existing_accepted_or_threshold_passed", "fallback_used": False} - crop_path = Path(spec["reference_crop_path"]) - attempts = max(1, int(asset_retries) + 1) if strict_asset_regeneration else 1 - last_error = None - for attempt in range(1, attempts + 1): - try: - if asset_mode == "placeholder": - _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) - request_count = 0 - status = "placeholder" - elif asset_mode == "crop": - shutil.copyfile(crop_path, out_path) - request_count = 0 - status = "reference_crop" - else: - result = _api_generate_asset(spec, crop_path, out_path) - request_count = int(result.get("api_requests_attempted", 1)) - status = result.get("status", "generated") - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=strict_asset_regeneration) - return { - **spec, - **metrics, - **decision, - "status": status, - "asset_path": str(out_path), - "api_requests_attempted": request_count, - "attempt": attempt, - "economy_decision": "generated_once" if economy_mode else "generated", - "fallback_used": False, - } - except Exception as exc: - last_error = str(exc) - if asset_mode == "api": - try: - shutil.copyfile(crop_path, out_path) - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) - return { - **spec, - **metrics, - **decision, - "status": "api_failed_reference_crop_fallback", - "asset_path": str(out_path), - "api_requests_attempted": 1, - "attempt": attempt, - "economy_decision": "fallback_after_api_error", - "fallback_used": True, - "error": last_error, - } - except Exception: - pass - return {**spec, "status": "failed", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": last_error} - - workers = max(1, min(int(asset_workers), 12)) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(run_one, spec) for spec in specs] - for future in as_completed(futures): - report = future.result() - reports.append(report) - api_requests += int(report.get("api_requests_attempted") or 0) - reports.sort(key=lambda item: str(item.get("slot_id"))) - program["assets"] = [{"id": spec["asset_id"], "path": f"assets/{spec['asset_id']}.png", "source": "slot_asset"} for spec in specs] - summary = { - "summary": "Asset generation report for reusable editable rebuild workflow.", - "asset_mode": asset_mode, - "economy_mode": economy_mode, - "asset_retries": asset_retries, - "strict_asset_regeneration": strict_asset_regeneration, - "api_requests_attempted": api_requests, - "assets": reports, - } - economy = { - "summary": "Economy policy decisions.", - "api_requests_attempted": api_requests, - "accepted_assets_file": str(out / "accepted_assets.json"), - "regenerate_slots": sorted(regenerate_slots), - "thresholds": {key: {"min": value[0], "max": value[1]} for key, value in ASSET_THRESHOLDS.items()}, - "assets": [{ - "slot_id": item.get("slot_id"), - "slot_type": item.get("slot_type"), - "economy_decision": item.get("economy_decision"), - "selected_reason": item.get("selected_reason"), - "api_requests_attempted": item.get("api_requests_attempted"), - "fallback_used": item.get("fallback_used"), - } for item in reports], - } - write_json(out / "asset_generation_report.json", summary) - write_json(out / "asset_economy_report.json", economy) - - def image_size(path_text: str | None) -> list[int] | None: - if not path_text or not Path(path_text).exists(): - return None - with Image.open(path_text) as img: - return list(img.size) - - write_json(out / "asset_ratio_fit_report.json", { - "summary": "Aspect-ratio fit report.", - "assets": [{ - "slot_id": item.get("slot_id"), - "slot_aspect_ratio": item.get("slot_aspect_ratio"), - "generation_aspect_ratio": item.get("generation_aspect_ratio"), - "generated_image_size": image_size(item.get("asset_path")), - "foreground_bbox_fill_percent": item.get("foreground_bbox_fill_percent"), - "slot_fit_waste_percent": None, - "aspect_ratio_match_status": "matched" if item.get("generation_aspect_ratio") else "unknown", - } for item in reports], - }) - return reports, summary - - -def _export_preview(pptx_path: Path, out: Path) -> Path | None: - preview = out / "rebuild_preview.png" - if platform.system() != "Windows": - write_text(out / "preview_export_error.txt", "PowerPoint preview export is only available on Windows with Microsoft PowerPoint installed.") - return None - try: - import win32com.client - app = win32com.client.Dispatch("PowerPoint.Application") - app.Visible = True - deck = app.Presentations.Open(str(pptx_path), WithWindow=False) - deck.Slides(1).Export(str(preview), "PNG") - deck.Close() - app.Quit() - return preview - except Exception as exc: - write_text(out / "preview_export_error.txt", str(exc)) - return None - - -def _ppt_package_counts(pptx_path: Path, reference_path: Path) -> dict: - from pptx import Presentation - - prs = Presentation(str(pptx_path)) - slide_count = len(prs.slides) - shape_count = 0 - picture_count = 0 - text_shape_count = 0 - connector_count = 0 - for slide in prs.slides: - for shape in slide.shapes: - shape_count += 1 - if getattr(shape, "shape_type", None) == 13: - picture_count += 1 - if getattr(shape, "has_text_frame", False) and shape.text_frame.text.strip(): - text_shape_count += 1 - shape_type_text = str(getattr(shape, "shape_type", "")) - if "CONNECTOR" in shape_type_text or "LINE" in shape_type_text: - connector_count += 1 - return { - "ppt_page_count": slide_count, - "shape_count": shape_count, - "picture_count": picture_count, - "text_shape_count": text_shape_count, - "connector_count": connector_count, - "contains_full_reference_image": False, - "reference_path": str(reference_path), - } - - -def _load_json_or_empty(path: Path) -> dict: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - return data if isinstance(data, dict) else {} - - -def _program_from_contracts(out: Path) -> dict: - geometry = _load_json_or_empty(out / "reference_geometry.json") - controls = _load_json_or_empty(out / "reference_controls.json") - text_program = _load_json_or_empty(out / "text_program.json") - canvas = geometry.get("canvas") if isinstance(geometry.get("canvas"), dict) else {"width_in": 10.0, "height_in": 5.625, "background": "#FFFFFF"} - if "width_in" not in canvas or "height_in" not in canvas: - width_px = int(canvas.get("width_px") or 1600) - height_px = int(canvas.get("height_px") or 900) - width_in, height_in = _canvas_inches(width_px, height_px) - canvas = {**canvas, "width_in": width_in, "height_in": height_in, "background": canvas.get("background") or "#FFFFFF"} - layout = { - "canvas": canvas, - "palette": geometry.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52"], - "panels": geometry.get("panels") or [{ - "id": "reference_canvas", - "title": "Editable Rebuild", - "bbox_percent": _bbox(0.025, 0.065, 0.95, 0.84), - "editable_in": "pptx", - }], - } - slots = geometry.get("slots") or _load_json_or_empty(out / "slot_inventory.json").get("slots") or [] - program = { - "canvas": canvas, - "style": _style_from_layout(layout), - "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, - "panels": layout["panels"], - "cards": geometry.get("cards") or [], - "slots": slots, - "assets": [{"id": slot.get("asset_id") or slot.get("id"), "path": f"assets/{slot.get('asset_id') or slot.get('id')}.png", "source": "slot_asset"} for slot in slots], - "arrows": controls.get("arrows") or [], - "labels": [], - "groups": [], - "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}], - } - if isinstance(text_program.get("items"), list): - program["text_program"] = text_program - program["text_program_path"] = "text_program.json" - return program - - -def _compile_existing_contracts(reference_path: Path, out: Path, export_preview: bool = False) -> dict: - program = _program_from_contracts(out) - write_json(out / "figure_program.json", program) - pptx_path = compile_ppt(program, out) - preview = _export_preview(pptx_path, out) if export_preview else None - counts = _ppt_package_counts(pptx_path, reference_path) - report_path = out / "composition_quality_report.json" - report = _load_json_or_empty(report_path) - report["rebuild_editable_summary"] = counts - report["compile_only"] = True - write_json(report_path, report) - return { - "summary": "Editable rebuild recompiled from existing JSON contracts.", - "ok": True, - "out_dir": str(out), - "reference": str(reference_path), - "pptx": str(pptx_path), - "preview": str(preview) if preview else None, - "asset_count": len(program.get("assets", [])), - "slot_count": len(program.get("slots", [])), - "connector_count": len(program.get("arrows", [])), - "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, - "compile_only": True, - } - - -def rebuild_editable( - reference: str | Path, - out: str | Path, - asset_mode: str = "api", - asset_workers: int = 4, - asset_retries: int = 1, - economy_mode: bool = True, - text_mode: str = "ocr", - control_mode: str = "hybrid", - layout_mode: str = "hybrid", - export_preview: bool = False, - regenerate_slots: str | list[str] | None = None, - strict_asset_regeneration: bool = False, - skip_analysis: bool = False, - compile_only: bool = False, - ocr_engine: str = "paddle", - ocr_lang: str = "en_ch", - ocr_adapter: Callable | None = None, - vlm_layout_adapter: Callable | None = None, - control_adapter: Callable | None = None, - semantic_adapter: Callable | None = None, -) -> dict: - reference_path = Path(reference) - if not reference_path.exists(): - raise FileNotFoundError(reference_path) - out_path = ensure_dir(out) - if compile_only: - return _compile_existing_contracts(reference_path, out_path, export_preview=export_preview) - input_dir = ensure_dir(out_path / "inputs") - archived_reference = input_dir / reference_path.name - shutil.copyfile(reference_path, archived_reference) - write_json(out_path / "input_manifest.json", { - "summary": "Input manifest for reference-only editable rebuild.", - "reference": str(reference_path), - "archived_reference": str(archived_reference), - "asset_mode": asset_mode, - "text_mode": text_mode, - "control_mode": control_mode, - "layout_mode": layout_mode, - "skip_analysis": skip_analysis, - "compile_only": compile_only, - }) - - if skip_analysis and (out_path / "reference_geometry.json").exists(): - program = _program_from_contracts(out_path) - reference_geometry = _load_json_or_empty(out_path / "reference_geometry.json") - else: - program, reference_geometry = _build_program(archived_reference, out_path, layout_mode=layout_mode, vlm_layout_adapter=vlm_layout_adapter) - write_json(out_path / "reference_geometry.json", reference_geometry) - - if skip_analysis and (out_path / "reference_controls.json").exists(): - reference_controls = _load_json_or_empty(out_path / "reference_controls.json") - else: - reference_controls = localize_reference_controls( - archived_reference, - program.get("slots", []), - program.get("style", {}).get("palette", []), - out_path, - mode=control_mode, - control_adapter=control_adapter, - ) - program["arrows"] = reference_controls.get("arrows", []) - write_json(out_path / "reference_controls.json", reference_controls) - - if text_mode != "off": - extractor_mode = "ocr" if text_mode == "ocr" else "heuristic" - build_text_layer( - archived_reference, - program, - program.get("style", {}), - out_path, - text_extractor_mode=extractor_mode, - ocr_engine=ocr_engine, - ocr_lang=ocr_lang, - ocr_adapter=ocr_adapter, - ) - else: - write_json(out_path / "reference_text_geometry.json", {"summary": "Text extraction skipped.", "detection_mode": "off", "regions": []}) - write_json(out_path / "ocr_text_quality_report.json", {"summary": "Text extraction skipped.", "mode": "off", "status": "skipped"}) - - text_geometry = _load_json_or_empty(out_path / "reference_text_geometry.json") - semantic_report = _load_json_or_empty(out_path / "slot_semantic_report.json") if skip_analysis else {} - if not skip_analysis: - semantic_slots, semantic_report = plan_slot_semantics( - archived_reference, - program.get("slots", []), - program.get("panels", []), - program.get("arrows", []), - text_geometry, - semantic_adapter=semantic_adapter, - ) - program["slots"] = semantic_slots - reference_geometry["slots"] = semantic_slots - write_json(out_path / "reference_geometry.json", reference_geometry) - write_json(out_path / "slot_semantic_report.json", semantic_report) - elif not semantic_report: - semantic_report = {"summary": "Semantic planning skipped by --skip-analysis.", "semantic_vlm_status": "skipped", "slots": []} - write_json(out_path / "slot_semantic_report.json", semantic_report) - write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) - - specs = _make_asset_specs(program, archived_reference, out_path) - write_json(out_path / "asset_generation_specs.json", {"summary": "Slot-level asset generation specs.", "asset_mode": asset_mode, "specs": specs}) - regen = set() - if isinstance(regenerate_slots, str): - regen = {item.strip() for item in regenerate_slots.split(",") if item.strip()} - elif isinstance(regenerate_slots, list): - regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} - asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration) - vlm_validation_report = build_rebuild_vlm_validation_report(out_path, reference_geometry, reference_controls, semantic_report, asset_summary) +from __future__ import annotations - write_json(out_path / "figure_program.json", program) - pptx_path = compile_ppt(program, out_path) - preview = _export_preview(pptx_path, out_path) if export_preview else None +import importlib as _importlib +import sys as _sys - counts = _ppt_package_counts(pptx_path, archived_reference) - report_path = out_path / "composition_quality_report.json" - try: - report = json.loads(report_path.read_text(encoding="utf-8")) - except Exception: - report = {} - report["rebuild_editable_summary"] = counts - report["no_full_image_policy"] = { - "status": "pass", - "contains_full_reference_image": False, - "policy": "reference image is archived only; final PPT uses slot assets and editable objects", - } - write_json(report_path, report) +_implementation = _importlib.import_module(".workflows.rebuild_editable", __package__) - result = { - "summary": "Reusable image-to-editable-PPT rebuild complete.", - "ok": True, - "out_dir": str(out_path), - "reference": str(reference_path), - "pptx": str(pptx_path), - "preview": str(preview) if preview else None, - "asset_mode": asset_mode, - "asset_workers": asset_workers, - "asset_retries": asset_retries, - "economy_mode": economy_mode, - "api_requests_attempted": asset_summary.get("api_requests_attempted", 0), - "asset_count": len(asset_reports), - "slot_count": len(program.get("slots", [])), - "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, - "connector_count": len(program.get("arrows", [])), - "text_mode": text_mode, - "control_mode": control_mode, - "layout_mode": layout_mode, - "reports": { - "input_manifest": str(out_path / "input_manifest.json"), - "reference_geometry": str(out_path / "reference_geometry.json"), - "reference_text_geometry": str(out_path / "reference_text_geometry.json"), - "reference_controls": str(out_path / "reference_controls.json"), - "slot_inventory": str(out_path / "slot_inventory.json"), - "slot_semantic_report": str(out_path / "slot_semantic_report.json"), - "rebuild_vlm_validation_report": str(out_path / "rebuild_vlm_validation_report.json"), - "asset_generation_specs": str(out_path / "asset_generation_specs.json"), - "asset_generation_report": str(out_path / "asset_generation_report.json"), - "asset_economy_report": str(out_path / "asset_economy_report.json"), - "figure_program": str(out_path / "figure_program.json"), - "composition_quality_report": str(out_path / "composition_quality_report.json"), - }, - } - write_json(out_path / "rebuild_result.json", result) - return result +_sys.modules[__name__] = _implementation diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py new file mode 100644 index 0000000..d049060 --- /dev/null +++ b/rfs/evaluation/__init__.py @@ -0,0 +1,22 @@ +"""Deterministic and model-assisted quality evaluation APIs.""" + +from .rebuild_visual import apply_rebuild_corrections, run_rebuild_visual_critic, run_rebuild_visual_quality_check +from .benchmarking import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case +from .pdf_extraction_benchmark import run_pdf_extraction_stress_suite +from .outputs import validate_output + +__all__ = [ + "apply_rebuild_corrections", + "run_rebuild_visual_critic", + "run_rebuild_visual_quality_check", + "list_benchmark_cases", + "audit_paper_image_run", + "fetch_benchmark_case", + "run_benchmark_case", + "run_fast_benchmark_case", + "run_fast_benchmark_suite", + "run_pdf_extraction_stress_suite", + "score_benchmark_case", + "validate_benchmark_case", + "validate_output", +] diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py new file mode 100644 index 0000000..ece8ee7 --- /dev/null +++ b/rfs/evaluation/benchmarking.py @@ -0,0 +1,1206 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from pathlib import Path +from typing import Any + +from PIL import Image, ImageChops, ImageFilter, ImageStat +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE +import requests + +from ..utils import ensure_dir, read_json, write_json, write_text + + +SUITES = {"paper-to-image", "image-to-ppt"} + + +def _load(path: Path) -> dict: + if not path.exists(): + return {} + try: + value = read_json(path) + except Exception: + return {} + return value if isinstance(value, dict) else {} + + +def _mean(values: list[float]) -> float: + return round(sum(values) / max(1, len(values)), 4) + + +def _clamp(value: object) -> float: + try: + return round(max(0.0, min(1.0, float(value))), 4) + except Exception: + return 0.0 + + +def _threshold_failures(metrics: dict, thresholds: dict) -> list[str]: + maximum_metrics = { + "hallucination_count", + "forbidden_content_count", + "plan_forbidden_content_count", + "full_slide_image_count", + "blocking_visual_issue_count", + "evidence_conflict_count", + "unsupported_visual_claim_count", + "missing_paper_priority_count", + } + failures = [] + for key, threshold in thresholds.items(): + value = metrics.get(key) + if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)): + continue + if key in maximum_metrics and float(value) > float(threshold): + failures.append(f"{key}={value} above maximum {threshold}") + elif key not in maximum_metrics and float(value) < float(threshold): + failures.append(f"{key}={value} below minimum {threshold}") + return failures + + +def _normalized_text(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + + +def _normalized_label_variants(value: object) -> set[str]: + label = str(value or "").strip() + variants = {_normalized_text(label)} + without_suffix = re.sub(r"\s*\([^()]{1,24}\)\s*$", "", label).strip() + if without_suffix: + variants.add(_normalized_text(without_suffix)) + return {item for item in variants if item} + + +def _score_planning_contract(expected: dict, specification: dict) -> dict[str, Any]: + if not specification: + return {"available": False} + actual_entities = [] + for field in ("inputs", "modules", "outputs", "innovations", "must_show"): + for index, item in enumerate(specification.get(field, []) if isinstance(specification.get(field), list) else []): + if not isinstance(item, dict): + continue + label = next((str(item.get(key) or "").strip() for key in ("visible_label", "name", "text", "statement", "label", "id") if str(item.get(key) or "").strip()), "") + actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label), "normalized_variants": _normalized_label_variants(label), "field": field}) + expanded_relations = set() + for index, item in enumerate(specification.get("relations", []) if isinstance(specification.get("relations"), list) else []): + if not isinstance(item, dict): + continue + source = str(item.get("source") or item.get("source_id") or "") + target = str(item.get("target") or item.get("target_id") or "") + if source and target: + expanded_relations.add((source, target)) + label = str(item.get("label") or "").strip() + if source and target and label: + label_id = f"relation_label_{index}" + actual_entities.append({"id": label_id, "label": label, "normalized": _normalized_text(label), "normalized_variants": _normalized_label_variants(label), "field": "relation_label"}) + expanded_relations.add((source, label_id)) + expanded_relations.add((label_id, target)) + actual_relations = expanded_relations + adjacency: dict[str, set[str]] = {} + for source, target in actual_relations: + adjacency.setdefault(source, set()).add(target) + + def connected(source: str, target: str, max_hops: int = 2) -> bool: + if (source, target) in actual_relations: + return True + frontier = {source} + visited = {source} + for _ in range(max_hops): + frontier = {neighbor for node in frontier for neighbor in adjacency.get(node, set()) if neighbor not in visited} + if target in frontier: + return True + visited.update(frontier) + return False + + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] + mapping: dict[str, str] = {} + matched = [] + used_actual_ids: set[str] = set() + candidate_scores: dict[str, dict[str, int]] = {} + for item in expected_entities: + expected_id = str(item.get("id")) + primary = _normalized_text(item.get("label")) + aliases = [_normalized_text(value) for value in item.get("aliases", []) if _normalized_text(value)] + def match_score(actual: dict[str, Any]) -> int: + value = actual["normalized"] + variants = actual.get("normalized_variants") or {value} + if not value: + return 0 + bonus = 2 if actual.get("field") in {"inputs", "modules", "outputs", "innovations"} else 1 if actual.get("field") == "relation_label" else 0 + if primary and primary in variants: + return 5 + bonus + if any(alias in variants for alias in aliases): + return 4 + bonus + if any(alias in value or value in alias for alias in aliases): + return 3 + bonus + if primary and (primary in value or value in primary): + return 1 + bonus + return 0 + candidate_scores[expected_id] = {actual["id"]: match_score(actual) for actual in actual_entities} + ranked = sorted( + ((score, actual) for actual in actual_entities if actual["id"] not in used_actual_ids and (score := candidate_scores[expected_id][actual["id"]]) > 0), + key=lambda pair: pair[0], + reverse=True, + ) + match = ranked[0][1] if ranked and ranked[0][0] > 0 else None + if match: + mapping[expected_id] = match["id"] + used_actual_ids.add(match["id"]) + matched.append(expected_id) + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] + for expected_id, current_actual in list(mapping.items()): + blocked = set(mapping.values()) - {current_actual} + + def relation_support(actual_id: str) -> int: + support = 0 + for relation in expected_relations: + source_expected = str(relation.get("source")) + target_expected = str(relation.get("target")) + if source_expected == expected_id and target_expected in mapping: + support += int(connected(actual_id, mapping[target_expected])) + elif target_expected == expected_id and source_expected in mapping: + support += int(connected(mapping[source_expected], actual_id)) + return support + + alternatives = [ + actual_id + for actual_id, score in candidate_scores.get(expected_id, {}).items() + if score > 0 and actual_id not in blocked + ] + if alternatives: + mapping[expected_id] = max( + alternatives, + key=lambda actual_id: ( + candidate_scores[expected_id][actual_id], + relation_support(actual_id), + actual_id == current_actual, + ), + ) + relation_matches = 0 + matched_relations = [] + missing_relations = [] + for relation in expected_relations: + source = mapping.get(str(relation.get("source"))) + target = mapping.get(str(relation.get("target"))) + if source and target and connected(source, target): + relation_matches += 1 + matched_relations.append(f"{relation.get('source')}->{relation.get('target')}") + else: + missing_relations.append(f"{relation.get('source')}->{relation.get('target')}") + actual_text = " ".join(item["label"] for item in actual_entities).casefold() + forbidden = [label for label in expected.get("forbidden_labels", []) if str(label).casefold() in actual_text] + return { + "available": True, + "entity_recall": _clamp(len(matched) / max(1, len(expected_entities))), + "relation_recall": _clamp(relation_matches / max(1, len(expected_relations))), + "matched_entity_ids": matched, + "entity_mapping": mapping, + "missing_entity_ids": [str(item.get("id")) for item in expected_entities if str(item.get("id")) not in matched], + "matched_relations": matched_relations, + "missing_relations": missing_relations, + "forbidden_labels_found": forbidden, + } + + +def validate_benchmark_case(case_dir: str | Path) -> dict[str, Any]: + root = Path(case_dir).resolve() + case = _load(root / "case.json") + errors: list[str] = [] + warnings: list[str] = [] + suite = str(case.get("suite") or "") + if suite not in SUITES: + errors.append(f"case.json suite must be one of {sorted(SUITES)}") + if not str(case.get("case_id") or "").strip(): + errors.append("case.json requires case_id") + if not isinstance(case.get("thresholds"), dict): + errors.append("case.json requires thresholds object") + + required_files = ["case.json"] + if suite == "paper-to-image": + paper_relative = str(case.get("paper") or "paper.md") + required_files.append(str(case.get("expected_semantics") or "expected_semantics.json")) + if not (root / paper_relative).exists(): + source_path = root / str(case.get("source") or "source.json") + if source_path.exists(): + warnings.append(f"paper input not fetched yet: {paper_relative}") + else: + errors.append(f"missing required file: {paper_relative}; no source.json is available") + expected = _load(root / str(case.get("expected_semantics") or "expected_semantics.json")) + if not expected.get("entities"): + errors.append("expected_semantics.json requires entities") + if not isinstance(expected.get("relations"), list): + errors.append("expected_semantics.json requires relations list") + if not isinstance(expected.get("forbidden_labels", []), list): + errors.append("expected_semantics.json forbidden_labels must be a list") + elif suite == "image-to-ppt": + required_files.extend([str(case.get("reference_image") or "reference.png"), str(case.get("expected_objects") or "expected_objects.json")]) + expected = _load(root / str(case.get("expected_objects") or "expected_objects.json")) + if not isinstance(expected.get("objects"), list): + errors.append("expected_objects.json requires objects list") + if not isinstance(expected.get("relations", []), list): + errors.append("expected_objects.json relations must be a list") + + for relative in required_files: + if relative and not (root / relative).exists(): + errors.append(f"missing required file: {relative}") + references = case.get("positive_references", []) + case.get("negative_references", []) + for relative in references: + if not (root / str(relative)).exists(): + warnings.append(f"missing optional reference: {relative}") + + return { + "summary": "Benchmark case validation report.", + "ok": not errors, + "case_dir": str(root), + "case_id": case.get("case_id"), + "suite": suite, + "errors": errors, + "warnings": warnings, + } + + +def fetch_benchmark_case(case_dir: str | Path, force: bool = False) -> dict[str, Any]: + root = Path(case_dir).resolve() + case = _load(root / "case.json") + source_path = root / str(case.get("source") or "source.json") + source = _load(source_path) + if str(case.get("suite") or "") != "paper-to-image": + return {"summary": "Benchmark source fetch is only required for paper-to-image cases.", "ok": True, "case_dir": str(root), "status": "not_required"} + if not source: + return {"summary": "Benchmark source metadata is missing.", "ok": False, "case_dir": str(root), "status": "missing_source", "error": str(source_path)} + target = root / str(case.get("paper") or "inputs/paper.pdf") + if target.exists() and not force: + return { + "summary": "Benchmark paper source already exists.", + "ok": True, + "case_dir": str(root), + "status": "reused", + "paper": str(target), + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + } + urls = source.get("paper_urls") if isinstance(source.get("paper_urls"), list) else [] + errors = [] + for url in urls: + try: + response = requests.get(str(url), headers={"User-Agent": "ResearchFigureStudio benchmark fetch/0.1"}, timeout=120) + response.raise_for_status() + content = response.content + if not content.startswith(b"%PDF"): + raise ValueError("downloaded response is not a PDF") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + report = { + "summary": "Benchmark paper source fetched for local use.", + "ok": True, + "case_dir": str(root), + "status": "downloaded", + "paper": str(target), + "source_url": str(url), + "bytes": len(content), + "sha256": digest, + "license_note": source.get("license_note"), + } + write_json(target.parent / "fetch_report.json", report) + return report + except Exception as exc: + errors.append({"url": str(url), "error": str(exc)}) + return {"summary": "Unable to fetch benchmark paper source.", "ok": False, "case_dir": str(root), "status": "failed", "errors": errors} + + +def list_benchmark_cases(benchmarks_root: str | Path, suite: str | None = None) -> dict[str, Any]: + root = Path(benchmarks_root).resolve() + cases = [] + suites = [suite] if suite else sorted(SUITES) + for suite_name in suites: + case_root = root / suite_name / "cases" + if not case_root.exists(): + continue + for case_dir in sorted(path for path in case_root.iterdir() if path.is_dir()): + validation = validate_benchmark_case(case_dir) + cases.append({ + "case_id": validation.get("case_id") or case_dir.name, + "suite": suite_name, + "case_dir": str(case_dir), + "valid": validation["ok"], + "errors": validation["errors"], + }) + return {"summary": "Available ResearchFigureStudio benchmark cases.", "benchmarks_root": str(root), "cases": cases} + + +def _select_candidate_review(run: Path) -> dict: + report = _load(run / "candidate_review.json") + candidates = [item for item in report.get("candidates", []) if isinstance(item, dict)] + selected_id = report.get("selected_candidate_id") + selected = next((item for item in candidates if item.get("candidate_id") == selected_id), None) + if not selected and candidates: + selected = max(candidates, key=lambda item: float(item.get("score") or 0.0)) + return selected.get("review", {}) if isinstance(selected, dict) else {} + + +def _unexpected_visible_labels(review: dict, specification: dict) -> list[str]: + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + if "unsupported_unexpected_labels" in ocr: + return [str(value) for value in ocr.get("unsupported_unexpected_labels", []) if str(value).strip()] + if ocr.get("unexpected_labels"): + return [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] + allowed = { + _normalized_text(value) + for value in specification.get("required_labels", []) if _normalized_text(value) + } + allowed.update( + _normalized_text(value) + for value in specification.get("repeatable_labels", []) if _normalized_text(value) + ) + allowed.update( + _normalized_text(item.get("label")) + for item in specification.get("relations", []) if isinstance(item, dict) and _normalized_text(item.get("label")) + ) + detected = ocr.get("detected_labels", []) + return [str(value) for value in detected if _normalized_text(value) and _normalized_text(value) not in allowed] + + +def _audit_candidate_stability(candidate_report: dict, specification: dict) -> dict[str, Any]: + candidates = [ + item for item in candidate_report.get("candidates", []) + if isinstance(item, dict) and str(item.get("candidate_id") or "").startswith("candidate_") and not item.get("repair_source") + ] + requested = int(candidate_report.get("requested_candidates") or len(candidates)) + if requested < 2: + return {"summary": "Repeated-generation stability audit has insufficient samples.", "seed_count": requested, "available": False} + records = [] + pass_count = 0 + for item in candidates: + review = item.get("review", {}) if isinstance(item.get("review"), dict) else {} + unexpected = _unexpected_visible_labels(review, specification) + effective_pass = bool(item.get("production_pass")) and not unexpected + pass_count += int(effective_pass) + records.append({ + "candidate_id": item.get("candidate_id"), + "reported_production_pass": bool(item.get("production_pass")), + "effective_production_pass": effective_pass, + "unexpected_labels": unexpected, + }) + pass_rate = round(pass_count / max(1, requested), 4) + return { + "summary": "Repeated-generation stability audited against the normalized visible-label whitelist.", + "available": True, + "seed_count": requested, + "production_pass_count": pass_count, + "production_pass_rate": pass_rate, + "stable": pass_rate >= 0.8, + "candidates": records, + } + + +def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: + case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() + case = _load(case_root / "case.json") + expected = _load(case_root / str(case.get("expected_semantics") or "expected_semantics.json")) + candidate_report = _load(run / "candidate_review.json") + review = _select_candidate_review(run) + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict)] + selected_candidate_id = candidate_report.get("selected_candidate_id") + selected_candidate_record = next((item for item in candidate_records if item.get("candidate_id") == selected_candidate_id), None) + if selected_candidate_record is None and candidate_records: + selected_candidate_record = max(candidate_records, key=lambda item: float(item.get("score") or 0.0)) + deterministic_overlay = bool(candidate_report.get("deterministic_overlay")) + overlay_report = selected_candidate_record.get("overlay_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("overlay_report"), dict) else {} + geometry_report = selected_candidate_record.get("geometry_alignment_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("geometry_alignment_report"), dict) else {} + layout_blueprint = _load(run / "layout_blueprint.json") + semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} + route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} + benchmark_review = _load(run / "benchmark_review.json") + if benchmark_review: + review = {**review, **benchmark_review} + + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + paper_alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + clarity = review.get("clarity", {}) if isinstance(review.get("clarity"), dict) else {} + information = review.get("information", {}) if isinstance(review.get("information"), dict) else {} + stability = review.get("stability", {}) if isinstance(review.get("stability"), dict) else {} + if not stability: + stability = _load(run / "stability_report.json") or (candidate_report.get("stability", {}) if isinstance(candidate_report.get("stability"), dict) else {}) + if isinstance(benchmark_review.get("stability"), dict): + stability = benchmark_review["stability"] + specification = _load(run / "figure_specification.json") + review_ground_truth = _load(run / "review_ground_truth.json") + repair_history = _load(run / "repair_history.json") + planning_contract = _score_planning_contract(expected, specification) + stability_audit = _audit_candidate_stability(candidate_report, specification) + + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] + missing_entities = scientific.get("missing_modules", []) or benchmark_review.get("missing_entities", []) + missing_relations = scientific.get("missing_relations", []) or benchmark_review.get("missing_relations", []) + invented = scientific.get("invented_items", []) or benchmark_review.get("invented_items", []) + forbidden_found = ocr.get("forbidden_labels_found", []) or benchmark_review.get("forbidden_labels_found", []) + missing_labels = ocr.get("missing_labels", []) or benchmark_review.get("missing_labels", []) + misspelled = ocr.get("misspelled_labels", []) or benchmark_review.get("misspelled_labels", []) + + entity_recall = _clamp((len(expected_entities) - len(missing_entities)) / max(1, len(expected_entities))) + relation_recall = _clamp((len(expected_relations) - len(missing_relations)) / max(1, len(expected_relations))) + exact_label_rate = _clamp((len(expected_entities) - len(missing_labels) - len(misspelled)) / max(1, len(expected_entities))) + scientific_score = _clamp(scientific.get("score", _mean([entity_recall, relation_recall]))) + aesthetic_dimensions = [aesthetic.get(key) for key in ("hierarchy", "balance", "whitespace", "color", "icon_consistency", "readability") if aesthetic.get(key) is not None] + aesthetic_score = _clamp(aesthetic.get("score", _mean([_clamp(value) for value in aesthetic_dimensions]))) + clarity_score = _clamp(clarity.get("score", aesthetic.get("readability", 0.0))) + information_score = _clamp(information.get("score", information.get("coverage", scientific_score))) + stability_score = _clamp( + stability_audit.get("production_pass_rate") + if stability_audit.get("available") + else stability.get("production_pass_rate", stability.get("score", 0.0)) + ) + paper_alignment_score = _clamp(paper_alignment.get("score")) if paper_alignment else None + topology_score = _clamp(topology.get("score")) if topology else None + required_priority_ids = { + str(item.get("id") or "").strip() + for item in review_ground_truth.get("priorities", []) + if isinstance(item, dict) and item.get("required") and str(item.get("id") or "").strip() + } + supported_priority_ids = { + str(item.get("id") or "").strip() + for item in paper_alignment.get("priorities", []) + if isinstance(item, dict) + and str(item.get("status") or "").strip().casefold() in {"supported", "present", "correct"} + and str(item.get("id") or "").strip() + } + evidence_priority_coverage = ( + _clamp(len(required_priority_ids & supported_priority_ids) / max(1, len(required_priority_ids))) + if required_priority_ids else None + ) + evidence_conflicts = list(paper_alignment.get("evidence_conflicts", [])) if paper_alignment else [] + unsupported_visual_claims = list(paper_alignment.get("unsupported_visual_claims", [])) if paper_alignment else [] + missing_priorities = list(paper_alignment.get("missing_priorities", [])) if paper_alignment else [] + repair_rounds = [item for item in repair_history.get("rounds", []) if isinstance(item, dict)] + accepted_repairs = [item for item in repair_rounds if item.get("accepted")] + rejected_repairs = [item for item in repair_rounds if not item.get("accepted")] + regression_rejections = [ + item for item in rejected_repairs + if any("decreased" in str(reason).casefold() or "pass to fail" in str(reason).casefold() for reason in item.get("rejection_reasons", [])) + ] + unexpected_labels = _unexpected_visible_labels(review, specification) + hard_failures = list(review.get("hard_errors", [])) + if not scientific or not aesthetic or scientific.get("engineering_only") or aesthetic.get("engineering_only"): + hard_failures.append("frozen benchmark judge did not provide scientific and aesthetic sections") + if invented: + hard_failures.append("invented scientific content") + if forbidden_found: + hard_failures.append("forbidden labels found") + if relation_recall < 1.0: + hard_failures.append("required relations missing or incorrect") + if exact_label_rate < 1.0: + hard_failures.append("required labels missing or misspelled") + if unexpected_labels: + hard_failures.append("unexpected visible labels outside the normalized figure contract") + evidence_review_required = bool(review_ground_truth.get("strict_evidence_required")) + if evidence_review_required and not paper_alignment: + hard_failures.append("paper-evidence alignment review is missing") + if paper_alignment and (not paper_alignment.get("passed") or missing_priorities): + hard_failures.append("required paper priorities are missing or unverified") + if evidence_conflicts: + hard_failures.append("candidate conflicts with quoted paper evidence") + if unsupported_visual_claims: + hard_failures.append("candidate contains unsupported visual claims") + if topology and not topology.get("passed", False): + hard_failures.append("visible connector topology failed review") + if stability_audit.get("available") and not stability_audit.get("stable"): + hard_failures.append("repeated generation stability below production threshold") + plan_entity_min = float(case.get("thresholds", {}).get("plan_entity_recall", 1.0)) + plan_relation_min = float(case.get("thresholds", {}).get("plan_relation_recall", 1.0)) + if planning_contract.get("available") and float(planning_contract.get("entity_recall", 0.0)) < plan_entity_min: + hard_failures.append("paper planning missed required benchmark entities") + if planning_contract.get("available") and float(planning_contract.get("relation_recall", 0.0)) < plan_relation_min: + hard_failures.append("paper planning missed required benchmark relations") + if planning_contract.get("forbidden_labels_found"): + hard_failures.append("paper planning introduced forbidden benchmark labels") + if deterministic_overlay and not overlay_report: + hard_failures.append("deterministic overlay report is missing for the selected candidate") + if deterministic_overlay and not geometry_report: + hard_failures.append("visual substrate geometry report is missing for the selected candidate") + if deterministic_overlay and geometry_report and not ( + geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ): + hard_failures.append("visual substrate geometry normalization gate failed") + if deterministic_overlay and overlay_report and not overlay_report.get("passed", False): + hard_failures.append("deterministic overlay did not compile every required label and connector") + if deterministic_overlay and route_quality and not route_quality.get("passed", False): + hard_failures.append("deterministic connector routing quality gate failed") + + metrics = { + "entity_recall": entity_recall, + "relation_recall": relation_recall, + "exact_label_rate": exact_label_rate, + "scientific_score": scientific_score, + "information_score": information_score, + "clarity_score": clarity_score, + "aesthetic_score": aesthetic_score, + "stability_score": stability_score, + "paper_alignment_score": paper_alignment_score, + "evidence_priority_coverage": evidence_priority_coverage, + "evidence_conflict_count": len(evidence_conflicts), + "unsupported_visual_claim_count": len(unsupported_visual_claims), + "missing_paper_priority_count": len(missing_priorities), + "topology_score": topology_score, + "repair_attempt_count": len(repair_rounds), + "accepted_repair_count": len(accepted_repairs), + "rejected_repair_count": len(rejected_repairs), + "repair_regression_rejection_count": len(regression_rejections), + "hallucination_count": len(invented), + "forbidden_content_count": len(forbidden_found), + "unexpected_label_count": len(unexpected_labels), + "substrate_geometry_pass": bool(geometry_report.get("passed")) if deterministic_overlay else None, + "plan_entity_recall": planning_contract.get("entity_recall"), + "plan_relation_recall": planning_contract.get("relation_recall"), + "plan_forbidden_content_count": len(planning_contract.get("forbidden_labels_found", [])), + "deterministic_overlay": deterministic_overlay, + "overlay_label_count": overlay_report.get("label_count") if overlay_report else None, + "overlay_connector_count": overlay_report.get("connector_count") if overlay_report else None, + "route_crossing_count": route_quality.get("crossing_count") if route_quality else None, + "route_shared_segment_overlap": route_quality.get("shared_segment_overlap") if route_quality else None, + "route_node_obstacle_intersections": route_quality.get("node_obstacle_intersections") if route_quality else None, + } + thresholds = case.get("thresholds", {}) + threshold_failures = _threshold_failures(metrics, thresholds) + if paper_alignment and topology: + total_score = round(0.28 * scientific_score + 0.20 * float(paper_alignment_score or 0.0) + 0.12 * information_score + 0.12 * clarity_score + 0.10 * float(topology_score or 0.0) + 0.13 * aesthetic_score + 0.05 * stability_score, 4) + else: + total_score = round(0.40 * scientific_score + 0.15 * information_score + 0.15 * clarity_score + 0.20 * aesthetic_score + 0.10 * stability_score, 4) + return { + "summary": "Paper-to-image benchmark score.", + "suite": "paper-to-image", + "case_id": case.get("case_id"), + "case_dir": str(case_root), + "run_dir": str(run), + "metrics": metrics, + "planning_contract": planning_contract, + "deterministic_overlay": { + "enabled": deterministic_overlay, + "selected_candidate_id": selected_candidate_record.get("candidate_id") if isinstance(selected_candidate_record, dict) else None, + "overlay_report": overlay_report, + "route_quality": route_quality, + }, + "paper_alignment": paper_alignment, + "repair_history": repair_history, + "stability_audit": stability_audit, + "total_score": total_score, + "hard_failures": sorted(set(hard_failures)), + "threshold_failures": threshold_failures, + "passed": not hard_failures and not threshold_failures, + "policy": "paper-evidence alignment, scientific content, topology, terminology, and hallucination failures cannot be offset by aesthetics", + } + + +def audit_paper_image_run( + run_dir: str | Path, + case_dir: str | Path | None = None, + candidate_image: str | Path | None = None, + review_model: str | None = None, + ocr_engine: str = "off", + ocr_lang: str = "en_ch", + critic_adapter=None, + topology_adapter=None, +) -> dict[str, Any]: + """Re-scan an existing generated image against its paper evidence contract. + + This deliberately does not regenerate the image. It upgrades legacy runs to + the current frozen review gates and leaves an actionable repair plan when + evidence, topology, OCR, or aesthetic checks fail. + """ + + run = Path(run_dir).resolve() + if not run.exists(): + raise FileNotFoundError(f"Paper-to-image run does not exist: {run}") + from ..paper_to_image.critics import review_candidate + from ..paper_to_image.review_contract import build_repair_plan, compile_review_ground_truth + + plan = {} + for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): + value = _load(run / f"{name}.json") + if value: + plan[name] = value + if not isinstance(plan.get("figure_specification"), dict): + raise ValueError(f"figure_specification.json is missing or invalid in {run}") + + document_model = _load(run / "document_model.json") + key_evidence = _load(run / "key_evidence.json") + evidence_records = document_model.get("evidence", []) if isinstance(document_model.get("evidence"), list) else key_evidence.get("evidence", []) + ground_truth = compile_review_ground_truth(plan, evidence_records) + plan["review_ground_truth"] = ground_truth + write_json(run / "review_ground_truth.json", ground_truth) + + candidate_report = _load(run / "candidate_review.json") + selected_id = str(candidate_report.get("selected_candidate_id") or "").strip() + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict) and str(item.get("path") or "").strip()] + selected_record = next((item for item in candidate_records if str(item.get("candidate_id")) == selected_id), None) + candidate = Path(candidate_image).resolve() if candidate_image else run / "selected_image.png" + if candidate_image: + selected_id = "explicit_reaudit_candidate" + elif not candidate.exists(): + if selected_record is None and candidate_records: + selected_record = max(candidate_records, key=lambda item: (bool(item.get("production_pass")), float(item.get("score") or 0.0))) + if selected_record is None: + raise FileNotFoundError(f"No reviewable candidate image is recorded in {run}") + selected_id = str(selected_record.get("candidate_id") or "reaudit_candidate") + candidate = Path(str(selected_record.get("path"))) + if not candidate.is_absolute(): + candidate = run / candidate + blueprint = run / "layout_blueprint.png" + if not candidate.exists(): + raise FileNotFoundError(f"Selected candidate image is missing in {run}") + if not blueprint.exists(): + raise FileNotFoundError(f"layout_blueprint.png is missing in {run}") + template = _load(run / "selected_template.json") + preferences = _load(run / "preferences.json") + acceptable_ratios = [ + str(value) for value in (preferences.get("requested_aspect_ratio"), preferences.get("aspect_ratio")) + if value and str(value) != "auto" + ] + generation_parameters = _load(run / "generation_parameters.json") + layout_blueprint = _load(run / "layout_blueprint.json") + semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} + route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} + overlay_report = selected_record.get("overlay_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("overlay_report"), dict) else {} + geometry_report = selected_record.get("geometry_alignment_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("geometry_alignment_report"), dict) else {} + review = review_candidate( + candidate, + blueprint, + plan, + template, + mode="vlm", + model=review_model, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + critic_adapter=critic_adapter, + topology_adapter=topology_adapter, + acceptable_aspect_ratios=acceptable_ratios, + require_visual_enrichment=str(generation_parameters.get("asset_mode") or "image2") == "image2", + ) + repair_plan = build_repair_plan(review, selected_id, 0) + write_json(run / "benchmark_review.json", review) + write_json(run / "paper_alignment_report.json", { + "summary": "Independent paper-evidence re-audit for the selected complete image.", + "candidates": [{"candidate_id": selected_id, **review.get("paper_alignment", {})}], + }) + write_json(run / "reaudit_repair_plan.json", repair_plan) + benchmark = score_benchmark_case(case_dir, run, run) if case_dir else None + result = { + "summary": "Existing complete image re-audited against quoted paper evidence and current visual gates.", + "ok": True, + "run_dir": str(run), + "candidate_id": selected_id, + "candidate": str(candidate), + "review_ground_truth": str(run / "review_ground_truth.json"), + "required_priority_count": ground_truth.get("required_priority_count"), + "grounded_required_priority_count": ground_truth.get("grounded_required_priority_count"), + "production_pass": bool( + review.get("production_pass") + and ( + not candidate_report.get("deterministic_overlay") + or geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ) + ), + "overall_score": review.get("overall_score"), + "paper_alignment_score": review.get("paper_alignment", {}).get("score"), + "topology_score": review.get("topology", {}).get("score"), + "deterministic_overlay": bool(candidate_report.get("deterministic_overlay")), + "overlay_report": overlay_report, + "geometry_alignment_report": geometry_report, + "route_quality": route_quality, + "repair_item_count": repair_plan.get("item_count"), + "benchmark": benchmark, + } + write_json(run / "paper_image_reaudit.json", result) + return result + + +def _image_similarity(reference: Path, preview: Path) -> dict[str, float | None]: + if not reference.exists() or not preview.exists(): + return {"pixel_similarity": None, "edge_similarity": None, "color_similarity": None} + with Image.open(reference) as ref_image, Image.open(preview) as out_image: + ref = ref_image.convert("RGB").resize((512, 512), Image.Resampling.LANCZOS) + out = out_image.convert("RGB").resize((512, 512), Image.Resampling.LANCZOS) + difference = ImageChops.difference(ref, out) + rms = math.sqrt(sum(value * value for value in ImageStat.Stat(difference).rms) / 3.0) + pixel_similarity = _clamp(1.0 - rms / 255.0) + ref_edge = ref.convert("L").filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > 32 else 0) + out_edge = out.convert("L").filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > 32 else 0) + edge_diff = ImageChops.difference(ref_edge, out_edge) + edge_similarity = _clamp(1.0 - ImageStat.Stat(edge_diff).mean[0] / 255.0) + ref_mean, out_mean = ImageStat.Stat(ref).mean, ImageStat.Stat(out).mean + color_error = sum(abs(a - b) for a, b in zip(ref_mean, out_mean)) / (3.0 * 255.0) + return {"pixel_similarity": pixel_similarity, "edge_similarity": edge_similarity, "color_similarity": _clamp(1.0 - color_error)} + + +def _ppt_editability(pptx_path: Path) -> dict[str, Any]: + if not pptx_path.exists(): + return {"status": "missing", "full_slide_image_count": 0, "text_shape_count": 0, "connector_count": 0, "shape_count": 0} + presentation = Presentation(str(pptx_path)) + full_slide_images = 0 + text_shapes = 0 + connectors = 0 + shapes = 0 + slide_width, slide_height = float(presentation.slide_width), float(presentation.slide_height) + for slide in presentation.slides: + for shape in slide.shapes: + shapes += 1 + if getattr(shape, "has_text_frame", False) and str(getattr(shape, "text", "")).strip(): + text_shapes += 1 + if shape.shape_type == MSO_SHAPE_TYPE.LINE: + connectors += 1 + if shape.shape_type == MSO_SHAPE_TYPE.PICTURE: + coverage = float(shape.width) * float(shape.height) / max(1.0, slide_width * slide_height) + if coverage >= 0.90: + full_slide_images += 1 + return { + "status": "pass" if full_slide_images == 0 else "blocked", + "full_slide_image_count": full_slide_images, + "text_shape_count": text_shapes, + "connector_count": connectors, + "shape_count": shapes, + } + + +def _bbox_iou(left: dict, right: dict) -> float: + lx0, ly0 = float(left.get("x", 0)), float(left.get("y", 0)) + lx1, ly1 = lx0 + float(left.get("w", 0)), ly0 + float(left.get("h", 0)) + rx0, ry0 = float(right.get("x", 0)), float(right.get("y", 0)) + rx1, ry1 = rx0 + float(right.get("w", 0)), ry0 + float(right.get("h", 0)) + intersection = max(0.0, min(lx1, rx1) - max(lx0, rx0)) * max(0.0, min(ly1, ry1) - max(ly0, ry0)) + union = max(0.000001, float(left.get("w", 0)) * float(left.get("h", 0)) + float(right.get("w", 0)) * float(right.get("h", 0)) - intersection) + return intersection / union + + +def _center_error(left: dict, right: dict) -> float: + lx = float(left.get("x", 0)) + float(left.get("w", 0)) / 2 + ly = float(left.get("y", 0)) + float(left.get("h", 0)) / 2 + rx = float(right.get("x", 0)) + float(right.get("w", 0)) / 2 + ry = float(right.get("y", 0)) + float(right.get("h", 0)) / 2 + return math.sqrt((lx - rx) ** 2 + (ly - ry) ** 2) + + +def _match_expected_objects(expected_objects: list[dict], geometry: dict) -> tuple[list[dict], dict[str, str]]: + actual = [] + for collection, object_type in (("panels", "panel"), ("cards", "card"), ("slots", "asset")): + actual.extend({**item, "benchmark_type": object_type} for item in geometry.get(collection, []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)) + unmatched = list(actual) + matches = [] + id_mapping: dict[str, str] = {} + for expected in expected_objects: + expected_box = expected.get("bbox_percent") if isinstance(expected.get("bbox_percent"), dict) else {} + expected_type = str(expected.get("type") or "") + same_id = next((item for item in unmatched if str(item.get("id")) == str(expected.get("id"))), None) + candidates = [item for item in unmatched if expected_type in {"", str(item.get("benchmark_type"))}] + if not candidates: + candidates = unmatched + chosen = same_id or max(candidates, key=lambda item: _bbox_iou(expected_box, item["bbox_percent"]), default=None) + if not chosen: + matches.append({"expected_id": expected.get("id"), "status": "unmatched", "iou": 0.0, "center_error": 1.0}) + continue + unmatched.remove(chosen) + iou = _bbox_iou(expected_box, chosen["bbox_percent"]) + center_error = _center_error(expected_box, chosen["bbox_percent"]) + id_mapping[str(expected.get("id"))] = str(chosen.get("id")) + matches.append({ + "expected_id": expected.get("id"), + "actual_id": chosen.get("id"), + "expected_type": expected_type, + "actual_type": chosen.get("benchmark_type"), + "iou": round(iou, 4), + "center_error": round(center_error, 4), + "status": "matched" if iou >= 0.30 else "low_overlap", + }) + return matches, id_mapping + + +def score_image_to_ppt(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: + case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() + case = _load(case_root / "case.json") + expected = _load(case_root / str(case.get("expected_objects") or "expected_objects.json")) + reference = case_root / str(case.get("reference_image") or "reference.png") + preview = run / str(case.get("preview") or "rebuild_preview.png") + pptx = run / str(case.get("pptx") or "editable_composition.pptx") + similarity = _image_similarity(reference, preview) + editability = _ppt_editability(pptx) + visual = _load(run / "rebuild_visual_quality_report.json") + semantic = _load(run / "semantic_binding_report.json") + composition = _load(run / "composition_quality_report.json") + alignment = _load(run / "text_alignment_report.json") + geometry = _load(run / "reference_geometry.json") + + expected_objects = [item for item in expected.get("objects", []) if isinstance(item, dict)] + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict)] + object_matches, id_mapping = _match_expected_objects(expected_objects, geometry) + object_coverage = _clamp(sum(1 for item in object_matches if item["status"] == "matched") / max(1, len(expected_objects))) if expected_objects else 1.0 + mean_object_iou = _mean([float(item["iou"]) for item in object_matches]) if object_matches else 1.0 + mean_center_error = _mean([float(item["center_error"]) for item in object_matches]) if object_matches else 0.0 + actual_relations = { + (str(item.get("source_id") or item.get("source") or ""), str(item.get("target_id") or item.get("target") or "")) + for item in composition.get("arrows", []) + if isinstance(item, dict) + } + matched_relations = 0 + for relation in expected_relations: + source = id_mapping.get(str(relation.get("source"))) + target = id_mapping.get(str(relation.get("target"))) + if source and target and (source, target) in actual_relations: + matched_relations += 1 + relation_coverage = _clamp(matched_relations / max(1, len(expected_relations))) if expected_relations else 1.0 + max_text_delta = max((float(item.get("center_delta_percent") or 0.0) for item in alignment.get("items", []) if isinstance(item, dict)), default=0.0) + text_alignment_score = _clamp(1.0 - max_text_delta / 0.10) + blocking_visual_issues = int(visual.get("blocking_issue_count") or 0) + semantic_binding_score = _clamp(semantic.get("mapped_entity_count", 0) / max(1, semantic.get("entity_count", 0))) if semantic else 1.0 + editable_score = 0.0 if editability["status"] == "blocked" else _clamp((min(1, editability["text_shape_count"]) + min(1, editability["connector_count"]) + 1) / 3) + fidelity_values = [value for value in similarity.values() if isinstance(value, float)] + render_fidelity = _mean(fidelity_values) if fidelity_values else 0.0 + metrics = { + **similarity, + "render_fidelity": render_fidelity, + "object_coverage": object_coverage, + "mean_object_iou": mean_object_iou, + "mean_center_error": mean_center_error, + "relation_coverage": relation_coverage, + "text_alignment_score": text_alignment_score, + "semantic_binding_score": semantic_binding_score, + "editability_score": editable_score, + "full_slide_image_count": editability["full_slide_image_count"], + "blocking_visual_issue_count": blocking_visual_issues, + } + hard_failures = [] + if editability["full_slide_image_count"]: + hard_failures.append("full-slide reference image detected") + if blocking_visual_issues: + hard_failures.append("blocking visual issues remain") + thresholds = case.get("thresholds", {}) + threshold_failures = _threshold_failures(metrics, thresholds) + total_score = round(0.40 * render_fidelity + 0.20 * object_coverage + 0.15 * relation_coverage + 0.10 * text_alignment_score + 0.15 * editable_score, 4) + return { + "summary": "Image-to-editable-PPT benchmark score.", + "suite": "image-to-ppt", + "case_id": case.get("case_id"), + "case_dir": str(case_root), + "run_dir": str(run), + "metrics": metrics, + "editability": editability, + "object_matches": object_matches, + "total_score": total_score, + "hard_failures": hard_failures, + "threshold_failures": threshold_failures, + "passed": not hard_failures and not threshold_failures, + "limitations": [ + "pixel, edge, and color similarity are baseline metrics; calibrated SSIM/LPIPS and region annotations remain future work", + "mutation-based editability testing requires a dedicated render-after-edit harness", + ], + } + + +def score_benchmark_case(case_dir: str | Path, run_dir: str | Path, out: str | Path | None = None) -> dict[str, Any]: + validation = validate_benchmark_case(case_dir) + if not validation["ok"]: + return {**validation, "passed": False} + result = score_paper_to_image(case_dir, run_dir) if validation["suite"] == "paper-to-image" else score_image_to_ppt(case_dir, run_dir) + target = ensure_dir(out or run_dir) + write_json(target / "benchmark_result.json", result) + lines = [ + "# Benchmark Result", + "", + f"- Suite: {result.get('suite')}", + f"- Case: {result.get('case_id')}", + f"- Passed: {result.get('passed')}", + f"- Total score: {result.get('total_score')}", + "", + "## Metrics", + ] + lines.extend(f"- {key}: {value}" for key, value in result.get("metrics", {}).items()) + lines.extend(["", "## Hard failures"]) + lines.extend(f"- {item}" for item in result.get("hard_failures", [])) + lines.extend(["", "## Threshold failures"]) + lines.extend(f"- {item}" for item in result.get("threshold_failures", [])) + write_text(target / "benchmark_result.md", "\n".join(lines) + "\n") + return result + + +def run_benchmark_case(case_dir: str | Path, out: str | Path) -> dict[str, Any]: + case_root = Path(case_dir).resolve() + case_preflight = _load(case_root / "case.json") + if str(case_preflight.get("suite") or "") == "paper-to-image": + paper_path = case_root / str(case_preflight.get("paper") or "paper.md") + if not paper_path.exists(): + fetched = fetch_benchmark_case(case_root) + if not fetched.get("ok"): + return {**fetched, "passed": False} + validation = validate_benchmark_case(case_dir) + if not validation["ok"]: + return {**validation, "passed": False} + root = Path(case_dir).resolve() + target = ensure_dir(out).resolve() + case = _load(root / "case.json") + config = case.get("run_config", {}) if isinstance(case.get("run_config"), dict) else {} + if validation["suite"] == "paper-to-image": + from ..paper_to_image import run_paper_to_image + + workflow_result = run_paper_to_image( + paper=root / str(case.get("paper") or "paper.md"), + out=target, + preferences_path=(root / str(case["preferences"])) if case.get("preferences") else None, + positive_references=[str(root / str(path)) for path in case.get("positive_references", [])], + negative_references=[str(root / str(path)) for path in case.get("negative_references", [])], + planner_mode=str(config.get("planner_mode") or "heuristic"), + asset_mode=str(config.get("image_asset_mode") or "placeholder"), + candidates=int(config.get("image_candidates") or 1), + aspect_ratio=str(config.get("aspect_ratio") or "auto"), + language=str(config.get("language") or "English"), + review_mode=str(config.get("review_mode") or "heuristic"), + domain_profile=str(config.get("domain_profile") or "auto"), + template=str(config.get("template") or "auto"), + repair_rounds=int(config.get("repair_rounds") or 0), + ocr_engine=str(config.get("ocr_engine") or "off"), + ocr_lang=str(config.get("ocr_lang") or "en_ch"), + ) + else: + from ..editable_rebuild import rebuild_editable + + workflow_result = rebuild_editable( + reference=root / str(case.get("reference_image") or "reference.png"), + out=target, + asset_mode=str(config.get("asset_mode") or "placeholder"), + asset_policy=str(config.get("asset_policy") or "smart-api"), + text_mode=str(config.get("text_mode") or "off"), + layout_mode=str(config.get("layout_mode") or "heuristic"), + control_mode=str(config.get("control_mode") or "heuristic"), + design_plan_mode=str(config.get("design_plan_mode") or "heuristic"), + export_preview=True, + ocr_engine=str(config.get("ocr_engine") or "off"), + ) + score = score_benchmark_case(root, target, target) + result = { + "summary": "Benchmark workflow and scoring completed.", + "ok": bool(workflow_result.get("ok")), + "suite": validation["suite"], + "case_id": validation["case_id"], + "out_dir": str(target), + "workflow_result": workflow_result, + "benchmark_result": score, + } + write_json(target / "benchmark_run.json", result) + return result + + +def run_fast_benchmark_case( + case_dir: str | Path, + out: str | Path, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "off", + rasterize_pdf_dpi: int | None = None, +) -> dict[str, Any]: + case_root = Path(case_dir).resolve() + case = _load(case_root / "case.json") + if str(case.get("suite") or "") != "paper-to-image": + return {"summary": "Fast framework benchmark only supports paper-to-image cases.", "ok": False, "case_dir": str(case_root)} + paper_path = case_root / str(case.get("paper") or "paper.md") + if not paper_path.exists(): + fetched = fetch_benchmark_case(case_root) + if not fetched.get("ok"): + return {**fetched, "passed": False} + validation = validate_benchmark_case(case_root) + if not validation.get("ok"): + return {**validation, "passed": False} + from ..paper_to_image import run_fast_framework_prompt + from .scanned_pdf import rasterize_pdf_as_scan + + preferences = case_root / str(case.get("preferences")) if case.get("preferences") else None + active_paper = paper_path + rasterized_input = None + if rasterize_pdf_dpi is not None and paper_path.suffix.casefold() == ".pdf": + rasterized_input = rasterize_pdf_as_scan(paper_path, Path(out) / "benchmark_inputs" / "paper_scanned.pdf", dpi=rasterize_pdf_dpi) + active_paper = rasterized_input + run = run_fast_framework_prompt( + paper=active_paper, + out=out, + deadline_seconds=deadline_seconds, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine, + preferences_path=preferences if preferences and preferences.exists() else None, + aspect_ratio=str(case.get("run_config", {}).get("aspect_ratio") or "16:9"), + ) + score = score_benchmark_case(case_root, out) + result = { + "summary": "Fast paper framework benchmark completed.", + "ok": bool(run.get("ok")), + "case_id": case.get("case_id"), + "rasterized_input": str(rasterized_input) if rasterized_input else None, + "rasterize_pdf_dpi": int(rasterize_pdf_dpi) if rasterized_input else None, + "run": run, + "benchmark": score, + "passed_planning_thresholds": not any("plan_" in value for value in score.get("threshold_failures", [])), + } + write_json(Path(out) / "fast_benchmark_result.json", result) + return result + + +def run_fast_benchmark_suite( + benchmarks_root: str | Path, + out: str | Path, + case_ids: list[str] | None = None, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "off", + rasterize_pdf_dpi: int | None = None, +) -> dict[str, Any]: + root = Path(benchmarks_root).resolve() + target = ensure_dir(out).resolve() + selected = set(case_ids or []) + listing = list_benchmark_cases(root, suite="paper-to-image") + cases = [item for item in listing.get("cases", []) if item.get("valid") and (not selected or str(item.get("case_id")) in selected)] + results = [] + for item in cases: + case_id = str(item.get("case_id")) + case_out = target / case_id + try: + result = run_fast_benchmark_case( + item["case_dir"], + case_out, + deadline_seconds=deadline_seconds, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine, + rasterize_pdf_dpi=rasterize_pdf_dpi, + ) + except Exception as exc: + result = {"summary": "Fast benchmark case raised an exception.", "ok": False, "case_id": case_id, "error": str(exc)} + run = result.get("run", {}) if isinstance(result.get("run"), dict) else {} + benchmark = result.get("benchmark", {}) if isinstance(result.get("benchmark"), dict) else {} + metrics = benchmark.get("metrics", {}) if isinstance(benchmark.get("metrics"), dict) else {} + provider = run.get("provider", {}) if isinstance(run.get("provider"), dict) else {} + extraction_quality = run.get("extraction_quality", {}) if isinstance(run.get("extraction_quality"), dict) else {} + results.append({ + "case_id": case_id, + "ok": bool(result.get("ok")), + "production_ready": bool(run.get("production_ready")), + "passed_planning_thresholds": bool(result.get("passed_planning_thresholds")), + "contract_source": run.get("contract_source"), + "cache_hit": bool(run.get("cache_hit")), + "document_cache_hit": bool(run.get("document_cache_hit")), + "rasterized_input": result.get("rasterized_input"), + "elapsed_seconds": run.get("elapsed_seconds"), + "stage_timings": run.get("stage_timings", {}), + "extraction_quality": extraction_quality, + "provider": provider, + "plan_entity_recall": metrics.get("plan_entity_recall"), + "plan_relation_recall": metrics.get("plan_relation_recall"), + "plan_forbidden_content_count": metrics.get("plan_forbidden_content_count"), + "threshold_failures": benchmark.get("threshold_failures", []), + "error": result.get("error"), + }) + + def numbers(field: str) -> list[float]: + values = [] + for item in results: + value = item.get(field) + if isinstance(value, (int, float)): + values.append(float(value)) + return values + + def timing(name: str) -> list[float]: + return [float(item.get("stage_timings", {}).get(name)) for item in results if isinstance(item.get("stage_timings", {}).get(name), (int, float))] + + def extraction_metric(name: str) -> list[float]: + return [float(item.get("extraction_quality", {}).get(name)) for item in results if isinstance(item.get("extraction_quality", {}).get(name), (int, float))] + + def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * fraction) - 1)) + return round(ordered[index], 4) + + provider_attempts = sum(int(item.get("provider", {}).get("attempts") or 0) for item in results) + provider_retries = sum(int(item.get("provider", {}).get("retries_used") or 0) for item in results) + provider_calls = [item for item in results if item.get("provider", {}).get("attempts")] + failure_categories: dict[str, int] = {} + for item in provider_calls: + for category in item.get("provider", {}).get("failure_categories", []) or []: + failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1 + total_times = numbers("elapsed_seconds") + entity_scores = numbers("plan_entity_recall") + relation_scores = numbers("plan_relation_recall") + aggregate = { + "case_count": len(results), + "successful_run_count": sum(bool(item.get("ok")) for item in results), + "production_ready_count": sum(bool(item.get("production_ready")) for item in results), + "planning_threshold_pass_count": sum(bool(item.get("passed_planning_thresholds")) for item in results), + "cache_hit_count": sum(bool(item.get("cache_hit")) for item in results), + "cache_hit_rate": round(sum(bool(item.get("cache_hit")) for item in results) / max(1, len(results)), 4), + "document_cache_hit_count": sum(bool(item.get("document_cache_hit")) for item in results), + "document_cache_hit_rate": round(sum(bool(item.get("document_cache_hit")) for item in results) / max(1, len(results)), 4), + "rasterized_case_count": sum(bool(item.get("rasterized_input")) for item in results), + "mean_plan_entity_recall": _mean(entity_scores), + "mean_plan_relation_recall": _mean(relation_scores), + "forbidden_content_total": sum(int(item.get("plan_forbidden_content_count") or 0) for item in results), + "mean_total_seconds": _mean(total_times), + "p95_total_seconds": percentile(total_times, 0.95), + "mean_document_preparation_seconds": _mean(timing("document_preparation_seconds")), + "mean_document_extraction_seconds": _mean(timing("document_extraction_seconds")), + "mean_semantic_compilation_seconds": _mean(timing("semantic_compilation_seconds")), + "mean_readable_page_ratio": _mean(extraction_metric("readable_page_ratio")), + "mean_evidence_page_coverage_ratio": _mean(extraction_metric("evidence_page_coverage_ratio")), + "mean_evidence_char_count": _mean(extraction_metric("evidence_char_count")), + "max_detected_column_count": max((int(value) for value in extraction_metric("max_column_count")), default=0), + "multi_column_page_total": sum(int(value) for value in extraction_metric("multi_column_page_count")), + "mean_section_count": _mean(extraction_metric("section_count")), + "typographic_heading_total": sum(int(value) for value in extraction_metric("typographic_heading_count")), + "merged_heading_line_total": sum(int(value) for value in extraction_metric("merged_heading_line_count")), + "figure_caption_total": sum(int(value) for value in extraction_metric("figure_caption_count")), + "table_caption_total": sum(int(value) for value in extraction_metric("table_caption_count")), + "formula_total": sum(int(value) for value in extraction_metric("formula_count")), + "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), + "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), + "ocr_attempted_page_total": sum(int(value) for value in extraction_metric("ocr_attempted_count")), + "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), + "ocr_incomplete_run_count": sum(not bool(item.get("extraction_quality", {}).get("ocr_run_complete", True)) for item in results), + "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), + "repeated_margin_noise_removed_total": sum(int(value) for value in extraction_metric("repeated_margin_noise_removed_count")), + "native_hyphenation_repair_total": sum(int(value) for value in extraction_metric("native_hyphenation_repair_count")), + "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), + "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), + "mean_ocr_latin_known_word_ratio": _mean(extraction_metric("ocr_latin_known_word_ratio")), + "provider_call_count": len(provider_calls), + "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), + "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, + "provider_attempts": provider_attempts, + "provider_retries_used": provider_retries, + "provider_failure_categories": failure_categories, + } + report = { + "summary": "Fast paper framework benchmark suite completed.", + "ok": bool(results) and all(item.get("ok") for item in results), + "benchmarks_root": str(root), + "out_dir": str(target), + "planner_mode": planner_mode, + "planner_model": planner_model, + "deadline_seconds": deadline_seconds, + "ocr_engine": ocr_engine, + "rasterize_pdf_dpi": rasterize_pdf_dpi, + "selected_case_ids": [item.get("case_id") for item in cases], + "aggregate": aggregate, + "cases": results, + } + write_json(target / "fast_suite_report.json", report) + return report diff --git a/rfs/evaluation/outputs.py b/rfs/evaluation/outputs.py new file mode 100644 index 0000000..29fd1e2 --- /dev/null +++ b/rfs/evaluation/outputs.py @@ -0,0 +1,5 @@ +"""Stable output-validation exports.""" + +from .validator import validate_output + +__all__ = ["validate_output"] diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py new file mode 100644 index 0000000..667c0e4 --- /dev/null +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +import io +import math +import time +from pathlib import Path +from typing import Any, Callable + +from ..paper_to_image.analyzer import parse_paper +from ..utils import ensure_dir, write_json +from .scanned_pdf import rasterize_pdf_as_scan + + +def _save_document(document: Any, target: Path) -> Path: + target.parent.mkdir(parents=True, exist_ok=True) + document.save(str(target)) + document.close() + return target + + +def _native_two_column_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 48), "Abstract", fontname="hebo", fontsize=13) + page.insert_textbox((45, 68, 555, 125), "A deterministic two-column extraction benchmark with enough scientific text for parser quality gates.", fontsize=10) + page.insert_text((45, 155), "2 Method", fontname="hebo", fontsize=13) + page.insert_textbox((45, 185, 275, 260), "LEFT_TOP encoder evidence begins the method flow and establishes the input representation.", fontsize=10) + page.insert_textbox((45, 300, 275, 375), "LEFT_BOTTOM decoder evidence follows the encoder in the left reading column.", fontsize=10) + page.insert_textbox((325, 185, 555, 260), "RIGHT_TOP retrieval evidence belongs after the complete left column.", fontsize=10) + page.insert_textbox((325, 300, 555, 375), "RIGHT_BOTTOM output evidence finishes the two-column reading order.", fontsize=10) + page.insert_textbox((45, 430, 555, 485), "Figure 1: Overview of the encoder, decoder, retrieval module, and final output.", fontsize=10) + page.insert_text((45, 540), "3 Conclusion", fontname="hebo", fontsize=13) + page.insert_textbox((45, 560, 555, 620), "The benchmark concludes with a complete evidence-grounded pipeline.", fontsize=10) + return _save_document(document, target) + + +def _bold_heading_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 70, 555, 130), "A paper with unnumbered bold sections validates typography-aware section recovery.", fontsize=10) + page.insert_text((45, 175), "Model Architecture", fontname="hebo", fontsize=11) + page.insert_textbox((45, 195, 555, 260), "The architecture contains an image encoder, a prompt encoder, and a mask decoder.", fontsize=10) + page.insert_text((45, 305), "Encoder and Decoder Stacks", fontname="hebo", fontsize=10) + page.insert_textbox((45, 325, 555, 390), "The encoder representation conditions the decoder before the final prediction.", fontsize=10) + page.insert_text((45, 435), "Conclusion", fontname="hebo", fontsize=11) + page.insert_textbox((45, 455, 555, 520), "Typography-based boundaries preserve the scientific section context.", fontsize=10) + return _save_document(document, target) + + +def _rotated_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 75, 555, 145), "A rotated scientific page must retain displayed coordinates and readable text.", fontsize=10) + page.insert_text((45, 190), "2 Method", fontname="hebo", fontsize=12) + page.insert_textbox((45, 215, 555, 290), "The rotated encoder sends its representation to a decoder and final output.", fontsize=10) + page.insert_textbox((45, 335, 555, 390), "Figure 1: Rotated architecture overview with encoder and decoder stages.", fontsize=10) + page.set_rotation(90) + return _save_document(document, target) + + +def _mixed_scan_fixture(target: Path) -> Path: + import fitz + + scan_source = fitz.open() + scan_page = scan_source.new_page(width=600, height=800) + scan_page.insert_text((50, 65), "2 Method", fontname="hebo", fontsize=18) + scan_page.insert_textbox((50, 105, 550, 190), "The scanned encoder passes evidence to the scanned decoder and output module.", fontsize=15) + scan_page.insert_textbox((50, 245, 550, 330), "Figure 1: Scanned architecture overview with encoder decoder and output.", fontsize=15) + pixmap = scan_page.get_pixmap(dpi=180, alpha=False) + scan_png = pixmap.tobytes("png") + scan_source.close() + + document = fitz.open() + page1 = document.new_page(width=600, height=800) + page1.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page1.insert_textbox((45, 75, 555, 155), "A mixed PDF combines native text with one scanned method page for local OCR fallback.", fontsize=10) + page2 = document.new_page(width=600, height=800) + page2.insert_image(page2.rect, stream=scan_png) + page3 = document.new_page(width=600, height=800) + page3.insert_text((45, 50), "3 Conclusion", fontname="hebo", fontsize=12) + page3.insert_textbox((45, 75, 555, 155), "The mixed extraction path preserves native pages and recovers the scanned method evidence.", fontsize=10) + return _save_document(document, target) + + +def _repeated_margin_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + sections = [ + ("Abstract", "A document encoder grounds paper evidence before framework generation."), + ("2 Method", "The document encoder sends grounded entities to a relation decoder and final output."), + ("3 Experiments", "Experiments measure entity recall, relation recall, and extraction latency."), + ("4 Conclusion", "The system preserves evidence citations and editable scientific labels."), + ] + for page_number, (heading, body) in enumerate(sections, 1): + page = document.new_page(width=600, height=800) + page.insert_text((45, 25), "Proceedings of the Example Conference 2026", fontsize=8) + page.insert_text((45, 785), f"Anonymous Paper 1234 | Page {page_number}", fontsize=8) + page.insert_text((45, 70), heading, fontname="hebo", fontsize=12) + page.insert_textbox((45, 95, 555, 220), body, fontsize=10) + return _save_document(document, target) + + +def _native_chinese_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + font = fitz.Font(fontname="china-s") + page.insert_font(fontname="CJK", fontbuffer=font.buffer) + page.insert_text((45, 50), "\u6458\u8981", fontname="CJK", fontsize=13) + page.insert_textbox((45, 75, 555, 145), "\u672c\u6587\u63d0\u51fa\u8bba\u6587\u7f16\u7801\u5668\u548c\u5173\u7cfb\u89e3\u7801\u5668,\u751f\u6210\u53ef\u7f16\u8f91\u6846\u67b6\u56fe\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 185), "2 \u65b9\u6cd5", fontname="CJK", fontsize=13) + page.insert_textbox((45, 210, 555, 300), "\u8f93\u5165\u8bba\u6587\u9996\u5148\u8fdb\u5165\u6587\u6863\u7f16\u7801\u5668\u3002\u5173\u7cfb\u89e3\u7801\u5668\u8fde\u63a5\u5b9e\u4f53,\u6700\u540e\u8f93\u51fa\u53ef\u7f16\u8f91\u7684\u6f14\u793a\u6587\u7a3f\u56fe\u3002", fontname="CJK", fontsize=10) + page.insert_textbox((45, 340, 555, 400), "\u56fe 1:\u8bba\u6587\u7f16\u7801\u5668\u3001\u5173\u7cfb\u89e3\u7801\u5668\u4e0e\u53ef\u7f16\u8f91\u8f93\u51fa\u7684\u7cfb\u7edf\u6846\u67b6\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 455), "3 \u5b9e\u9a8c", fontname="CJK", fontsize=13) + page.insert_textbox((45, 480, 555, 545), "\u5b9e\u9a8c\u8bc4\u4f30\u5b9e\u4f53\u53ec\u56de\u7387\u3001\u5173\u7cfb\u53ec\u56de\u7387\u548c\u5904\u7406\u65f6\u95f4\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 600), "4 \u7ed3\u8bba", fontname="CJK", fontsize=13) + page.insert_textbox((45, 625, 555, 690), "\u8be5\u65b9\u6cd5\u80fd\u591f\u4fdd\u6301\u8bba\u6587\u672f\u8bed\u3001\u8bc1\u636e\u9875\u7801\u548c\u6a21\u5757\u5173\u7cfb\u3002", fontname="CJK", fontsize=10) + return _save_document(document, target) + + +def _native_spanish_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Resumen", fontname="hebo", fontsize=13) + page.insert_textbox((45, 75, 555, 145), "Proponemos un codificador de documentos que transforma el artículo en evidencia estructurada y genera una figura editable.", fontsize=10) + page.insert_text((45, 185), "2 Método", fontname="hebo", fontsize=13) + page.insert_textbox((45, 210, 555, 295), "El documento de entrada pasa al codificador de documentos. El decodificador de relaciones conecta las entidades y produce una salida editable.", fontsize=10) + page.insert_textbox((45, 335, 555, 395), "Figura 1: Codificador de documentos, decodificador de relaciones y salida editable.", fontsize=10) + page.insert_text((45, 455), "3 Experimentos", fontname="hebo", fontsize=13) + page.insert_textbox((45, 480, 555, 545), "Evaluamos la recuperación de entidades, las relaciones correctas y el tiempo de procesamiento.", fontsize=10) + page.insert_text((45, 600), "4 Conclusión", fontname="hebo", fontsize=13) + page.insert_textbox((45, 625, 555, 690), "El método conserva la terminología científica y las referencias de evidencia.", fontsize=10) + return _save_document(document, target) + + +def _hyphenated_native_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 75, 555, 145), "We introduce a trans-\nformer encoder for evidence-grounded figure generation.", fontsize=10) + page.insert_text((45, 185), "2 Method", fontname="hebo", fontsize=12) + page.insert_textbox((45, 210, 555, 300), "The trans-\nformer encoder passes the input representation to a relation de-\ncoder and produces an editable output.", fontsize=10) + return _save_document(document, target) + + +def _formula_table_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 45), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 65, 555, 115), "We introduce a calibrated relation encoder and evaluate it with a dense ablation table.", fontsize=10) + page.insert_text((45, 145), "2 Method", fontname="hebo", fontsize=12) + page.insert_text((80, 185), "Q = X W_q K = X W_k V = X W_v", fontname="cour", fontsize=10) + page.insert_text((80, 215), "A = softmax(Q K^T / sqrt(d)) V", fontname="cour", fontsize=10) + page.insert_textbox((45, 245, 555, 295), "The relation encoder sends the calibrated representation to the prediction head.", fontsize=10) + page.insert_text((45, 335), "3 Experiments", fontname="hebo", fontsize=12) + page.insert_textbox((45, 360, 555, 390), "Table 1: Ablation results for encoder depth and relation accuracy.", fontsize=10) + rows = [("Model", "Depth", "Accuracy"), ("Base", "6", "81.2"), ("Large", "12", "84.7"), ("Ours", "18", "87.9")] + top = 420 + for row in rows: + for left, value in zip((65, 260, 390), row): + page.insert_text((left, top), value, fontsize=10) + top += 28 + page.insert_textbox((45, 555, 555, 600), "The ablation shows that calibrated relation encoding improves accuracy.", fontsize=10) + page.insert_text((45, 650), "4 Conclusion", fontname="hebo", fontsize=12) + page.insert_textbox((45, 675, 555, 720), "The method preserves directed evidence relations and calibrated outputs.", fontsize=10) + return _save_document(document, target) + + +def _rotated_repeated_margin_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + for page_number in range(1, 5): + page = document.new_page(width=600, height=800) + page.insert_text((45, 25), "Rotated Conference Header 2026", fontsize=8) + page.insert_text((45, 785), f"Rotated Paper 42 | Page {page_number}", fontsize=8) + page.insert_text((45, 75), f"{page_number} Method Stage", fontname="hebo", fontsize=12) + page.insert_textbox((45, 105, 555, 220), f"Stage {page_number} preserves rotated scientific evidence and semantic reading order.", fontsize=10) + page.set_rotation(90) + return _save_document(document, target) + + +def _skewed_scan_fixture(source: Path, target: Path, angle: float = 2.0) -> Path: + import fitz + from PIL import Image + + source_document = fitz.open(str(source)) + try: + pixmap = source_document[0].get_pixmap(dpi=144, alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))).convert("RGB") + finally: + source_document.close() + rotated = image.rotate(float(angle), resample=Image.Resampling.BICUBIC, expand=False, fillcolor="white") + payload = io.BytesIO() + rotated.save(payload, format="PNG") + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_image(page.rect, stream=payload.getvalue()) + return _save_document(document, target) + + +def _degraded_scan_fixture(source: Path, target: Path) -> Path: + import fitz + from PIL import Image, ImageFilter + + source_document = fitz.open(str(source)) + try: + pixmap = source_document[0].get_pixmap(dpi=120, alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))).convert("L") + finally: + source_document.close() + degraded = image.resize((420, 560), Image.Resampling.LANCZOS).filter(ImageFilter.GaussianBlur(0.3)).convert("RGB") + payload = io.BytesIO() + degraded.save(payload, format="JPEG", quality=45, optimize=True) + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_image(page.rect, stream=payload.getvalue()) + return _save_document(document, target) + + +def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: + if "page_002" not in image_path.name: + return [] + lines = [ + ("2 Method", 110), + ("The scanned encoder passes evidence to the scanned decoder and output module.", 220), + ("Figure 1: Scanned architecture overview with encoder decoder and output.", 420), + ] + return [ + { + "text": text, + "confidence": 0.99, + "quad": [[90, top], [1240, top], [1240, top + 55], [90, top + 55]], + } + for text, top in lines + ] + + +def _render_preview(source: Path, target: Path, page_index: int = 0) -> Path: + import fitz + + document = fitz.open(str(source)) + try: + pixmap = document[page_index].get_pixmap(dpi=110, alpha=False) + target.parent.mkdir(parents=True, exist_ok=True) + pixmap.save(str(target)) + finally: + document.close() + return target + + +def _check(name: str, passed: bool, actual: Any, expected: Any) -> dict[str, Any]: + return {"name": name, "passed": bool(passed), "actual": actual, "expected": expected} + + +def _run_case( + case_id: str, + fixture: Path, + out_dir: Path, + checks: Callable[[dict[str, Any]], list[dict[str, Any]]], + *, + ocr_engine: str = "off", + ocr_adapter: Callable | None = None, + preview_page: int = 0, +) -> dict[str, Any]: + started = time.monotonic() + parsed = parse_paper(fixture, ocr_engine=ocr_engine, ocr_adapter=ocr_adapter) + elapsed = round(time.monotonic() - started, 4) + case_dir = ensure_dir(out_dir / case_id) + write_json(case_dir / "document_model.json", parsed) + write_json(case_dir / "extraction_report.json", parsed.get("extraction_report", {})) + preview = _render_preview(fixture, case_dir / f"preview_page_{preview_page + 1}.png", preview_page) + assertions = checks(parsed) + ocr_page_durations = parsed.get("extraction_report", {}).get("ocr_page_durations", []) + result = { + "case_id": case_id, + "ok": all(item.get("passed") for item in assertions), + "fixture": str(fixture), + "preview": str(preview), + "elapsed_seconds": elapsed, + "pdf_type": parsed.get("extraction_report", {}).get("pdf_type"), + "page_count": parsed.get("page_count"), + "section_count": parsed.get("extraction_report", {}).get("section_count"), + "figure_caption_count": parsed.get("extraction_report", {}).get("figure_caption_count"), + "ocr_pages": parsed.get("extraction_report", {}).get("ocr_pages", []), + "ocr_spacing_repair_count": parsed.get("extraction_report", {}).get("ocr_spacing_repair_count", 0), + "ocr_page_durations": ocr_page_durations, + "ocr_stage_seconds": { + name: round(sum(float(item.get(name) or 0.0) for item in ocr_page_durations), 4) + for name in ("render_seconds", "detection_seconds", "classification_seconds", "recognition_seconds", "postprocess_seconds", "inference_seconds") + }, + "assertions": assertions, + } + write_json(case_dir / "benchmark_result.json", result) + return result + + +def _percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * fraction) - 1)) + return round(ordered[index], 4) + + +def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> dict[str, Any]: + root = ensure_dir(out).resolve() + fixtures = ensure_dir(root / "fixtures") + native = _native_two_column_fixture(fixtures / "native_two_column.pdf") + bold = _bold_heading_fixture(fixtures / "unnumbered_bold_sections.pdf") + rotated = _rotated_fixture(fixtures / "rotated_native_page.pdf") + mixed = _mixed_scan_fixture(fixtures / "mixed_scan.pdf") + repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") + chinese = _native_chinese_fixture(fixtures / "native_chinese.pdf") + spanish = _native_spanish_fixture(fixtures / "native_spanish.pdf") + hyphenated = _hyphenated_native_fixture(fixtures / "hyphenated_native.pdf") + formula_table = _formula_table_fixture(fixtures / "formula_table_dense.pdf") + rotated_margin = _rotated_repeated_margin_fixture(fixtures / "rotated_repeated_margin.pdf") + + results = [ + _run_case( + "native_two_column", + native, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("cross_column_caption_order", parsed["pages"][0]["text"].index("RIGHT_BOTTOM") < parsed["pages"][0]["text"].index("Figure 1:") < parsed["pages"][0]["text"].index("3 Conclusion"), parsed["pages"][0]["text"], "RIGHT_BOTTOM < Figure 1 < Conclusion"), + _check("figure_caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "conclusion")), parsed["extraction_report"].get("section_coverage"), "abstract/method/conclusion"), + ], + ), + _run_case( + "unnumbered_bold_sections", + bold, + root, + lambda parsed: [ + _check("model_architecture_heading", "Model Architecture" in parsed.get("headings", []), parsed.get("headings", []), "Model Architecture"), + _check("stack_heading", "Encoder and Decoder Stacks" in parsed.get("headings", []), parsed.get("headings", []), "Encoder and Decoder Stacks"), + _check("section_hint", any(item.get("section_hint") == "Model Architecture" and "image encoder" in str(item.get("text") or "").casefold() for item in parsed.get("evidence", [])), [item.get("section_hint") for item in parsed.get("evidence", []) if "image encoder" in str(item.get("text") or "").casefold()], "Model Architecture"), + ], + ), + _run_case( + "rotated_native_page", + rotated, + root, + lambda parsed: [ + _check("rotation", parsed["extraction_report"].get("rotated_pages") == [1], parsed["extraction_report"].get("rotated_pages"), [1]), + _check("display_coordinates", all(0 <= block["bbox"][0] <= block["bbox"][2] <= parsed["pages"][0]["width"] and 0 <= block["bbox"][1] <= block["bbox"][3] <= parsed["pages"][0]["height"] for block in parsed["pages"][0]["blocks"]), parsed["pages"][0]["blocks"], "all blocks inside displayed page"), + _check("semantic_reading_order", parsed["pages"][0]["text"].index("Abstract") < parsed["pages"][0]["text"].index("2 Method") < parsed["pages"][0]["text"].index("Figure 1:"), parsed["pages"][0]["text"], "Abstract < Method < Figure 1"), + _check("caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), + ], + ), + _run_case( + "mixed_scan_fixture_adapter", + mixed, + root, + lambda parsed: [ + _check("mixed_pdf", parsed["extraction_report"].get("pdf_type") == "mixed", parsed["extraction_report"].get("pdf_type"), "mixed"), + _check("local_ocr", parsed["extraction_report"].get("ocr_pages") == [2], parsed["extraction_report"].get("ocr_pages"), [2]), + _check("method_recovered", parsed["extraction_report"].get("section_coverage", {}).get("method") is True, parsed["extraction_report"].get("section_coverage"), "method=true"), + _check("scanned_caption", any("Scanned architecture overview" in str(item.get("caption") or "") for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "scanned Figure 1 caption"), + ], + ocr_engine="easyocr", + ocr_adapter=_fixture_ocr_adapter, + preview_page=1, + ), + _run_case( + "repeated_margin_noise", + repeated_margin, + root, + lambda parsed: [ + _check("noise_removed", parsed["extraction_report"].get("repeated_margin_noise_removed_count") == 8, parsed["extraction_report"].get("repeated_margin_noise_removed_count"), 8), + _check("header_absent", all("Proceedings of the Example Conference" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no repeated header evidence"), + _check("footer_absent", all("Anonymous Paper 1234" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no repeated footer evidence"), + _check("scientific_content_preserved", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder evidence"), + ], + ), + _run_case( + "native_chinese", + chinese, + root, + lambda parsed: [ + _check("unicode_preserved", "\u6587\u6863\u7f16\u7801\u5668" in parsed["pages"][0]["text"], parsed["pages"][0]["text"], "contains document encoder in Chinese"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Chinese abstract/method/experiments/conclusion"), + _check("figure_caption", any("\u8bba\u6587\u7f16\u7801\u5668" in str(item.get("caption") or "") for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "Chinese Figure 1 caption"), + _check("no_mojibake", parsed["extraction_report"].get("mojibake_rate") == 0.0, parsed["extraction_report"].get("mojibake_rate"), 0.0), + ], + ), + _run_case( + "native_spanish", + spanish, + root, + lambda parsed: [ + _check("unicode_preserved", "artículo" in parsed["pages"][0]["text"] and "Conclusión" in parsed["pages"][0]["text"], parsed["pages"][0]["text"], "Spanish accents preserved"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Spanish abstract/method/experiments/conclusion"), + _check("figure_caption", any("codificador de documentos" in str(item.get("caption") or "").casefold() for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "Spanish Figura 1 caption"), + _check("source_language_preserved", "document encoder" not in parsed["pages"][0]["text"].casefold() and "codificador de documentos" in parsed["pages"][0]["text"].casefold(), parsed["pages"][0]["text"], "Spanish terminology without English translation"), + ], + ), + _run_case( + "hyphenated_native", + hyphenated, + root, + lambda parsed: [ + _check("transformer_repaired", sum("transformer encoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])) >= 2, [item.get("text") for item in parsed.get("evidence", [])], "two transformer encoder mentions"), + _check("decoder_repaired", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder"), + _check("repair_count", parsed["extraction_report"].get("native_hyphenation_repair_count") == 3, parsed["extraction_report"].get("native_hyphenation_repair_count"), 3), + ], + ), + _run_case( + "formula_table_dense", + formula_table, + root, + lambda parsed: [ + _check("formulas", parsed["extraction_report"].get("formula_count") == 2, parsed["extraction_report"].get("formula_count"), 2), + _check("table_caption", parsed["extraction_report"].get("table_caption_count") == 1, parsed["extraction_report"].get("table_caption_count"), 1), + _check("table_columns", parsed["document_index"]["tables"][0].get("columns") == ["Model", "Depth", "Accuracy"], parsed["document_index"]["tables"][0].get("columns"), ["Model", "Depth", "Accuracy"]), + _check("table_rows", any(row.get("values") == ["Large", "12", "84.7"] for row in parsed["document_index"]["tables"][0].get("rows", [])), parsed["document_index"]["tables"][0].get("rows"), ["Large", "12", "84.7"]), + _check("structured_table_evidence", any(item.get("kind") == "table" and "Row: Large | 12 | 84.7" in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", []) if item.get("kind") == "table"], "structured Large row"), + _check("no_orphan_cells", not any(item.get("text") == "84.7" for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no isolated table cells"), + ], + ), + _run_case( + "rotated_repeated_margin", + rotated_margin, + root, + lambda parsed: [ + _check("all_pages_rotated", parsed["extraction_report"].get("rotated_pages") == [1, 2, 3, 4], parsed["extraction_report"].get("rotated_pages"), [1, 2, 3, 4]), + _check("rotated_noise_removed", parsed["extraction_report"].get("repeated_margin_noise_removed_count") == 8, parsed["extraction_report"].get("repeated_margin_noise_removed_count"), 8), + _check("repeated_section_boundaries", parsed["extraction_report"].get("section_count") == 4, parsed["extraction_report"].get("section_count"), 4), + _check("header_absent", all("Rotated Conference Header" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no rotated header evidence"), + _check("body_preserved", all(any(f"Stage {page_number} preserves" in item.get("text", "") for item in parsed.get("evidence", [])) for page_number in range(1, 5)), [item.get("text") for item in parsed.get("evidence", [])], "all rotated stage evidence"), + ], + ), + ] + + if ocr_engine != "off": + scanned_two_column = rasterize_pdf_as_scan(native, fixtures / "scanned_two_column.pdf", dpi=144) + skewed_two_column = _skewed_scan_fixture(native, fixtures / "skewed_two_column.pdf") + degraded_two_column = _degraded_scan_fixture(native, fixtures / "degraded_two_column.pdf") + scanned_formula_table = rasterize_pdf_as_scan(formula_table, fixtures / "scanned_formula_table.pdf", dpi=144) + scanned_spanish = rasterize_pdf_as_scan(spanish, fixtures / "scanned_spanish.pdf", dpi=144) + results.append(_run_case( + "mixed_scan_runtime_ocr", + mixed, + root, + lambda parsed: [ + _check("ocr_completed", 2 in parsed["extraction_report"].get("ocr_pages", []), parsed["extraction_report"].get("ocr_pages", []), "contains page 2"), + _check("method_spacing", "2 method" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains '2 Method'"), + _check("sentence_spacing", "scanned encoder passes evidence to the scanned decoder and output" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "spaced scientific sentence"), + _check("caption_spacing", "scanned architecture overview with encoder decoder and output" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "spaced caption"), + _check("repairs_recorded", parsed["extraction_report"].get("ocr_spacing_repair_count", 0) > 0, parsed["extraction_report"].get("ocr_spacing_repair_count", 0), ">0"), + ], + ocr_engine=ocr_engine, + preview_page=1, + )) + results.append(_run_case( + "scanned_two_column_runtime_ocr", + scanned_two_column, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("headings", all(value in parsed.get("headings", []) for value in ("Abstract", "Method", "Conclusion")), parsed.get("headings", []), "Abstract/Method/Conclusion"), + _check("caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "skewed_two_column_runtime_ocr", + skewed_two_column, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("mean_confidence", float(parsed["extraction_report"].get("mean_ocr_confidence") or 0.0) >= 0.9, parsed["extraction_report"].get("mean_ocr_confidence"), ">=0.9"), + _check("caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "degraded_two_column_quality_gate", + degraded_two_column, + root, + lambda parsed: [ + _check("quality_gate_applied", parsed["extraction_report"].get("ocr_latin_lexical_gate_applied") is True, parsed["extraction_report"].get("ocr_latin_lexical_gate_applied"), True), + _check("low_known_word_ratio", float(parsed["extraction_report"].get("ocr_latin_known_word_ratio") or 1.0) < 0.5, parsed["extraction_report"].get("ocr_latin_known_word_ratio"), "<0.5"), + _check("extraction_failed", parsed["extraction_report"].get("status") == "fail", parsed["extraction_report"].get("status"), "fail"), + _check("no_false_cache_scope", parsed["extraction_report"].get("sampled_scan_ready") is False, parsed["extraction_report"].get("sampled_scan_ready"), False), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "scanned_formula_table_runtime_ocr", + scanned_formula_table, + root, + lambda parsed: [ + _check("formula_recovered", parsed["extraction_report"].get("formula_count", 0) >= 1, parsed["extraction_report"].get("formula_count"), ">=1"), + _check("table_columns", parsed["document_index"]["tables"][0].get("columns") == ["Model", "Depth", "Accuracy"], parsed["document_index"]["tables"][0].get("columns"), ["Model", "Depth", "Accuracy"]), + _check("large_row", any(row.get("values") == ["Large", "12", "84.7"] for row in parsed["document_index"]["tables"][0].get("rows", [])), parsed["document_index"]["tables"][0].get("rows"), ["Large", "12", "84.7"]), + _check("structured_evidence", any(item.get("kind") == "table" and "Row: Large | 12 | 84.7" in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", []) if item.get("kind") == "table"], "structured Large row"), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "scanned_spanish_runtime_ocr", + scanned_spanish, + root, + lambda parsed: [ + _check("ocr_completed", parsed["extraction_report"].get("ocr_pages") == [1], parsed["extraction_report"].get("ocr_pages"), [1]), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Spanish abstract/method/experiments/conclusion"), + _check("figure_caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + _check("source_language_preserved", "document encoder" not in parsed["pages"][0]["text"].casefold() and "codificador de documentos" in parsed["pages"][0]["text"].casefold(), parsed["pages"][0]["text"], "Spanish terminology without English translation"), + _check("quality_gate_passed", parsed["extraction_report"].get("status") != "fail", parsed["extraction_report"].get("status"), "pass or warning"), + ], + ocr_engine=ocr_engine, + )) + + elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] + ocr_stage_totals = { + name: round(sum(float(item.get("ocr_stage_seconds", {}).get(name) or 0.0) for item in results), 4) + for name in ("render_seconds", "detection_seconds", "classification_seconds", "recognition_seconds", "postprocess_seconds", "inference_seconds") + } + report = { + "summary": "Deterministic multi-layout PDF extraction stress suite completed.", + "ok": all(item.get("ok") for item in results), + "out_dir": str(root), + "ocr_engine": ocr_engine, + "aggregate": { + "case_count": len(results), + "passed_case_count": sum(bool(item.get("ok")) for item in results), + "mean_elapsed_seconds": round(sum(elapsed) / max(1, len(elapsed)), 4), + "p95_elapsed_seconds": _percentile(elapsed, 0.95), + "ocr_case_count": sum(bool(item.get("ocr_pages")) for item in results), + "ocr_stage_seconds": ocr_stage_totals, + }, + "cases": results, + } + write_json(root / "pdf_extraction_stress_report.json", report) + return report diff --git a/rfs/evaluation/rebuild_coverage.py b/rfs/evaluation/rebuild_coverage.py new file mode 100644 index 0000000..d050f43 --- /dev/null +++ b/rfs/evaluation/rebuild_coverage.py @@ -0,0 +1,94 @@ +"""Canonical layout/control coverage audit for editable rebuilds.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from ..analysis.rebuild.controls import _append_unique as _append_unique_control, _normalize_controls +from ..utils import write_json + + +def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: + x, y = max(0.0, min(0.995, x)), max(0.0, min(0.995, y)) + w, h = max(0.001, min(w, 1.0 - x)), max(0.001, min(h, 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _collection(kind: str) -> str: + normalized = str(kind or "").strip().lower().replace("-", "_") + return {"panel": "panels", "panels": "panels", "card": "cards", "cards": "cards", "slot": "slots", "visual_slot": "slots", "slots": "slots", "legend": "legend_regions", "legend_region": "legend_regions", "legend_regions": "legend_regions"}.get(normalized, "") + + +def _exists(items: list[dict], candidate: dict) -> bool: + candidate_id = str(candidate.get("id") or "") + box = candidate.get("bbox_percent") if isinstance(candidate.get("bbox_percent"), dict) else None + for item in items: + if candidate_id and str(item.get("id") or "") == candidate_id: + return True + other = item.get("bbox_percent") if isinstance(item.get("bbox_percent"), dict) else None + if box and other: + ix = max(0.0, min(float(box["x"]) + float(box["w"]), float(other["x"]) + float(other["w"])) - max(float(box["x"]), float(other["x"]))) + iy = max(0.0, min(float(box["y"]) + float(box["h"]), float(other["y"]) + float(other["h"])) - max(float(box["y"]), float(other["y"]))) + if ix * iy / max(float(box["w"]) * float(box["h"]), 1e-9) > .72: + return True + return False + + +def _normalize_missing_layout_object(item: dict, program: dict, collection: str) -> dict | None: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): return None + prefix = {"panels": "panel", "cards": "card", "slots": "slot", "legend_regions": "legend"}.get(collection, "item") + used = {str(obj.get("id") or "") for key in ("panels", "cards", "slots", "legend_regions") for obj in program.get(key, []) or [] if isinstance(obj, dict)} + base, ident, suffix = str(item.get("id") or f"{prefix}_{len(used) + 1:02d}"), "", 2 + ident = base + while ident in used: ident, suffix = f"{base}_{suffix}", suffix + 1 + raw = item["bbox_percent"] + record = {**item, "id": ident, "bbox_percent": _bbox(float(raw.get("x", 0)), float(raw.get("y", 0)), float(raw.get("w", .001)), float(raw.get("h", .001))), "confidence": float(item.get("confidence") or .75), "detected_by": "combined_coverage_audit_vlm", "coverage_audit_added": True} + if collection == "slots": + record.setdefault("asset_id", ident); record.setdefault("paper_concept", record.get("prompt_subject") or record.get("label") or ident); record.setdefault("composition_type", "full_frame_icon"); record.setdefault("show_slot_caption", False); record.setdefault("panel_id", "reference_canvas"); record.setdefault("z_index", 20 + len(program.get("slots", []))) + elif collection == "cards": + record.setdefault("title", record.get("label") or ident); record.setdefault("semantic_role", "subcard_frame"); record.setdefault("shape_kind", "rounded_rect") + if str(record.get("stroke_color") or "").strip(): record.setdefault("color_status", "vlm_reference_estimated"); record.setdefault("color_source", "vlm") + else: record["stroke_color"] = "#59AFCB"; record.setdefault("color_status", "fallback_missing_vlm_color"); record.setdefault("color_source", "local_default") + record.setdefault("stroke_width_pt", 1.5); record["dash_style"] = "dash" if str(record.get("dash_style") or "solid").lower() == "dashed" else str(record.get("dash_style") or "solid"); record.setdefault("fill_color", "#FFFFFF"); record.setdefault("fill_transparency", 1.0); record.setdefault("corner_radius", .08); record.setdefault("z_index", 12); record.setdefault("editable_in", "pptx") + else: + record.setdefault("title", record.get("label") or ident); record.setdefault("editable_in", "pptx") + return record + + +def _combined_bindable_objects(program: dict) -> list[dict]: + result, seen = [], set() + for key in ("panels", "cards", "slots"): + for item in program.get(key, []) or []: + ident = str(item.get("id") or "") if isinstance(item, dict) else "" + if ident and ident not in seen and isinstance(item.get("bbox_percent"), dict): result.append(item); seen.add(ident) + return result + + +def _apply_combined_coverage_audit(reference: Path, out: Path, program: dict, reference_geometry: dict, reference_controls: dict, palette: list[str], flow_graph: dict | None, mode: str, adapter: Callable | None) -> tuple[dict, dict, dict, dict]: + audit = {"summary": "Combined layout/control coverage audit report.", "mode": mode, "status": "skipped", "missing_layout_count": 0, "appended_layout_count": 0, "missing_control_count": 0, "appended_control_count": 0} + if mode != "vlm" or not adapter: + audit["status"] = "skipped" if mode != "vlm" else "unavailable"; write_json(out / "reference_combined_coverage_audit.json", audit); return program, reference_geometry, reference_controls, audit + try: + raw = adapter(reference, reference_geometry, reference_controls, flow_graph) or {}; raw_layout, raw_controls = raw.get("missing_layout_objects", []), raw.get("missing_controls", []); invalid = appended = 0 + for item in raw_layout: + collection = _collection(item.get("kind") if isinstance(item, dict) else ""); candidate = _normalize_missing_layout_object(item, program, collection) if collection else None + if not candidate: invalid += 1; continue + target = program.setdefault(collection, []) + if _exists(target, candidate): continue + target.append(candidate); reference_geometry.setdefault(collection, []) + if not _exists(reference_geometry[collection], candidate): reference_geometry[collection].append(candidate) + appended += 1 + arrows, added = list(reference_controls.get("arrows", []) if isinstance(reference_controls.get("arrows"), list) else []), 0 + normalized = _normalize_controls(raw_controls, _combined_bindable_objects(program), palette) + for candidate in normalized: + candidate = {**candidate, "detected_by": "combined_coverage_audit_vlm", "binding_source": "combined_coverage_audit_vlm"} + if _append_unique_control(arrows, candidate): added += 1 + if added: reference_controls = {**reference_controls, "arrows": arrows, "merged_arrow_count": len(arrows), "combined_coverage_appended_count": added}; program["arrows"] = arrows + audit.update({"status": "used", "raw_audit": raw, "missing_layout_count": len(raw_layout), "valid_missing_layout_count": max(0, len(raw_layout) - invalid), "appended_layout_count": appended, "invalid_layout_count": invalid, "missing_control_count": len(raw_controls), "valid_missing_control_count": len(normalized), "appended_control_count": added}) + except Exception as exc: audit.update({"status": "fallback", "fallback_reason": f"combined_coverage_audit_failed:{exc}"}) + write_json(out / "reference_combined_coverage_audit.json", audit) + return program, reference_geometry, reference_controls, audit + + +__all__ = ["_apply_combined_coverage_audit", "_combined_bindable_objects", "_normalize_missing_layout_object"] diff --git a/rfs/evaluation/rebuild_visual.py b/rfs/evaluation/rebuild_visual.py new file mode 100644 index 0000000..9b1537b --- /dev/null +++ b/rfs/evaluation/rebuild_visual.py @@ -0,0 +1,530 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +from ..utils import write_json +from ..providers import call_vlm_json, resolve_vlm_model + + +def _bbox(item: dict) -> dict | None: + box = item.get("bbox_percent") + return box if isinstance(box, dict) else None + + +def _valid_bbox(box: dict | None) -> bool: + if not isinstance(box, dict): + return False + try: + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + except Exception: + return False + return 0 <= x <= 1 and 0 <= y <= 1 and w > 0 and h > 0 and x + w <= 1.0001 and y + h <= 1.0001 + + +def _overlap_area(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + return ix * iy + + +def _overlap_ratio(a: dict, b: dict) -> float: + area = _overlap_area(a, b) + if area <= 0: + return 0.0 + a_area = float(a["w"]) * float(a["h"]) + b_area = float(b["w"]) * float(b["h"]) + return area / max(min(a_area, b_area), 0.000001) + + +def _visible_text_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + return [item for item in text_program.get("items", []) if isinstance(item, dict) and item.get("visible", True)] + + +def _text_alignment_group_key(item: dict) -> tuple[str, str] | None: + for key in ("paragraph_id", "text_group_id", "group_id", "source_group_id"): + value = str(item.get(key) or "").strip() + if value: + return (key, value) + return None + + +def _detect_text_issues(program: dict) -> list[dict]: + issues = [] + items = _visible_text_items(program) + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": "text_bbox_out_of_bounds", "text_id": item.get("id"), "reason": "text bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > 0.18: + issues.append({ + "type": "text_overlap", + "text_id": left.get("id"), + "other_text_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": "visible text boxes overlap substantially", + }) + grouped: dict[tuple[str, str], list[dict]] = {} + for item in items: + key = _text_alignment_group_key(item) + if key: + grouped.setdefault(key, []).append(item) + for (group_key, group_id), group in grouped.items(): + if len(group) < 3: + continue + centers = [ + float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2 + for item in group + if _valid_bbox(_bbox(item)) + ] + if len(centers) >= 3 and max(centers) - min(centers) > 0.035: + issues.append({ + "type": "text_group_misaligned", + "group_key": group_key, + "group_id": group_id, + "text_ids": [item.get("id") for item in group], + "center_y_span": round(max(centers) - min(centers), 4), + "reason": "explicit text group is visually uneven", + }) + return issues + + +def _detect_object_issues(program: dict) -> list[dict]: + issues = [] + for collection_name in ("panels", "slots"): + items = [item for item in program.get(collection_name, []) if isinstance(item, dict)] + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": f"{collection_name[:-1]}_bbox_out_of_bounds", "id": item.get("id"), "reason": "bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > (0.35 if collection_name == "slots" else 0.18): + issues.append({ + "type": f"{collection_name[:-1]}_overlap", + "id": left.get("id"), + "other_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": f"{collection_name[:-1]} boxes overlap substantially", + }) + return issues + + +def _detect_arrow_issues(program: dict) -> list[dict]: + issues = [] + for arrow in program.get("arrows", []) or []: + if not isinstance(arrow, dict): + continue + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + issues.append({"type": "arrow_missing_path", "arrow_id": arrow.get("id"), "reason": "arrow has no usable path_percent"}) + return issues + + +def _detect_ownership_issues(program: dict, ownership_report: dict | None) -> list[dict]: + issues = [] + text_ids = {str(item.get("source_reference_text_id") or item.get("id") or "") for item in _visible_text_items(program)} + report = ownership_report if isinstance(ownership_report, dict) else {} + for item in report.get("items", []) or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or "") + ownership = str(item.get("layer_ownership") or "") + included = bool(item.get("included_in_text_program")) + if ownership != "editable_text_layer" and text_id in text_ids: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "non-editable-owned text is present in text_program"}) + if ownership == "editable_text_layer" and not included: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "editable-owned text is missing from text_program"}) + return issues + + +def run_rebuild_visual_quality_check( + out_dir: str | Path, + program: dict, + reference_geometry: dict | None = None, + reference_controls: dict | None = None, + ownership_report: dict | None = None, + mode: str = "heuristic", +) -> dict[str, Any]: + text_issues = _detect_text_issues(program) + object_issues = _detect_object_issues(program) + arrow_issues = _detect_arrow_issues(program) + ownership_issues = _detect_ownership_issues(program, ownership_report) + issues = text_issues + object_issues + arrow_issues + ownership_issues + blocking_types = {"text_overlap", "text_bbox_out_of_bounds", "arrow_missing_path", "text_layer_ownership_conflict"} + blocking = [item for item in issues if item.get("type") in blocking_types] + report = { + "summary": "Deterministic rebuild visual quality report.", + "mode": mode, + "status": "blocked" if blocking else ("warning" if issues else "pass"), + "issue_count": len(issues), + "blocking_issue_count": len(blocking), + "text_issue_count": len(text_issues), + "object_issue_count": len(object_issues), + "arrow_issue_count": len(arrow_issues), + "ownership_issue_count": len(ownership_issues), + "issues": issues, + "reference_geometry_status": (reference_geometry or {}).get("status"), + "reference_controls_status": (reference_controls or {}).get("status"), + "policy": "deterministic check only; no program mutation", + } + write_json(Path(out_dir) / "rebuild_visual_quality_report.json", report) + return report + + +def _brief_text(program: dict) -> list[dict]: + return [{ + "id": item.get("id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "text": item.get("text"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "align": item.get("align"), + "visible": item.get("visible", True), + "z_index": item.get("z_index"), + "layer_ownership": item.get("layer_ownership"), + } for item in _visible_text_items(program)] + + +def _brief_objects(program: dict, key: str) -> list[dict]: + return [{ + "id": item.get("id"), + "bbox_percent": item.get("bbox_percent"), + "title": item.get("title"), + "paper_concept": item.get("paper_concept"), + } for item in program.get(key, []) or [] if isinstance(item, dict)] + + +def _brief_arrows(program: dict) -> list[dict]: + return [{ + "id": item.get("id"), + "source_id": item.get("source_id") or item.get("source"), + "target_id": item.get("target_id") or item.get("target"), + "path_percent": item.get("path_percent"), + "render_style": item.get("render_style"), + "stroke_width_pt": item.get("stroke_width_pt"), + "line_pattern": item.get("line_pattern"), + } for item in program.get("arrows", []) or [] if isinstance(item, dict)] + + +def run_rebuild_visual_critic( + out_dir: str | Path, + reference_path: str | Path, + preview_path: str | Path | None, + program: dict, + deterministic_report: dict, + mode: str = "heuristic", + model: str | None = None, + iteration: int = 0, + vlm_timeout: int | None = None, + vlm_retries: int | None = None, +) -> dict: + out = Path(out_dir) + effective_mode = str(mode or "off").lower() + if effective_mode == "off": + critic = {"summary": "Rebuild visual critic skipped.", "mode": "off", "status": "skipped", "patches": [], "warnings": []} + elif effective_mode == "heuristic": + critic = { + "summary": "Heuristic rebuild visual critic produced no mutating patches.", + "mode": "heuristic", + "status": deterministic_report.get("status", "pass"), + "patches": [], + "issues": deterministic_report.get("issues", []), + "warnings": [], + } + elif effective_mode == "vlm": + if not preview_path or not Path(preview_path).exists(): + critic = { + "summary": "VLM rebuild visual critic skipped because preview PNG is unavailable.", + "mode": "vlm", + "status": "skipped", + "patches": [], + "warnings": ["preview_png_unavailable"], + } + else: + model_name = resolve_vlm_model("RFS_REBUILD_CRITIC_MODEL", "RFS_CRITIC_MODEL", explicit_model=model) + prompt = f""" +You are a visual patch generator for an editable PowerPoint reconstruction. +Compare image 1 (reference) and image 2 (current PPT preview). +Only output JSON. Do not output markdown, prose, Python, SVG, or PPT code. + +Every fix must be a concrete patch operation with exact fields. Do not say "optimize", "improve", or "make better" unless you also provide exact patch fields. +Examples of concrete text instructions: +- update_text with font_size_pt: 10, z_index: 8 +- update_text with bbox_percent: {{"x":0.615,"y":0.338,"w":0.12,"h":0.03}}, align:"center" +- update_arrow with path_percent:[[0.1,0.2],[0.3,0.2]], render_style:"line_connector" + +Allowed patch operations: +- update_text: text_id required; may set bbox_percent, font_size_pt, align, visible, z_index, layer_ownership. +- update_slot: slot_id required; may set bbox_percent only. +- update_panel: panel_id required; may set bbox_percent only. +- update_arrow: arrow_id required; may set path_percent, render_style, stroke_width_pt, line_pattern. + +Forbidden: +- Do not change text content. +- Do not request crop masking or image pixel edits. +- Do not write PPT code. +- Do not invent new objects. + +Return schema: +{{ + "summary": "...", + "mode": "vlm", + "status": "pass|needs_patch|blocked", + "patches": [ + {{"op":"update_text","text_id":"text_ref","font_size_pt":10,"z_index":8,"reason":"specific reason"}}, + {{"op":"update_slot","slot_id":"slot_1","bbox_percent":{{"x":0.1,"y":0.2,"w":0.2,"h":0.1}},"reason":"specific reason"}}, + {{"op":"update_arrow","arrow_id":"arrow_1","path_percent":[[0.1,0.2],[0.3,0.2]],"reason":"specific reason"}} + ], + "warnings": [] +}} + +Deterministic issues: +{json.dumps(deterministic_report.get("issues", []), ensure_ascii=False)} + +Text items: +{json.dumps(_brief_text(program), ensure_ascii=False)} + +Panels: +{json.dumps(_brief_objects(program, "panels"), ensure_ascii=False)} + +Slots: +{json.dumps(_brief_objects(program, "slots"), ensure_ascii=False)} + +Arrows: +{json.dumps(_brief_arrows(program), ensure_ascii=False)} +""".strip() + critic = call_vlm_json(prompt, [reference_path, preview_path], model=model_name, timeout=vlm_timeout, retries=vlm_retries) + critic.setdefault("summary", "VLM rebuild visual critic returned concrete patch operations.") + critic.setdefault("mode", "vlm") + critic.setdefault("status", "needs_patch" if critic.get("patches") else "pass") + critic.setdefault("warnings", []) + critic["vlm_model"] = model_name + else: + raise ValueError(f"Unsupported rebuild visual critic mode: {mode}") + write_json(out / f"rebuild_visual_critic_iter_{iteration}.json", critic) + return critic + + +def _limit_bbox_change(original: dict, requested: dict, center_limit: float, size_limit: float) -> dict: + ox, oy, ow, oh = float(original["x"]), float(original["y"]), float(original["w"]), float(original["h"]) + rx = float(requested.get("x", ox)) + ry = float(requested.get("y", oy)) + rw = float(requested.get("w", ow)) + rh = float(requested.get("h", oh)) + ocx, ocy = ox + ow / 2, oy + oh / 2 + rcx, rcy = rx + rw / 2, ry + rh / 2 + dcx = max(-center_limit, min(center_limit, rcx - ocx)) + dcy = max(-center_limit, min(center_limit, rcy - ocy)) + min_w, max_w = ow * (1 - size_limit), ow * (1 + size_limit) + min_h, max_h = oh * (1 - size_limit), oh * (1 + size_limit) + nw = max(0.001, min(max_w, max(min_w, rw))) + nh = max(0.001, min(max_h, max(min_h, rh))) + nx = ocx + dcx - nw / 2 + ny = ocy + dcy - nh / 2 + clamped_x = max(0.0, min(0.995, nx)) + clamped_y = max(0.0, min(0.995, ny)) + return { + "x": round(clamped_x, 4), + "y": round(clamped_y, 4), + "w": round(max(0.001, min(nw, 1.0 - clamped_x)), 4), + "h": round(max(0.001, min(nh, 1.0 - clamped_y)), 4), + } + + +def _limit_font_change(original: object, requested: object) -> float | None: + try: + original_value = float(original) + requested_value = float(requested) + except Exception: + return None + if original_value <= 0: + return round(max(1.0, requested_value), 2) + return round(max(original_value * 0.8, min(original_value * 1.2, requested_value)), 2) + + +def _coerce_path(path: object) -> list[list[float]] | None: + if not isinstance(path, list) or len(path) < 2: + return None + points = [] + for item in path: + if not isinstance(item, list) or len(item) < 2: + return None + points.append([round(max(0.0, min(1.0, float(item[0]))), 4), round(max(0.0, min(1.0, float(item[1]))), 4)]) + return points + + +def _refresh_ownership_report(out: Path, program: dict) -> None: + path = out / "text_layer_ownership_report.json" + if not path.exists(): + return + try: + report = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return + text_program = program.get("text_program", {}) if isinstance(program.get("text_program"), dict) else {} + by_source = {str(item.get("source_reference_text_id") or ""): item for item in text_program.get("items", []) if isinstance(item, dict)} + changed = False + for item in report.get("items", []) or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or "") + if text_id in by_source: + item["layer_ownership"] = by_source[text_id].get("layer_ownership", item.get("layer_ownership")) + item["included_in_text_program"] = bool(by_source[text_id].get("visible", True)) and item["layer_ownership"] == "editable_text_layer" + changed = True + if changed: + write_json(path, report) + + +def apply_rebuild_corrections(out_dir: str | Path, program: dict, critic: dict, iteration: int = 0) -> tuple[dict, dict]: + out = Path(out_dir) + updated = copy.deepcopy(program) + patches = critic.get("patches", []) if isinstance(critic.get("patches"), list) else [] + applied = [] + rejected = [] + changed_slots: set[str] = set() + changed_arrows = False + text_program = updated.get("text_program") if isinstance(updated.get("text_program"), dict) else {"items": []} + text_by_id = {str(item.get("id") or ""): item for item in text_program.get("items", []) if isinstance(item, dict)} + slots_by_id = {str(item.get("id") or ""): item for item in updated.get("slots", []) if isinstance(item, dict)} + panels_by_id = {str(item.get("id") or ""): item for item in updated.get("panels", []) if isinstance(item, dict)} + arrows_by_id = {str(item.get("id") or ""): item for item in updated.get("arrows", []) if isinstance(item, dict)} + + for patch in patches: + if not isinstance(patch, dict): + rejected.append({"patch": patch, "reason": "patch_not_object"}) + continue + op = str(patch.get("op") or "").strip() + if op == "update_text": + text_id = str(patch.get("text_id") or "") + item = text_by_id.get(text_id) + if not item: + rejected.append({"patch": patch, "reason": "unknown_text_id"}) + continue + changed = {} + if isinstance(patch.get("bbox_percent"), dict) and isinstance(item.get("bbox_percent"), dict): + item["bbox_percent"] = _limit_bbox_change(item["bbox_percent"], patch["bbox_percent"], center_limit=0.03, size_limit=0.20) + item["center_percent"] = {"x": round(float(item["bbox_percent"]["x"]) + float(item["bbox_percent"]["w"]) / 2, 4), "y": round(float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2, 4)} + item["width_percent"] = item["bbox_percent"]["w"] + item["height_percent"] = item["bbox_percent"]["h"] + changed["bbox_percent"] = item["bbox_percent"] + if patch.get("font_size_pt") is not None: + limited = _limit_font_change(item.get("font_size_pt"), patch.get("font_size_pt")) + if limited is not None: + item["font_size_pt"] = limited + changed["font_size_pt"] = limited + if str(patch.get("align") or "").lower() in {"left", "center", "right"}: + item["align"] = str(patch["align"]).lower() + changed["align"] = item["align"] + if patch.get("visible") is not None: + item["visible"] = bool(patch.get("visible")) + changed["visible"] = item["visible"] + if patch.get("z_index") is not None: + try: + item["z_index"] = int(max(0, min(1000, int(patch.get("z_index"))))) + changed["z_index"] = item["z_index"] + except Exception: + pass + ownership = str(patch.get("layer_ownership") or "") + if ownership in {"editable_text_layer", "raster_asset_layer", "decorative_asset_text", "ignore"}: + item["layer_ownership"] = ownership + if ownership != "editable_text_layer": + item["visible"] = False + changed["layer_ownership"] = ownership + if changed: + applied.append({"op": op, "text_id": text_id, "applied": changed, "reason": patch.get("reason")}) + else: + rejected.append({"patch": patch, "reason": "no_allowed_text_fields"}) + elif op in {"update_slot", "update_panel"}: + key = "slot_id" if op == "update_slot" else "panel_id" + object_id = str(patch.get(key) or "") + item = (slots_by_id if op == "update_slot" else panels_by_id).get(object_id) + if not item or not isinstance(patch.get("bbox_percent"), dict) or not isinstance(item.get("bbox_percent"), dict): + rejected.append({"patch": patch, "reason": f"unknown_or_invalid_{key}"}) + continue + item["bbox_percent"] = _limit_bbox_change(item["bbox_percent"], patch["bbox_percent"], center_limit=0.05, size_limit=0.20) + if op == "update_slot": + changed_slots.add(object_id) + applied.append({"op": op, key: object_id, "applied": {"bbox_percent": item["bbox_percent"]}, "reason": patch.get("reason")}) + elif op == "update_arrow": + arrow_id = str(patch.get("arrow_id") or "") + item = arrows_by_id.get(arrow_id) + if not item: + rejected.append({"patch": patch, "reason": "unknown_arrow_id"}) + continue + changed = {} + path = _coerce_path(patch.get("path_percent")) + if path: + item["path_percent"] = path + changed["path_percent"] = path + if str(patch.get("render_style") or "") in {"filled_block_arrow", "line_connector", "elbow_connector", "branch_line_connector", "dashed_loop_connector"}: + item["render_style"] = str(patch["render_style"]) + changed["render_style"] = item["render_style"] + if patch.get("stroke_width_pt") is not None: + try: + item["stroke_width_pt"] = round(max(0.25, min(12.0, float(patch["stroke_width_pt"]))), 2) + changed["stroke_width_pt"] = item["stroke_width_pt"] + except Exception: + pass + if str(patch.get("line_pattern") or "") in {"solid", "dash", "dashed"}: + item["line_pattern"] = "dash" if str(patch["line_pattern"]) == "dashed" else str(patch["line_pattern"]) + changed["line_pattern"] = item["line_pattern"] + if changed: + changed_arrows = True + applied.append({"op": op, "arrow_id": arrow_id, "applied": changed, "reason": patch.get("reason")}) + else: + rejected.append({"patch": patch, "reason": "no_allowed_arrow_fields"}) + else: + rejected.append({"patch": patch, "reason": "unsupported_operation"}) + + correction_report = { + "summary": "Applied controlled rebuild visual critic corrections.", + "iteration": iteration, + "status": "changed" if applied else "no_changes", + "applied_count": len(applied), + "rejected_count": len(rejected), + "changed_slot_ids": sorted(changed_slots), + "changed_arrows": changed_arrows, + "applied": applied, + "rejected": rejected, + "limits": { + "text_center_move_percent": 0.03, + "text_font_scale": "80%-120%", + "slot_panel_center_move_percent": 0.05, + "slot_panel_size_scale": "80%-120%", + }, + } + write_json(out / f"rebuild_corrections_iter_{iteration}.json", correction_report) + if isinstance(updated.get("text_program"), dict): + write_json(out / "text_program.json", updated["text_program"]) + _refresh_ownership_report(out, updated) + return updated, correction_report diff --git a/rfs/evaluation/scanned_pdf.py b/rfs/evaluation/scanned_pdf.py new file mode 100644 index 0000000..8f67aed --- /dev/null +++ b/rfs/evaluation/scanned_pdf.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + + +def rasterize_pdf_as_scan(source: str | Path, target: str | Path, dpi: int = 144, jpeg_quality: int = 78) -> Path: + """Create an image-only PDF fixture while preserving displayed page sizes.""" + import fitz + + source_path = Path(source).resolve() + target_path = Path(target).resolve() + if not source_path.exists(): + raise FileNotFoundError(f"PDF does not exist: {source_path}") + if source_path.suffix.casefold() != ".pdf": + raise ValueError(f"Rasterized scan input must be a PDF: {source_path}") + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.unlink(missing_ok=True) + document = fitz.open(str(source_path)) + scanned = fitz.open() + try: + for page in document: + pixmap = page.get_pixmap(dpi=max(72, min(300, int(dpi))), alpha=False) + image = pixmap.tobytes("jpeg", jpg_quality=max(40, min(95, int(jpeg_quality)))) + output_page = scanned.new_page(width=float(page.rect.width), height=float(page.rect.height)) + output_page.insert_image(output_page.rect, stream=image) + scanned.set_metadata({ + "title": "ResearchFigureStudio rasterized scan benchmark", + "author": "ResearchFigureStudio", + "subject": "Deterministic image-only PDF benchmark fixture", + "keywords": "ResearchFigureStudio,scan,benchmark", + "creator": "ResearchFigureStudio", + "producer": "ResearchFigureStudio", + "creationDate": "", + "modDate": "", + }) + scanned.save(str(target_path), garbage=4, deflate=True, no_new_id=True, preserve_metadata=False) + finally: + scanned.close() + document.close() + return target_path diff --git a/rfs/evaluation/validator.py b/rfs/evaluation/validator.py new file mode 100644 index 0000000..413a5c2 --- /dev/null +++ b/rfs/evaluation/validator.py @@ -0,0 +1,721 @@ +from __future__ import annotations + +"""Canonical output validation implementation.""" + +import json +from pathlib import Path + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff"} +REQUIRED = [ + "input_manifest.json", + "reference_geometry.json", + "reference_control_candidates.json", + "slot_overlay.png", + "reference_control_overlay.png", + "reference_controls.json", + "arrow_style_profile.json", + "selected_arrow_routes.json", + "arrow_quality_report.json", + "slot_inventory.json", + "reference_style_profile.json", + "style_sheet.md", + "layout_plan.json", + "figure_program.json", + "reference_text_geometry.json", + "text_program.json", + "ocr_text_quality_report.json", + "text_alignment_report.json", + "slot_visual_spec.json", + "reference_slot_prompt_brief.json", + "slot_prompt_plan.json", + "prompts.md", + "asset_quality_report.json", + "asset_complexity_report.json", + "composition_quality_report.json", + "asset_visual_review.json", + "asset_contact_sheet.png", + "asset_candidate_contact_sheet.png", + "editable_composition.pptx", + "alignment_review.md", + "critic_report.md", + "visual_critic_iter_0.json", +] +FORBIDDEN_MARKERS = ["ok_after_crop", "cover-crop", "fill-crop", "forced crop", "crop-to-ratio", "single full diagram", "vector-only", "svg-only", "baked labels"] +COARSE_RATIO_PRESETS = {"1:1", "4:3", "3:4", "16:9", "9:16", "2:1", "1:2", "3:2", "2:3"} +ARROW_ASSET_ID_TERMS = ("arrow", "dashed_arc", "dashed_arrows", "transition_arrow", "loop_dashed", "graph_connector") +COMPLEXITY_BLOCKERS = {"too_simple", "generic_icon", "reference_crop_ignored", "single_object_on_blank_background", "style_drift"} + + +def _front_summary(path: Path) -> bool: + if path.suffix.lower() == ".json": + try: + data = json.loads(path.read_text(encoding="utf-8")) + return isinstance(data, dict) and bool(str(data.get("summary", "")).strip()) + except Exception: + return False + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + text = line.strip() + if not text: + continue + return text.lstrip("#").strip().lower() == "summary" + return False + + +def _is_precise_ratio(value: object) -> bool: + text = str(value or "").strip() + if text in COARSE_RATIO_PRESETS: + return False + parts = text.split(":") + if len(parts) != 2: + return False + try: + return all("." in part and len(part.split(".", 1)[1]) >= 3 and float(part) > 0 for part in parts) + except Exception: + return False + + +def _validate_geometry_item(item: dict, label: str, errors: list[str]) -> None: + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "aspect_ratio_decimal", "aspect_ratio_w_h", "target_pixels_exact"]: + if key not in item: + errors.append(f"{label} missing {key}") + if not _is_precise_ratio(item.get("aspect_ratio_w_h")): + errors.append(f"{label} must use precise decimal aspect_ratio_w_h") + try: + if float(item.get("aspect_ratio_decimal", 0)) <= 0: + errors.append(f"{label} aspect_ratio_decimal must be positive") + except Exception: + errors.append(f"{label} aspect_ratio_decimal must be numeric") + + +def _pixel_pair(value: object) -> tuple[float, float] | None: + if not isinstance(value, dict): + return None + try: + return float(value["width"]), float(value["height"]) + except Exception: + return None + + +def _color_family(hex_color: str) -> str: + text = str(hex_color).strip().lstrip("#") + if len(text) < 6: + return "unknown" + try: + r, g, b = int(text[0:2], 16), int(text[2:4], 16), int(text[4:6], 16) + except Exception: + return "unknown" + if max(r, g, b) - min(r, g, b) < 24: + return "neutral" + if r >= g and r >= b: + return "warm" + if g >= r and g >= b: + return "green" + return "blue" + + +def _arrow_like_asset_id(value: object) -> bool: + text = str(value or "").lower() + return any(term in text for term in ARROW_ASSET_ID_TERMS) + + +def validate_output(out_dir: str | Path) -> dict: + root = Path(out_dir) + errors = [] + warnings = [] + if not root.exists(): + return {"summary": "Framework output validation result.", "ok": False, "errors": [f"Missing output directory: {root}"], "warnings": []} + + for name in REQUIRED: + path = root / name + if not path.exists() or path.stat().st_size == 0: + errors.append(f"Missing non-empty {name}") + elif path.suffix.lower() in {".md", ".json"} and not _front_summary(path): + errors.append(f"{name} must start with Summary or top-level summary") + + try: + manifest = json.loads((root / "input_manifest.json").read_text(encoding="utf-8")) + if not manifest.get("paper_archived"): + errors.append("input_manifest.json missing archived paper path") + if not manifest.get("reference_archived"): + errors.append("input_manifest.json missing archived reference path") + except Exception as exc: + errors.append(f"Invalid input_manifest.json: {exc}") + + try: + geometry = json.loads((root / "reference_geometry.json").read_text(encoding="utf-8")) + if not isinstance(geometry.get("slots"), list) or len(geometry.get("slots", [])) < 25: + errors.append("reference_geometry.json must contain at least 25 slot geometry entries") + if not isinstance(geometry.get("panels"), list) or not geometry.get("panels"): + errors.append("reference_geometry.json must contain panel geometry entries") + for item in geometry.get("slots", []): + if isinstance(item, dict): + _validate_geometry_item(item, f"reference_geometry slot {item.get('id')}", errors) + if _arrow_like_asset_id(item.get("id")): + errors.append(f"reference_geometry slot {item.get('id')} looks like an arrow/control and must be moved to reference_controls.json") + for item in geometry.get("panels", []): + if isinstance(item, dict): + _validate_geometry_item(item, f"reference_geometry panel {item.get('id')}", errors) + for item in geometry.get("controls", []): + if isinstance(item, dict): + _validate_geometry_item(item, f"reference_geometry control {item.get('id')}", errors) + if not str(item.get("source_id", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing source_id") + if not str(item.get("target_id", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing target_id") + if not str(item.get("source_anchor", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing source_anchor") + if not str(item.get("target_anchor", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing target_anchor") + if str(item.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"reference_geometry control {item.get('id')} must render as a PPT shape, not an image asset") + palette = geometry.get("reference_palette", []) + if not isinstance(palette, list) or len({str(c).upper() for c in palette}) < 4: + errors.append("reference_geometry.json reference_palette must contain at least 4 distinct colors") + if not isinstance(geometry.get("color_tokens"), list) or not geometry.get("color_tokens"): + errors.append("reference_geometry.json must contain color_tokens") + except Exception as exc: + errors.append(f"Invalid reference_geometry.json: {exc}") + + try: + candidates_doc = json.loads((root / "reference_control_candidates.json").read_text(encoding="utf-8")) + if not str(candidates_doc.get("effective_mode", "")).strip(): + errors.append("reference_control_candidates.json missing effective_mode") + if not isinstance(candidates_doc.get("candidates"), list): + errors.append("reference_control_candidates.json candidates must be a list") + if str(candidates_doc.get("slot_overlay_path", "")).strip() and not (root / str(candidates_doc.get("slot_overlay_path"))).exists(): + errors.append("reference_control_candidates.json slot_overlay_path does not exist") + if str(candidates_doc.get("control_overlay_path", "")).strip() and not (root / str(candidates_doc.get("control_overlay_path"))).exists(): + errors.append("reference_control_candidates.json control_overlay_path does not exist") + for item in candidates_doc.get("candidates", []) if isinstance(candidates_doc.get("candidates"), list) else []: + cid = item.get("id") + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "path_percent", "editable_in", "render_policy"]: + if key not in item: + errors.append(f"reference_control_candidates item {cid} missing {key}") + if _arrow_like_asset_id(item.get("asset_id")): + errors.append(f"reference_control_candidates item {cid} must not bind to an image asset") + except Exception as exc: + errors.append(f"Invalid reference_control_candidates.json: {exc}") + + try: + controls_doc = json.loads((root / "reference_controls.json").read_text(encoding="utf-8")) + controls = controls_doc.get("controls", []) + if not isinstance(controls, list): + errors.append("reference_controls.json controls must be a list") + for item in controls if isinstance(controls, list) else []: + cid = item.get("id") + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "source_id", "target_id", "source_anchor", "target_anchor", "path_percent", "style_token_id", "editable_in", "render_policy"]: + if key not in item: + errors.append(f"reference_controls item {cid} missing {key}") + if not str(item.get("source_id", "")).strip(): + errors.append(f"reference_controls item {cid} missing non-empty source_id") + if not str(item.get("target_id", "")).strip(): + errors.append(f"reference_controls item {cid} missing non-empty target_id") + if not isinstance(item.get("path_percent"), list) or len(item.get("path_percent", [])) < 2: + errors.append(f"reference_controls item {cid} must have at least 2 path_percent points") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"reference_controls item {cid} must be editable in PPTX") + if str(item.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"reference_controls item {cid} must use ppt_shape_not_image_asset") + except Exception as exc: + errors.append(f"Invalid reference_controls.json: {exc}") + + try: + arrow_style = json.loads((root / "arrow_style_profile.json").read_text(encoding="utf-8")) + if str(arrow_style.get("reference_priority", "")).lower() != "reference_image_hard_constraint": + errors.append("arrow_style_profile.json must keep reference_image_hard_constraint") + if not isinstance(arrow_style.get("style_rules"), dict) or not arrow_style.get("style_rules"): + errors.append("arrow_style_profile.json must contain style_rules") + if "reference" not in str(arrow_style.get("routing_principle", "")).lower(): + errors.append("arrow_style_profile.json routing_principle must explicitly preserve the reference image") + if not str(arrow_style.get("routing_algorithm", "")).strip(): + errors.append("arrow_style_profile.json must record routing_algorithm") + if "fallback" not in str(arrow_style.get("fallback_routing_policy", "")).lower(): + errors.append("arrow_style_profile.json must record fallback_routing_policy") + except Exception as exc: + errors.append(f"Invalid arrow_style_profile.json: {exc}") + + try: + routes_doc = json.loads((root / "selected_arrow_routes.json").read_text(encoding="utf-8")) + routes = routes_doc.get("routes", []) + if not isinstance(routes, list): + errors.append("selected_arrow_routes.json routes must be a list") + for item in routes if isinstance(routes, list) else []: + rid = item.get("id") + for key in ["source_id", "target_id", "semantic_role", "route_style", "path_percent", "style_token_id", "stroke_width_pt", "arrowhead_size", "line_cap", "routing_algorithm", "route_generation_status"]: + if key not in item or not str(item.get(key, "")).strip(): + errors.append(f"selected_arrow_routes item {rid} missing {key}") + if item.get("reference_locked") and item.get("reference_path_preserved") is not True: + errors.append(f"selected_arrow_routes item {rid} overrides a locked reference path") + if not isinstance(item.get("path_percent"), list) or len(item.get("path_percent", [])) < 2: + errors.append(f"selected_arrow_routes item {rid} must have at least 2 path_percent points") + if str(item.get("route_generation_status", "")).lower() == "aesthetic_tunnel_adjusted": + if not isinstance(item.get("reference_original_path_percent"), list) or len(item.get("reference_original_path_percent", [])) < 2: + errors.append(f"selected_arrow_routes item {rid} missing reference_original_path_percent for aesthetic adjustment") + if item.get("reference_tunnel_preserved") is not True: + errors.append(f"selected_arrow_routes item {rid} left its reference tunnel") + except Exception as exc: + errors.append(f"Invalid selected_arrow_routes.json: {exc}") + + try: + arrow_quality = json.loads((root / "arrow_quality_report.json").read_text(encoding="utf-8")) + if str(arrow_quality.get("status", "")).lower() in {"fail", "failed", "blocked"}: + errors.append("arrow_quality_report.json reports failed arrow quality") + if arrow_quality.get("reference_path_overrides"): + errors.append("arrow_quality_report.json contains reference_path_overrides") + if arrow_quality.get("reference_tunnel_violations"): + errors.append("arrow_quality_report.json contains reference_tunnel_violations") + if "total_crossing_count" not in arrow_quality: + errors.append("arrow_quality_report.json missing total_crossing_count") + if "average_aesthetic_score" not in arrow_quality: + errors.append("arrow_quality_report.json missing average_aesthetic_score") + except Exception as exc: + errors.append(f"Invalid arrow_quality_report.json: {exc}") + + try: + style_profile = json.loads((root / "reference_style_profile.json").read_text(encoding="utf-8")) + if not isinstance(style_profile.get("color_tokens"), list) or not style_profile.get("color_tokens"): + errors.append("reference_style_profile.json must contain color_tokens") + for key in ["style_summary", "illustration_style", "line_weight", "shadow_style", "corner_radius", "icon_detail_level", "visual_density", "text_policy", "reference_priority"]: + if not str(style_profile.get(key, "")).strip(): + errors.append(f"reference_style_profile.json missing {key}") + except Exception as exc: + errors.append(f"Invalid reference_style_profile.json: {exc}") + + try: + layout_plan = json.loads((root / "layout_plan.json").read_text(encoding="utf-8")) + if not isinstance(layout_plan.get("slots"), list) or len(layout_plan.get("slots", [])) < 25: + errors.append("layout_plan.json must contain at least 25 located slots") + if not isinstance(layout_plan.get("panels"), list) or not layout_plan.get("panels"): + errors.append("layout_plan.json must contain located panels") + except Exception as exc: + errors.append(f"Invalid layout_plan.json: {exc}") + + try: + program = json.loads((root / "figure_program.json").read_text(encoding="utf-8")) + for key in ["canvas", "panels", "slots", "assets", "labels", "arrows", "groups", "export_targets"]: + if key not in program: + errors.append(f"figure_program.json missing {key}") + slots = program.get("slots", []) + if len(slots) < 25: + errors.append(f"Too few slots: {len(slots)}") + if not any(str(t.get("type", "")).lower() == "pptx" or str(t.get("path", "")).lower().endswith(".pptx") for t in program.get("export_targets", []) if isinstance(t, dict)): + errors.append("figure_program.json export_targets must include pptx") + for slot in slots: + if _arrow_like_asset_id(slot.get("id")) or _arrow_like_asset_id(slot.get("asset_id")): + errors.append(f"Slot {slot.get('id')} looks like an arrow/control and must not be generated as an image asset") + fit_policy = str(slot.get("fit_policy", "")).lower() + if "crop" in fit_policy and "no_crop" not in fit_policy and "without_crop" not in fit_policy: + errors.append(f"Slot {slot.get('id')} uses a crop fit policy") + if not _is_precise_ratio(slot.get("target_canvas_ratio")): + errors.append(f"Slot {slot.get('id')} uses a coarse or non-decimal target_canvas_ratio") + for key in ["center_percent", "width_percent", "height_percent", "aspect_ratio_decimal", "aspect_ratio_w_h", "target_pixels", "target_pixels_exact", "generation_min_pixels"]: + if key not in slot: + errors.append(f"Slot {slot.get('id')} missing precise geometry key {key}") + target_pixels = _pixel_pair(slot.get("target_pixels")) + exact_pixels = _pixel_pair(slot.get("target_pixels_exact")) + if target_pixels and exact_pixels: + if abs(target_pixels[0] - exact_pixels[0]) > 0.01 or abs(target_pixels[1] - exact_pixels[1]) > 0.01: + errors.append(f"Slot {slot.get('id')} target_pixels must equal target_pixels_exact; use generation_min_pixels for minimum generation size") + if float(slot.get("min_content_fill_percent", 0)) < 85: + errors.append(f"Slot {slot.get('id')} has min fill below 85") + if float(slot.get("max_empty_margin_percent", 99)) > 10: + errors.append(f"Slot {slot.get('id')} allows too much empty margin") + if str(slot.get("slot_frame_policy", "")).lower() != "frameless_slot": + errors.append(f"Slot {slot.get('id')} must use frameless_slot") + crop_path = str(slot.get("reference_crop_path", "")).strip() + if not crop_path: + errors.append(f"Slot {slot.get('id')} missing reference_crop_path") + elif not (root / crop_path).exists(): + errors.append(f"Slot {slot.get('id')} reference crop does not exist: {crop_path}") + if not str(slot.get("reference_style_profile_path", "")).strip(): + errors.append(f"Slot {slot.get('id')} missing reference_style_profile_path") + if not isinstance(slot.get("local_color_token_ids"), list) or not slot.get("local_color_token_ids"): + errors.append(f"Slot {slot.get('id')} missing local_color_token_ids") + for key in ["visual_spec_id", "complexity_profile", "complexity_kind", "foreground_subject", "secondary_objects", "micro_details", "background_fill_elements", "scientific_mechanism_detail", "required_visual_complexity", "forbidden_simplification"]: + if key not in slot: + errors.append(f"Slot {slot.get('id')} missing visual complexity key {key}") + if str(slot.get("complexity_kind", "")).lower() != "legend_icon": + if not isinstance(slot.get("secondary_objects"), list) or len(slot.get("secondary_objects", [])) < 2: + errors.append(f"Slot {slot.get('id')} needs at least 2 secondary_objects") + if not isinstance(slot.get("micro_details"), list) or len(slot.get("micro_details", [])) < 2: + errors.append(f"Slot {slot.get('id')} needs at least 2 micro_details") + safe_area = float(slot.get("safe_area_percent", 0)) + if safe_area < 88: + errors.append(f"Slot {slot.get('id')} safe area below 88") + for asset in program.get("assets", []): + if _arrow_like_asset_id(asset.get("id")) or _arrow_like_asset_id(asset.get("slot_id")): + errors.append(f"Asset {asset.get('id')} looks like an arrow/control and must not be generated by image2") + if not str(asset.get("reference_crop_path", "")).strip(): + errors.append(f"Asset {asset.get('id')} missing reference_crop_path") + if not str(asset.get("visual_spec_id", "")).strip(): + errors.append(f"Asset {asset.get('id')} missing visual_spec_id") + for key in ["panels", "labels", "arrows", "groups"]: + for item in program.get(key, []): + if isinstance(item, dict) and str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"{key} item {item.get('id')} is not marked editable in PPTX") + for arrow in program.get("arrows", []): + if not str(arrow.get("source_id") or arrow.get("source") or "").strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing source_id") + if not str(arrow.get("target_id") or arrow.get("target") or "").strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing target_id") + if not isinstance(arrow.get("path_percent"), list) or len(arrow.get("path_percent", [])) < 2: + errors.append(f"Arrow/control {arrow.get('id')} must have at least 2 path_percent points") + if not str(arrow.get("source_anchor", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing source_anchor") + if not str(arrow.get("target_anchor", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing target_anchor") + if not str(arrow.get("style_token_id", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing style_token_id") + if str(arrow.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"Arrow/control {arrow.get('id')} must render as PPT shape, not image asset") + if not str(arrow.get("semantic_role", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing semantic_role") + if not str(arrow.get("route_style", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing route_style") + if not str(arrow.get("routing_algorithm", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing routing_algorithm") + if not str(arrow.get("route_generation_status", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing route_generation_status") + if arrow.get("reference_locked") and arrow.get("reference_path_preserved") is not True: + errors.append(f"Arrow/control {arrow.get('id')} overrides a locked reference path") + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + if not isinstance(style.get("color_tokens"), list) or not style.get("color_tokens"): + errors.append("figure_program.json style must include color_tokens") + if not str(style.get("reference_style_profile_path", "")).strip(): + errors.append("figure_program.json style must include reference_style_profile_path") + if not str(style.get("arrow_style_profile_path", "")).strip(): + errors.append("figure_program.json style must include arrow_style_profile_path") + if not str(program.get("text_program_path", "")).strip(): + errors.append("figure_program.json must include text_program_path") + if not str(program.get("reference_text_geometry_path", "")).strip(): + errors.append("figure_program.json must include reference_text_geometry_path") + if not str(program.get("text_alignment_report_path", "")).strip(): + errors.append("figure_program.json must include text_alignment_report_path") + text_program_inline = program.get("text_program") + if not isinstance(text_program_inline, dict) or not isinstance(text_program_inline.get("items"), list) or not text_program_inline.get("items"): + errors.append("figure_program.json must embed non-empty text_program items for PPT text rendering") + palette = style.get("reference_palette") or style.get("palette") or [] + distinct = {str(c).upper() for c in palette if str(c).strip()} + if len(distinct) < 4: + errors.append("figure_program.json style must preserve reference-derived palette tokens, not a hardcoded fallback template") + token_ids = {str(item.get("token_id")) for item in style.get("color_tokens", []) if isinstance(item, dict)} + for arrow in program.get("arrows", []): + if str(arrow.get("style_token_id")) not in token_ids: + errors.append(f"Arrow/control {arrow.get('id')} style_token_id is not present in reference color_tokens") + except Exception as exc: + errors.append(f"Invalid figure_program.json: {exc}") + + reference_text_ids: set[str] = set() + try: + text_geometry = json.loads((root / "reference_text_geometry.json").read_text(encoding="utf-8")) + if "publication" in str(text_geometry.get("policy", "")).lower() and "reference" not in str(text_geometry.get("policy", "")).lower(): + errors.append("reference_text_geometry.json policy must be reference-first, not publication-scale-first") + regions = text_geometry.get("text_regions", []) + if not isinstance(regions, list) or not regions: + errors.append("reference_text_geometry.json must contain non-empty text_regions") + for item in regions if isinstance(regions, list) else []: + rid = str(item.get("id") or "") + reference_text_ids.add(rid) + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "estimated_font_ratio", "color_hex", "role", "target_id"]: + if key not in item: + errors.append(f"reference_text_geometry item {rid} missing {key}") + try: + if float(item.get("width_percent", 0)) <= 0 or float(item.get("height_percent", 0)) <= 0: + errors.append(f"reference_text_geometry item {rid} has non-positive size") + if float(item.get("estimated_font_ratio", 0)) <= 0: + errors.append(f"reference_text_geometry item {rid} has non-positive estimated_font_ratio") + except Exception: + errors.append(f"reference_text_geometry item {rid} size/font fields must be numeric") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"reference_text_geometry item {rid} must be intended for editable PPTX text") + color = str(item.get("color_hex", "")) + if not color.startswith("#") or len(color.strip()) < 7: + errors.append(f"reference_text_geometry item {rid} must record reference color_hex") + if str(text_geometry.get("detection_mode", "")).lower() == "ocr": + if "confidence" not in item: + errors.append(f"reference_text_geometry OCR item {rid} missing confidence") + if not str(item.get("raw_text", "")).strip(): + errors.append(f"reference_text_geometry OCR item {rid} missing raw_text") + except Exception as exc: + errors.append(f"Invalid reference_text_geometry.json: {exc}") + + try: + ocr_report = json.loads((root / "ocr_text_quality_report.json").read_text(encoding="utf-8")) + if not str(ocr_report.get("mode", "")).strip(): + errors.append("ocr_text_quality_report.json missing mode") + if not str(ocr_report.get("status", "")).strip(): + errors.append("ocr_text_quality_report.json missing status") + if str(ocr_report.get("status", "")).lower() not in {"pass", "fallback", "not_run"}: + errors.append("ocr_text_quality_report.json status must be pass, fallback, or not_run") + if "text_region_count" not in ocr_report: + errors.append("ocr_text_quality_report.json missing text_region_count") + except Exception as exc: + errors.append(f"Invalid ocr_text_quality_report.json: {exc}") + + try: + text_program = json.loads((root / "text_program.json").read_text(encoding="utf-8")) + policy = str(text_program.get("policy", "")).lower() + if "paper_double_column" in policy or "minimum_effective_font" in policy: + errors.append("text_program.json must not apply fixed publication-scale font thresholds by default") + items = text_program.get("items", []) + if not isinstance(items, list) or not items: + errors.append("text_program.json must contain non-empty items") + for item in items if isinstance(items, list) else []: + tid = str(item.get("id") or "") + for key in ["text", "role", "target_id", "source_reference_text_id", "reference_binding", "bbox_percent", "center_percent", "width_percent", "height_percent", "estimated_font_ratio", "font_size_pt", "color_hex", "editable_in"]: + if key not in item: + errors.append(f"text_program item {tid} missing {key}") + for key in ["font_family_guess", "font_weight_guess", "fit_strategy"]: + if key not in item: + errors.append(f"text_program item {tid} missing {key}") + source_id = str(item.get("source_reference_text_id") or "") + if source_id not in reference_text_ids: + errors.append(f"text_program item {tid} source_reference_text_id does not exist in reference_text_geometry.json") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"text_program item {tid} must render as editable PPTX text") + try: + if float(item.get("font_size_pt", 0)) <= 0: + errors.append(f"text_program item {tid} font_size_pt must be positive") + if float(item.get("estimated_font_ratio", 0)) <= 0: + errors.append(f"text_program item {tid} estimated_font_ratio must be positive") + except Exception: + errors.append(f"text_program item {tid} font fields must be numeric") + binding = str(item.get("reference_binding", "")).lower() + if "reference" not in binding: + errors.append(f"text_program item {tid} must bind to a reference-derived text region") + except Exception as exc: + errors.append(f"Invalid text_program.json: {exc}") + + try: + text_alignment = json.loads((root / "text_alignment_report.json").read_text(encoding="utf-8")) + policy = str(text_alignment.get("policy", "")).lower() + if "fixed publication" in policy or "paper_double_column" in policy: + errors.append("text_alignment_report.json must check reference alignment, not fixed publication font thresholds") + if str(text_alignment.get("status", "")).lower() in {"fail", "failed", "blocked"}: + errors.append("text_alignment_report.json reports failed text alignment") + items = text_alignment.get("items", []) + if not isinstance(items, list) or not items: + errors.append("text_alignment_report.json must contain non-empty items") + for item in items if isinstance(items, list) else []: + tid = str(item.get("label_id") or "") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"text_alignment_report item {tid} is not editable in PPTX") + if str(item.get("status", "")).lower() in {"fail", "failed", "blocked"}: + errors.append(f"text_alignment_report item {tid} failed reference alignment") + try: + if float(item.get("center_delta_percent", 0)) > 0.05: + errors.append(f"text_alignment_report item {tid} center drift exceeds reference tolerance") + if float(item.get("width_delta_percent", 0)) > 0.08: + errors.append(f"text_alignment_report item {tid} width drift exceeds reference tolerance") + if float(item.get("height_delta_percent", 0)) > 0.06: + errors.append(f"text_alignment_report item {tid} height drift exceeds reference tolerance") + except Exception: + errors.append(f"text_alignment_report item {tid} delta fields must be numeric") + if not str(item.get("source_reference_text_id", "")).strip(): + errors.append(f"text_alignment_report item {tid} missing source_reference_text_id") + except Exception as exc: + errors.append(f"Invalid text_alignment_report.json: {exc}") + + try: + visual_spec = json.loads((root / "slot_visual_spec.json").read_text(encoding="utf-8")) + items = visual_spec.get("slots", []) + if not isinstance(items, list) or len(items) < 25: + errors.append("slot_visual_spec.json must contain at least 25 slot entries") + for item in items: + sid = item.get("slot_id") + for key in ["reference_crop_objects", "foreground_subject", "secondary_objects", "micro_details", "background_fill_elements", "scientific_mechanism_detail", "required_visual_complexity", "forbidden_simplification"]: + if key not in item: + errors.append(f"slot_visual_spec item {sid} missing {key}") + if str(item.get("complexity_kind", "")).lower() != "legend_icon": + if not isinstance(item.get("secondary_objects"), list) or len(item.get("secondary_objects", [])) < 2: + errors.append(f"slot_visual_spec item {sid} needs at least 2 secondary_objects") + if not isinstance(item.get("micro_details"), list) or len(item.get("micro_details", [])) < 2: + errors.append(f"slot_visual_spec item {sid} needs at least 2 micro_details") + forbidden_items = item.get("forbidden_simplification", []) if isinstance(item.get("forbidden_simplification"), list) else [] + forbidden = " ".join(str(x).lower() for x in forbidden_items) + if str(item.get("complexity_kind", "")).lower() != "legend_icon" and "simple icon" not in forbidden: + errors.append(f"slot_visual_spec item {sid} must forbid simple icon simplification") + except Exception as exc: + errors.append(f"Invalid slot_visual_spec.json: {exc}") + + try: + prompt_brief = json.loads((root / "reference_slot_prompt_brief.json").read_text(encoding="utf-8")) + brief_items = prompt_brief.get("slots", []) + if not isinstance(brief_items, list) or len(brief_items) < 25: + errors.append("reference_slot_prompt_brief.json must contain at least 25 slot brief entries") + for item in brief_items: + sid = item.get("slot_id") + if not str(item.get("slot_function", "")).strip(): + errors.append(f"reference_slot_prompt_brief item {sid} missing slot_function") + crop_path = str(item.get("reference_crop_path", "")).strip() + if not crop_path: + errors.append(f"reference_slot_prompt_brief item {sid} missing reference_crop_path") + elif not (root / crop_path).exists(): + errors.append(f"reference_slot_prompt_brief item {sid} crop file missing: {crop_path}") + if not str(item.get("reference_style_profile_path", "")).strip(): + errors.append(f"reference_slot_prompt_brief item {sid} missing reference_style_profile_path") + except Exception as exc: + errors.append(f"Invalid reference_slot_prompt_brief.json: {exc}") + + try: + prompt_plan = json.loads((root / "slot_prompt_plan.json").read_text(encoding="utf-8")) + items = prompt_plan.get("slots", []) + if not isinstance(items, list) or len(items) < 25: + errors.append("slot_prompt_plan.json must contain at least 25 slot prompt plans") + for item in items: + sid = item.get("slot_id") + for key in ["slot_function", "reference_slot_role", "reference_shape_language", "reference_local_style", "reference_prompt_hint", "visual_metaphor", "image_prompt_core"]: + if not str(item.get(key, "")).strip(): + errors.append(f"slot_prompt_plan item {sid} missing {key}") + for key in ["center_percent", "width_percent", "height_percent", "aspect_ratio_decimal", "aspect_ratio_w_h", "target_canvas_ratio", "target_pixels", "target_pixels_exact", "generation_min_pixels"]: + if key not in item: + errors.append(f"slot_prompt_plan item {sid} missing precise geometry key {key}") + crop_path = str(item.get("reference_crop_path", "")).strip() + if not crop_path: + errors.append(f"slot_prompt_plan item {sid} missing reference_crop_path") + elif not (root / crop_path).exists(): + errors.append(f"slot_prompt_plan item {sid} crop file missing: {crop_path}") + if not item.get("reference_crop_used"): + errors.append(f"slot_prompt_plan item {sid} must record reference_crop_used=true") + if not str(item.get("reference_style_profile_path", "")).strip(): + errors.append(f"slot_prompt_plan item {sid} missing reference_style_profile_path") + if not isinstance(item.get("local_color_token_ids"), list) or not item.get("local_color_token_ids"): + errors.append(f"slot_prompt_plan item {sid} missing local_color_token_ids") + for key in ["visual_spec_id", "complexity_kind", "foreground_subject", "secondary_objects", "micro_details", "background_fill_elements", "scientific_mechanism_detail", "required_visual_complexity", "forbidden_simplification"]: + if key not in item: + errors.append(f"slot_prompt_plan item {sid} missing visual complexity key {key}") + if not _is_precise_ratio(item.get("target_canvas_ratio")): + errors.append(f"slot_prompt_plan item {sid} uses coarse target_canvas_ratio") + core = str(item.get("image_prompt_core", "")).lower() + if "90-97" not in core or "10%" not in core: + errors.append(f"slot_prompt_plan item {sid} image_prompt_core must mention 90-97% fill and 10% margin") + if "reference_style_profile" not in core or "crop" not in core: + errors.append(f"slot_prompt_plan item {sid} image_prompt_core must mention reference_style_profile and local crop grounding") + if str(item.get("complexity_kind", "")).lower() != "legend_icon": + for phrase in ["dense mini scientific", "2-5", "standalone pictogram"]: + if phrase not in core: + errors.append(f"slot_prompt_plan item {sid} image_prompt_core must mention {phrase}") + if not isinstance(item.get("must_show"), list) or not item.get("must_show"): + errors.append(f"slot_prompt_plan item {sid} missing must_show") + except Exception as exc: + errors.append(f"Invalid slot_prompt_plan.json: {exc}") + + try: + report = json.loads((root / "asset_quality_report.json").read_text(encoding="utf-8")) + items = report.get("assets", []) + if len(items) < 25: + errors.append(f"Too few asset QA entries: {len(items)}") + for item in items: + aid = item.get("asset_id") or item.get("slot_id") + if "selected_candidate_index" not in item: + errors.append(f"Asset {aid} missing selected_candidate_index") + fill = float(item.get("content_fill_percent", 0)) + min_fill = float(item.get("min_content_fill_percent", 85)) + margin = float(item.get("empty_margin_percent", 100)) + max_margin = float(item.get("max_empty_margin_percent", 10)) + if fill < min_fill: + errors.append(f"Asset {aid} fill {fill} below {min_fill}") + if margin > max_margin: + errors.append(f"Asset {aid} margin {margin} above {max_margin}") + if str(item.get("edge_cutoff_status", "")).lower() not in {"ok", "none", "complete", "no_cutoff"}: + errors.append(f"Asset {aid} edge cutoff status is not ok") + except Exception as exc: + errors.append(f"Invalid asset_quality_report.json: {exc}") + + try: + complexity = json.loads((root / "asset_complexity_report.json").read_text(encoding="utf-8")) + items = complexity.get("assets", []) + if len(items) < 25: + errors.append(f"Too few asset complexity entries: {len(items)}") + for item in items: + aid = item.get("asset_id") or item.get("slot_id") + for key in ["detail_score", "detail_score_target", "object_count_estimate", "object_count_target", "simple_icon_risk", "reference_crop_match", "style_match", "selected_reason"]: + if key not in item: + errors.append(f"Asset complexity {aid} missing {key}") + try: + if float(item.get("detail_score", 0)) < float(item.get("detail_score_target", 0)): + errors.append(f"Asset complexity {aid} detail_score below target") + except Exception: + errors.append(f"Asset complexity {aid} detail_score fields must be numeric") + if item.get("simple_icon_risk") is True: + errors.append(f"Asset complexity {aid} has simple_icon_risk=true") + tags = item.get("complexity_issue_tags", []) if isinstance(item.get("complexity_issue_tags"), list) else [] + unresolved = sorted(COMPLEXITY_BLOCKERS.intersection(str(tag) for tag in tags)) + if unresolved: + errors.append(f"Asset complexity {aid} has unresolved complexity issue(s): {', '.join(unresolved)}") + if str(item.get("reference_crop_match", "")).startswith("missing"): + errors.append(f"Asset complexity {aid} missing reference crop match") + if str(item.get("style_match", "")).startswith("missing"): + errors.append(f"Asset complexity {aid} missing style match") + except Exception as exc: + errors.append(f"Invalid asset_complexity_report.json: {exc}") + + try: + comp = json.loads((root / "composition_quality_report.json").read_text(encoding="utf-8")) + items = comp.get("slots", []) + if not isinstance(items, list) or len(items) < 25: + errors.append("composition_quality_report.json must contain at least 25 slot entries") + for item in items: + sid = item.get("slot_id") + if item.get("tile_frame_added") is not False: + errors.append(f"Composition slot {sid} added an extra tile frame") + if item.get("caption_inside_image_slot") is not False: + errors.append(f"Composition slot {sid} has caption inside image slot") + if str(item.get("slot_frame_policy", "")).lower() != "frameless_slot": + errors.append(f"Composition slot {sid} must use frameless_slot") + fill = float(item.get("image_slot_area_fill_percent", 0)) + if fill < 95: + errors.append(f"Composition slot {sid} image fills {fill}% of slot, below 95%") + for item in comp.get("arrows", []) if isinstance(comp.get("arrows"), list) else []: + aid = item.get("arrow_id") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"Composition arrow {aid} is not editable in PPTX") + if str(item.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"Composition arrow {aid} must render as a PPT shape") + if int(item.get("segment_count", 0)) < 1: + errors.append(f"Composition arrow {aid} rendered no connector segments") + if not str(item.get("route_style", "")).strip(): + errors.append(f"Composition arrow {aid} missing route_style") + if not str(item.get("line_cap", "")).strip(): + errors.append(f"Composition arrow {aid} missing line_cap") + if not str(item.get("routing_algorithm", "")).strip(): + errors.append(f"Composition arrow {aid} missing routing_algorithm") + except Exception as exc: + errors.append(f"Invalid composition_quality_report.json: {exc}") + + try: + asset_review = json.loads((root / "asset_visual_review.json").read_text(encoding="utf-8")) + if str(asset_review.get("status", "")).lower() in {"needs_regeneration", "blocked", "fail", "failed"}: + issues = asset_review.get("issues", []) + errors.append(f"asset_visual_review.json has unresolved asset issues: {len(issues)}") + except Exception as exc: + errors.append(f"Invalid asset_visual_review.json: {exc}") + + try: + visual_critic = json.loads((root / "visual_critic_iter_0.json").read_text(encoding="utf-8")) + if str(visual_critic.get("status", "")).lower() in {"blocked", "needs_layout_refinement", "fail", "failed"}: + errors.append("visual_critic_iter_0.json reports unresolved layout or blocking issues") + if visual_critic.get("blocking_issues"): + errors.append(f"visual_critic_iter_0.json has blocking issues: {len(visual_critic.get('blocking_issues', []))}") + except Exception as exc: + errors.append(f"Invalid visual_critic_iter_0.json: {exc}") + + asset_count = len([p for p in (root / "assets").glob("*") if p.suffix.lower() in IMAGE_EXTS]) if (root / "assets").exists() else 0 + if asset_count < 25: + errors.append(f"Too few generated assets: {asset_count}") + + text_blob = "" + for name in ["prompts.md", "style_sheet.md", "reference_style_profile.json", "arrow_style_profile.json", "selected_arrow_routes.json", "arrow_quality_report.json", "figure_program.json", "reference_geometry.json", "reference_controls.json", "reference_text_geometry.json", "text_program.json", "ocr_text_quality_report.json", "text_alignment_report.json", "slot_visual_spec.json", "reference_slot_prompt_brief.json", "slot_prompt_plan.json", "asset_quality_report.json", "asset_complexity_report.json", "composition_quality_report.json", "asset_visual_review.json", "alignment_review.md", "critic_report.md", "visual_critic_iter_0.json"]: + path = root / name + if path.exists(): + text_blob += "\n" + path.read_text(encoding="utf-8", errors="ignore").lower() + hits = [m for m in FORBIDDEN_MARKERS if m in text_blob] + if hits: + errors.append("Forbidden marker(s) found: " + ", ".join(hits)) + + if not (root / "review.pdf").exists(): + warnings.append("review.pdf missing; PPTX may still be editable but export failed") + if not (root / "final_600dpi.png").exists(): + warnings.append("final_600dpi.png missing; PDF-to-PNG export may have failed") + + return {"summary": "Framework output validation result.", "ok": not errors, "errors": errors, "warnings": warnings, "asset_count": asset_count} diff --git a/rfs/evolution_contracts.py b/rfs/evolution_contracts.py new file mode 100644 index 0000000..f36f5c8 --- /dev/null +++ b/rfs/evolution_contracts.py @@ -0,0 +1,162 @@ +"""Cross-stage contracts for the Paperimage evolution pipeline. + +This module intentionally contains deterministic, inspectable checks only. A +trained Judge may later add scores and advice, but cannot replace the semantic +grounding gate maintained here. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .utils import write_json + + +def _items(value: object) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def _spec(contract: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(contract, dict): + return {} + value = contract.get("figure_specification") + return value if isinstance(value, dict) else contract + + +def _entities(contract: dict[str, Any] | None) -> list[dict[str, Any]]: + spec = _spec(contract) + direct = _items(spec.get("entities")) or _items(spec.get("modules")) + if direct: + return direct + entities: list[dict[str, Any]] = [] + for panel in _items(spec.get("panel_graphs")): + entities.extend(_items(panel.get("entities")) or _items(panel.get("modules"))) + return entities + + +def _relations(contract: dict[str, Any] | None) -> list[dict[str, Any]]: + spec = _spec(contract) + direct = _items(spec.get("relations")) + if direct: + return direct + relations: list[dict[str, Any]] = [] + for panel in _items(spec.get("panel_graphs")): + relations.extend(_items(panel.get("relations"))) + return relations + + +def build_scientific_ground_truth(contract: dict[str, Any] | None) -> dict[str, Any]: + """Normalize an existing paper contract into a Gate-friendly snapshot.""" + entities = _entities(contract) + relations = _relations(contract) + must_show = [] + normalized_entities = [] + for index, entity in enumerate(entities, start=1): + entity_id = str(entity.get("id") or f"entity_{index:02d}") + label = str(entity.get("label") or entity.get("name") or entity_id) + required = bool(entity.get("must_show", True)) + if required: + must_show.append(entity_id) + normalized_entities.append({ + "id": entity_id, + "label": label, + "must_show": required, + "evidence_ids": list(entity.get("evidence_ids") or []), + }) + return { + "summary": "Normalized scientific Ground Truth derived from the paper semantic contract.", + "schema_version": "1.0", + "entities": normalized_entities, + "relations": [{ + "id": str(item.get("id") or f"relation_{index:02d}"), + "source": str(item.get("source") or item.get("source_id") or ""), + "target": str(item.get("target") or item.get("target_id") or ""), + "type": str(item.get("type") or item.get("relation_type") or "data_flow"), + "evidence_ids": list(item.get("evidence_ids") or []), + } for index, item in enumerate(relations, start=1)], + "must_show": must_show, + "must_not_invent": list(_spec(contract).get("must_not_invent") or []), + "terminology": list(_spec(contract).get("terminology") or []), + "source": "paper_semantic_contract" if contract else "reference_only", + } + + +def build_scientific_grounding_report( + ground_truth: dict[str, Any], semantic_binding: dict[str, Any] | None, +) -> dict[str, Any]: + """Apply a deterministic semantic gate to the PPT semantic bindings.""" + if ground_truth.get("source") == "reference_only": + return { + "summary": "Scientific grounding skipped: reference-only rebuild has no paper contract.", + "status": "warning", + "gate_pass": True, + "judge_role": "bootstrap", + "trained": False, + "blocking_issues": [], + "missing_required_entities": [], + "incorrect_relations": [], + } + binding = semantic_binding if isinstance(semantic_binding, dict) else {} + mapped_entities = int(binding.get("mapped_entity_count") or 0) + entities = ground_truth.get("entities") or [] + skipped = binding.get("skipped_relations") or [] + missing = [] + if mapped_entities < len(entities): + mapped_ids = {str(item.get("entity_id") or item.get("semantic_entity_id") or "") for item in binding.get("mappings", []) if isinstance(item, dict) and float(item.get("match_score") or 0) > 0} + missing = [str(item.get("id")) for item in entities if str(item.get("id")) not in mapped_ids] + blockers = [] + if missing: + blockers.append("missing_required_entities") + if skipped: + blockers.append("unmapped_required_relations") + return { + "summary": "Deterministic scientific grounding gate for editable rebuild.", + "status": "pass" if not blockers else "fail", + "gate_pass": not blockers, + "judge_role": "bootstrap", + "trained": False, + "grounded_entities": [item.get("semantic_entity_id") for item in binding.get("mappings", []) if isinstance(item, dict)], + "missing_required_entities": missing, + "incorrect_relations": skipped, + "terminology_errors": [], + "blocking_issues": blockers, + } + + +def build_joint_judgement( + grounding: dict[str, Any], segmentation_validation: dict[str, Any], visual_quality: dict[str, Any], + program: dict[str, Any], +) -> dict[str, Any]: + """Build a model-independent joint evaluation artifact for later Judge training.""" + editability = { + "text_items": len((program.get("text_program") or {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, + "connectors": len(program.get("arrows", [])), + "slots": len(program.get("slots", [])), + } + blockers = list(grounding.get("blocking_issues") or []) + return { + "summary": "Bootstrap joint judgement for scientific preservation, segmentation, and editable rebuild quality.", + "judge_role": "bootstrap", + "trained": False, + "scientific": grounding, + "segmentation": segmentation_validation, + "visual": {"status": visual_quality.get("status"), "report": "rebuild_visual_quality_report.json"}, + "editability": editability, + "overall_status": "pass" if not blockers and segmentation_validation.get("status") == "pass" else "warning", + "blocking_issues": blockers, + } + + +def write_pipeline_contracts( + out_dir: str | Path, contract: dict[str, Any] | None, semantic_binding: dict[str, Any] | None, + segmentation_plan: dict[str, Any], segmentation_validation: dict[str, Any], visual_quality: dict[str, Any], program: dict[str, Any], +) -> dict[str, Any]: + out = Path(out_dir) + ground_truth = build_scientific_ground_truth(contract) + grounding = build_scientific_grounding_report(ground_truth, semantic_binding) + judgement = build_joint_judgement(grounding, segmentation_validation, visual_quality, program) + write_json(out / "scientific_ground_truth.json", ground_truth) + write_json(out / "scientific_grounding_report.json", grounding) + write_json(out / "joint_judgement.json", judgement) + return {"ground_truth": ground_truth, "grounding": grounding, "judgement": judgement} diff --git a/rfs/exporter.py b/rfs/exporter.py index 8b45444..f6a0ac7 100644 --- a/rfs/exporter.py +++ b/rfs/exporter.py @@ -1,27 +1,145 @@ -from __future__ import annotations +from __future__ import annotations +import multiprocessing import os from pathlib import Path +import queue +import signal +import time -def _export_pdf_with_powerpoint(pptx_path: Path, pdf_path: Path) -> bool: - if os.name != "nt": - return False +def _powerpoint_export_worker(pptx_path: str, pdf_path: str, png_path: str, dpi: int, result_queue) -> None: + powerpoint = None + presentation = None try: import win32com.client # type: ignore - powerpoint = win32com.client.Dispatch("PowerPoint.Application") - presentation = powerpoint.Presentations.Open(str(pptx_path), WithWindow=False) + import win32process # type: ignore + + powerpoint = win32com.client.DispatchEx("PowerPoint.Application") + try: + _thread_id, process_id = win32process.GetWindowThreadProcessId(int(powerpoint.HWND)) + result_queue.put({"event": "powerpoint_pid", "pid": int(process_id)}) + except Exception: + pass + presentation = powerpoint.Presentations.Open(str(pptx_path), ReadOnly=True, Untitled=False, WithWindow=False) presentation.SaveAs(str(pdf_path), 32) - presentation.Close() - powerpoint.Quit() - return pdf_path.exists() + width_px = max(1200, int(float(presentation.PageSetup.SlideWidth) / 72.0 * dpi)) + height_px = max(700, int(float(presentation.PageSetup.SlideHeight) / 72.0 * dpi)) + presentation.Slides(1).Export(str(png_path), "PNG", width_px, height_px) + result_queue.put({ + "event": "complete", + "pdf_ok": Path(pdf_path).exists(), + "png_ok": Path(png_path).exists(), + "width_px": width_px, + "height_px": height_px, + }) + except Exception as exc: + result_queue.put({"event": "error", "error": f"{type(exc).__name__}: {exc}"}) + finally: + try: + if presentation is not None: + presentation.Close() + except Exception: + pass + try: + if powerpoint is not None: + powerpoint.Quit() + except Exception: + pass + + +def _export_with_powerpoint( + pptx_path: Path, + pdf_path: Path, + png_path: Path, + *, + dpi: int = 600, + timeout_seconds: float = 60.0, +) -> dict: + if os.name != "nt": + return {"status": "unavailable", "pdf_ok": False, "png_ok": False, "error": "PowerPoint export requires Windows"} + started = time.monotonic() + context = multiprocessing.get_context("spawn") + result_queue = context.Queue() + process = context.Process( + target=_powerpoint_export_worker, + args=(str(pptx_path.resolve()), str(pdf_path.resolve()), str(png_path.resolve()), int(dpi), result_queue), + daemon=True, + ) + powerpoint_pid = None + try: + process.start() + deadline = time.monotonic() + max(5.0, float(timeout_seconds)) + terminal = None + while process.is_alive() and time.monotonic() < deadline: + try: + message = result_queue.get(timeout=0.25) + except queue.Empty: + continue + if message.get("event") == "powerpoint_pid": + powerpoint_pid = message.get("pid") + elif message.get("event") in {"complete", "error"}: + terminal = message + process.join(timeout=1.0) + while terminal is None: + try: + message = result_queue.get_nowait() + except queue.Empty: + break + if message.get("event") == "powerpoint_pid": + powerpoint_pid = message.get("pid") + elif message.get("event") in {"complete", "error"}: + terminal = message + if process.is_alive(): + process.terminate() + process.join(timeout=2.0) + if powerpoint_pid: + try: + os.kill(int(powerpoint_pid), signal.SIGTERM) + except Exception: + pass + return { + "status": "timeout", + "pdf_ok": pdf_path.exists(), + "png_ok": png_path.exists(), + "error": f"PowerPoint export exceeded {max(5.0, float(timeout_seconds)):.1f}s", + "elapsed_seconds": round(time.monotonic() - started, 3), + } + if terminal and terminal.get("event") == "complete": + return {**terminal, "status": "ok", "elapsed_seconds": round(time.monotonic() - started, 3)} + return { + "status": "failed", + "pdf_ok": pdf_path.exists(), + "png_ok": png_path.exists(), + "error": (terminal or {}).get("error") or f"PowerPoint export worker exited with code {process.exitcode}", + "elapsed_seconds": round(time.monotonic() - started, 3), + } + except Exception as exc: + if process.is_alive(): + process.terminate() + return { + "status": "failed", + "pdf_ok": pdf_path.exists(), + "png_ok": png_path.exists(), + "error": f"{type(exc).__name__}: {exc}", + "elapsed_seconds": round(time.monotonic() - started, 3), + } + + +def _export_pdf_with_powerpoint(pptx_path: Path, pdf_path: Path) -> bool: + png_path = pdf_path.with_suffix(".preview.png") + result = _export_with_powerpoint(pptx_path, pdf_path, png_path, dpi=144, timeout_seconds=60) + try: + png_path.unlink(missing_ok=True) except Exception: - return False + pass + return bool(result.get("pdf_ok")) def _render_pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 600) -> bool: try: import fitz + doc = fitz.open(str(pdf_path)) page = doc.load_page(0) scale = dpi / 72.0 @@ -39,8 +157,16 @@ def export_outputs(pptx_path: str | Path, out_dir: str | Path) -> dict: review_pdf = out / "review.pdf" final_png = out / "final_600dpi.png" - if not _export_pdf_with_powerpoint(pptx, review_pdf): - return {"pdf": None, "png": None, "status": "pptx_only_export_failed"} - - png_ok = _render_pdf_to_png(review_pdf, final_png, dpi=600) - return {"pdf": str(review_pdf) if review_pdf.exists() else None, "png": str(final_png) if png_ok else None, "status": "ok" if png_ok else "pdf_only"} + timeout_seconds = float(os.getenv("RFS_POWERPOINT_EXPORT_TIMEOUT", "60") or 60) + result = _export_with_powerpoint(pptx, review_pdf, final_png, dpi=600, timeout_seconds=timeout_seconds) + png_ok = bool(result.get("png_ok")) + if not png_ok and review_pdf.exists(): + png_ok = _render_pdf_to_png(review_pdf, final_png, dpi=600) + pdf_ok = review_pdf.exists() + status = "ok" if pdf_ok and png_ok else "pdf_only" if pdf_ok else "png_only" if png_ok else f"pptx_only_export_{result.get('status', 'failed')}" + return { + "pdf": str(review_pdf) if pdf_ok else None, + "png": str(final_png) if png_ok else None, + "status": status, + "powerpoint_export": result, + } diff --git a/rfs/generation/__init__.py b/rfs/generation/__init__.py new file mode 100644 index 0000000..c1b1fe2 --- /dev/null +++ b/rfs/generation/__init__.py @@ -0,0 +1,15 @@ +"""Raster reference and slot-asset generation entry points.""" + +from .assets import build_slot_prompt, generate_assets + +__all__ = ["build_slot_prompt", "generate_and_select", "generate_assets"] + + +def __getattr__(name: str): + """Defer paper-image imports to avoid a slot-asset provider cycle.""" + + if name == "generate_and_select": + from ..paper_to_image.generator import generate_and_select + + return generate_and_select + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/rfs/generation/assets.py b/rfs/generation/assets.py new file mode 100644 index 0000000..276137d --- /dev/null +++ b/rfs/generation/assets.py @@ -0,0 +1,974 @@ +from __future__ import annotations + +import base64 +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import math +import os +import shutil +import time +from pathlib import Path +from statistics import median +from typing import Iterable + +import requests +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +from ..utils import ensure_dir, write_json, write_text + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff"} + +CANDIDATE_DIRECTIONS = [ + "dense mechanism-first scientific mini scene with foreground, secondary objects, and micro-detail", + "reference-crop reconstruction with layered local objects and edge-to-edge support detail", + "compact paper-figure mini panel with concrete objects, internal mechanism, and background cues", + "structured process card with visible before-and-after relation plus local reference texture", + "high-density method card focused on the named operation, not a standalone pictogram", +] + +NEGATIVE_PROMPT = ( + "Avoid: large empty margin, tiny centered object, sparse white background, isolated small icon on blank canvas, " + "cropped subject, cut-off edges, fake formulas, fake axes, fake metrics, readable paragraphs, paper title text, " + "watermarks, logos, screenshots, low-resolution blur, poster style, decorative clutter without scientific meaning, " + "extra white presentation tile, white matting, large white border, generic blue-green dashboard styling, " + "simple centered icon, single object on clean blank background, standalone pictogram." +) + + +def _parse_ratio(ratio: str) -> tuple[float, float]: + try: + left, right = ratio.split(":", 1) + return max(0.001, float(left)), max(0.001, float(right)) + except Exception: + return 1.0, 1.0 + + +def _ratio_value(ratio: str) -> float: + rw, rh = _parse_ratio(ratio) + return rw / max(rh, 1) + + +def _target_size(ratio: str, max_side: int = 768, min_side: int = 96) -> tuple[int, int]: + rw, rh = _parse_ratio(ratio) + if rw >= rh: + return max_side, max(min_side, int(max_side * rh / rw)) + return max(min_side, int(max_side * rw / rh)), max_side + + +def _as_list_text(value) -> str: + if isinstance(value, list): + items = [str(item).strip() for item in value if str(item).strip()] + return "; ".join(items) + return str(value or "").strip() + + +def _fallback_visual_metaphor(slot: dict) -> str: + concept = str(slot.get("paper_concept", "scientific method component")) + panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "") + low = f"{concept} {panel}".lower() + if "sparse" in low or "nonzero" in low: + return "draw a matrix grid with many pale empty cells and a few highlighted trainable nonzero cells" + if "empty" in low or "zero" in low: + return "draw a mostly empty matrix grid with muted zero cells and clear blank positions" + if "frozen" in low or "pre-trained" in low or "pretrained" in low: + return "draw a large locked frozen weight matrix slab, with icy muted cells and no trainable highlights" + if "lora" in low: + return "draw two slim low-rank matrix factors forming a compact adapter update beside a larger frozen weight matrix" + if "ift" in low or "fourier" in low or "frequency" in low: + return "draw a frequency-domain sparse grid transforming into a smooth spatial-domain matrix through wave arcs" + if "split" in low or "submat" in low: + return "draw one matrix split into multiple smaller aligned submatrices with visible separation cuts" + if "concat" in low: + return "draw multiple transformed matrix blocks stacked vertically into one taller concatenated matrix" + if "adaptor" in low or "adapter" in low: + return "draw a frozen shared projection block receiving matrix features and emitting a compact update" + if "multiplication" in low: + return "draw two matrix blocks combining through a multiplication junction into a compact output block" + if "budget" in low or "parameter" in low: + return "draw a tiny trainable budget highlighted against a large muted frozen parameter field" + if "benchmark" in low or "evaluation" in low or "glue" in low: + return "draw a benchmark suite board with multiple task tiles and abstract bars without readable numbers" + return f"draw a concrete paper-specific visual object for {concept}, showing its role inside {panel}" + + +def build_slot_prompt(slot: dict, style: dict, candidate_index: int = 1) -> str: + palette = ", ".join(style.get("palette", [])[:5]) + direction = CANDIDATE_DIRECTIONS[(candidate_index - 1) % len(CANDIDATE_DIRECTIONS)] + composition = slot.get("composition_type", "full_frame_icon") + visual_metaphor = str(slot.get("visual_metaphor") or "").strip() or _fallback_visual_metaphor(slot) + slot_function = str(slot.get("slot_function") or "").strip() + image_prompt_core = str(slot.get("image_prompt_core") or "").strip() + reference_content_priority = str(slot.get("reference_content_priority") or "reference_primary_visual_object").strip() + reference_visual_object = str(slot.get("reference_visual_object") or "").strip() + reference_visual_elements = _as_list_text(slot.get("reference_visual_elements")) + reference_color_palette = str(slot.get("reference_color_palette") or "").strip() + paper_label_mapping = str(slot.get("paper_label_mapping") or "").strip() + must_show = _as_list_text(slot.get("must_show")) or "the concrete scientific object, its internal structure, and its input-output relation" + avoid_showing = _as_list_text(slot.get("avoid_showing")) or "generic sci-fi dashboard; unrelated robot or brain; fake formulas; fake numeric chart text" + panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "paper method") + reference_slot_role = str(slot.get("reference_slot_role") or "slot in the user-provided reference layout").strip() + reference_shape_language = str(slot.get("reference_shape_language") or slot.get("target_canvas_ratio") or "").strip() + reference_local_style = str(slot.get("reference_local_style") or "match the reference figure's local card/icon discipline").strip() + reference_prompt_hint = str(slot.get("reference_prompt_hint") or "use the reference image as local layout/style guidance, not as scientific content").strip() + reference_crop_path = str(slot.get("reference_crop_path") or "").strip() + reference_style_profile_path = str(slot.get("reference_style_profile_path") or "reference_style_profile.json").strip() + local_color_token_ids = ", ".join(str(item) for item in slot.get("local_color_token_ids", []) if str(item).strip()) + complexity_kind = str(slot.get("complexity_kind") or "pipeline_module").strip() + required_visual_complexity = str(slot.get("required_visual_complexity") or "dense").strip() + foreground_subject = str(slot.get("foreground_subject") or reference_visual_object or visual_metaphor).strip() + secondary_objects = _as_list_text(slot.get("secondary_objects")) or "2-5 supporting local objects from the reference crop" + micro_details = _as_list_text(slot.get("micro_details")) or "small non-critical marks, separators, texture, and internal details" + background_fill_elements = _as_list_text(slot.get("background_fill_elements")) or "edge-to-edge local reference-colored support detail" + scientific_mechanism_detail = str(slot.get("scientific_mechanism_detail") or "show the paper concept as a visible mechanism, not a generic icon").strip() + forbidden_simplification = _as_list_text(slot.get("forbidden_simplification")) or "simple icon; centered icon; clean blank background; single object on white canvas" + is_simple_allowed = complexity_kind == "legend_icon" + + if composition == "scene_thumbnail": + type_clause = ( + "Make it a dense small mechanism thumbnail: one concrete foreground scientific object, multiple visible process cues, secondary context objects, and enough environment to show what changes. Use edge-to-edge supporting scene detail so the crop is visually full." + ) + elif composition == "full_bleed_card": + type_clause = ( + "Make it a compact dense mechanism card: the central object fills the card, with internal subparts, small supporting objects, and micro process cues that clarify this exact operation. The card surface should extend almost to the image edge, with no separate white mat." + ) + elif composition == "symbol_cutout": + type_clause = ( + "Make it a large complete symbolic operation object plus local reference-colored support detail; the operation symbol or object should be visually distinct from other slots. If it is a line icon, thicken the strokes, enlarge it to nearly touch the safe area, and add compact supporting shapes so it is not sparse." + ) + else: + type_clause = ( + "Make it a full-frame scientific mechanism object with a large distinct subject, mechanism-specific internal details, and supporting local objects. If the subject is a waveform, face, pose skeleton, document, database, badge, or legend icon, make it bold, oversized, and backed by local reference-colored detail rather than a tiny object on a blank white field." + ) + + exact_ratio = float(slot.get("aspect_ratio_decimal") or _ratio_value(slot.get("target_canvas_ratio", "1.000:1.000"))) + center = slot.get("center_percent") if isinstance(slot.get("center_percent"), dict) else {} + center_text = f"x={float(center.get('x', 0.5)):.3f}, y={float(center.get('y', 0.5)):.3f}" + width_percent = float(slot.get("width_percent", slot.get("bbox_percent", {}).get("w", 0))) + height_percent = float(slot.get("height_percent", slot.get("bbox_percent", {}).get("h", 0))) + + return ( + f"Create candidate {candidate_index} for one slot-level image block in a publication-quality AI/ML research framework figure. " + f"This block belongs to macro panel: {panel}. Slot id: {slot['id']}. Paper concept: {slot['paper_concept']}. " + f"Slot function: {slot_function or 'visualize this paper concept as one small editable figure asset'}. " + f"Model-planned image prompt core: {image_prompt_core or visual_metaphor}. " + f"Reference content priority: {reference_content_priority}. " + f"Reference visual object to recreate: {reference_visual_object or visual_metaphor}. " + f"Reference visual elements to preserve: {reference_visual_elements or must_show}. " + f"Reference local color palette to preserve: {reference_color_palette or 'inherit colors from the local reference slot, not one global palette'}. " + f"Reference style profile source: {reference_style_profile_path}. Local reference crop used by the prompt planner: {reference_crop_path or 'missing crop path - preserve described local crop anyway'}. " + f"Local reference color token ids to inherit: {local_color_token_ids or 'none recorded; use local reference colors described in prompt plan'}. " + f"Paper label mapping: {paper_label_mapping or 'paper terminology is added later as editable PPT text'}. " + f"Main visual metaphor: {visual_metaphor}. " + f"Must visibly show: {must_show}. " + f"Reference-local slot role: {reference_slot_role}. " + f"Reference shape language: {reference_shape_language}. " + f"Reference-local style: {reference_local_style}. " + f"Reference prompt hint: {reference_prompt_hint}. " + f"Slot visual complexity contract: complexity_profile={slot.get('complexity_profile', 'reference-dense')}; complexity_kind={complexity_kind}; required_visual_complexity={required_visual_complexity}. " + f"Foreground subject must be large: {foreground_subject}. " + f"Secondary objects to include: {secondary_objects}. " + f"Micro details to include: {micro_details}. " + f"Background fill elements: {background_fill_elements}. " + f"Scientific mechanism detail: {scientific_mechanism_detail}. " + f"Forbidden simplifications: {forbidden_simplification}. " + "Use the local reference slot object first; do not invent a different abstract paper-internal object when the reference shows a concrete object. " + "The goal is to redraw/recreate the reference slot's object as a clean high-resolution asset, then adapt only minor details to the paper concept. " + f"Reference geometry is mandatory: canvas aspect ratio = exact {exact_ratio:.3f}:1, target_canvas_ratio = {slot['target_canvas_ratio']}, " + f"slot center in full figure = {center_text}, slot width_percent = {width_percent:.3f}, slot height_percent = {height_percent:.3f}. " + "Generate this asset for the exact reference slot shape, not a generic preset ratio such as 1:1, 4:3, 3:4, 16:9, or 9:16. " + "No semantic cropping: the whole subject must be visible, no object extending outside the frame, no cut-off hands/edges/bottom. " + "Safe area means important subject remains complete inside the frame, but supporting background details should extend close to edges. " + "Useful visual content should fill 90-97% of the canvas; minimum 85%; maximum empty margin is strictly below 10% on every edge. " + "Use a full-frame composition, large subject, maximal detail density, rich scientific visual texture, minimal blank canvas, no tiny centered object. " + + ("" if is_simple_allowed else "This must be a dense mini scientific scene/card with 2-5 layered objects, edge-to-edge supporting detail, and not a standalone pictogram. ") + + "The subject plus its local reference-colored support surface should occupy almost the entire frame; avoid isolated cutout icons floating on white or transparent-looking empty space. " + "For thin symbols such as pose skeletons, waveforms, clocks, microphones, documents, badges, and legend icons, use thick readable strokes, enlarged scale, and compact supporting shapes to raise visual fill while keeping the subject complete. " + f"Candidate visual direction: {direction}. {type_clause} " + f"Style constraints: polished academic paper illustration, crisp edges, consistent line quality, clean high-resolution vector-illustration / 2.5D hybrid only when useful. " + f"Global palette may include {palette}, but local reference colors override the global palette. Preserve color variety across slots. " + "Style consistency matters, but visual identity must vary by the reference crop: monitors look like monitors, video strips look like video strips, microphones look like microphones, waveforms look like waveforms, face cards look like face cards, pose skeletons look like pose skeletons, clocks look like clocks, clipboards look like clipboards, databases look like databases. " + "Allow only very small decorative non-critical UI marks or blurred pseudo-labels. " + "Critical scientific labels, formulas, variables, panel IDs, arrow labels, and metric values will be added later in PowerPoint and must not be generated in the image. " + f"Do not show: {avoid_showing}. " + "Do not add a white presentation tile, white mat, extra card background, or artificial border unless that exact border exists in the local reference crop. " + f"{NEGATIVE_PROMPT}" + ) + + +def _draw_placeholder(slot: dict, path: Path, style: dict, candidate_index: int = 1) -> None: + width, height = _target_size(slot["target_canvas_ratio"], max_side=512) + palette = style.get("palette", ["#EAF5FF", "#DFF7EF", "#8EC9E8", "#66C6A4", "#163B4D"]) + bg = palette[(candidate_index - 1) % 2] + line = palette[4] if len(palette) > 4 else "#163B4D" + accent = palette[2 + ((candidate_index + len(slot["id"])) % max(1, min(2, len(palette) - 2)))] if len(palette) >= 4 else "#8EC9E8" + img = Image.new("RGB", (width, height), bg) + draw = ImageDraw.Draw(img) + + margin = max(8, int(min(width, height) * 0.035)) + draw.rounded_rectangle([margin, margin, width - margin, height - margin], radius=max(14, margin * 2), fill=bg, outline=line, width=max(3, margin // 4)) + + # Dense edge-to-edge detail so placeholder QA exercises high-fill layouts. + for i in range(8 + candidate_index): + x = int(width * (0.035 + i * 0.12)) % width + draw.line([x, margin, min(width - margin, x + int(width * 0.22)), height - margin], fill=accent, width=max(1, margin // 6)) + for i in range(6): + y = int(height * (0.10 + i * 0.15)) + draw.arc([margin + i * 5, y, width - margin - i * 4, y + int(height * 0.26)], 180, 360, fill=accent, width=max(1, margin // 7)) + + cx, cy = width // 2, height // 2 + main_w = int(width * (0.72 + 0.03 * ((candidate_index - 1) % 3))) + main_h = int(height * (0.64 + 0.03 * ((candidate_index - 1) % 2))) + kind = slot.get("composition_type", "full_frame_icon") + if kind in {"scene_thumbnail", "full_bleed_card"}: + draw.rounded_rectangle([cx - main_w // 2, cy - main_h // 2, cx + main_w // 2, cy + main_h // 2], radius=max(12, margin), fill="#FFFFFF", outline=line, width=max(3, margin // 4)) + panel_gap = max(4, margin // 3) + for row in range(2): + for col in range(2): + px0 = cx - main_w // 2 + margin + col * ((main_w - margin * 2) // 2) + py0 = cy - main_h // 2 + margin + row * ((main_h - margin * 2) // 2) + px1 = px0 + (main_w - margin * 2) // 2 - panel_gap + py1 = py0 + (main_h - margin * 2) // 2 - panel_gap + draw.rounded_rectangle([px0, py0, px1, py1], radius=max(6, margin // 2), fill=accent if (row + col + candidate_index) % 2 == 0 else "#DFF7EF", outline=line, width=max(1, margin // 8)) + draw.polygon([(px0 + panel_gap, py1 - panel_gap), (px0 + (px1 - px0) // 2, py0 + panel_gap), (px1 - panel_gap, py1 - panel_gap)], fill="#9AD7C4") + else: + radius = min(main_w, main_h) // 2 + draw.ellipse([cx - radius, cy - radius, cx + radius, cy + radius], fill="#FFFFFF", outline=line, width=max(4, margin // 3)) + for angle_idx in range(12): + ang = angle_idx * math.pi / 6 + candidate_index * 0.08 + x1 = cx + int(math.cos(ang) * radius * 0.18) + y1 = cy + int(math.sin(ang) * radius * 0.18) + x2 = cx + int(math.cos(ang) * radius * 0.82) + y2 = cy + int(math.sin(ang) * radius * 0.82) + draw.line([x1, y1, x2, y2], fill=line, width=max(3, margin // 4)) + draw.ellipse([x2 - margin // 2, y2 - margin // 2, x2 + margin // 2, y2 + margin // 2], fill=accent, outline=line) + inner = int(radius * 0.36) + draw.rounded_rectangle([cx - inner, cy - inner, cx + inner, cy + inner], radius=max(8, margin // 2), fill=accent, outline=line, width=max(2, margin // 6)) + + try: + font = ImageFont.truetype("arial.ttf", max(9, int(min(width, height) * 0.045))) + except Exception: + font = ImageFont.load_default() + draw.text((margin * 1.2, height - margin * 2.2), "UI", fill=line, font=font) + path.parent.mkdir(parents=True, exist_ok=True) + img.save(path) + + +def _supported_gemini_ratio(ratio: str) -> str: + value = _ratio_value(ratio) + supported = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "16:9": 16 / 9, "9:16": 9 / 16} + return min(supported, key=lambda key: abs(supported[key] - value)) + + +def _image2_model_name() -> str: + model = os.getenv("RFS_IMAGE_MODEL") or os.getenv("IMAGE_MODEL") or "image-2" + # Yunwu currently exposes GPT Image 2 under this model id. Keep "image-2" + # as the user-facing logical name while sending the listed model id. + if model == "image-2": + return "gpt-image-2" + return model + + +def _image2_size(ratio: str) -> str: + value = _ratio_value(ratio) + if value >= 1.25: + return "1536x1024" + if value <= 0.80: + return "1024x1536" + return "1024x1024" + + +def _write_image_response(data: dict, output_path: Path) -> bool: + items = data.get("data") if isinstance(data.get("data"), list) else [] + if not items: + return False + item = items[0] + output_path.parent.mkdir(parents=True, exist_ok=True) + if item.get("b64_json"): + output_path.write_bytes(base64.b64decode(item["b64_json"])) + return True + if item.get("url"): + response = requests.get(item["url"], timeout=180) + response.raise_for_status() + output_path.write_bytes(response.content) + return True + return False + + +def _call_image2(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: + api_base = os.getenv("API_BASE", "https://yunwu.ai/v1").rstrip("/") + api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") + model = _image2_model_name() + if not api_key: + raise RuntimeError("API_KEY/GEMINI_API_KEY is required for --asset-mode image2") + payload = { + "model": model, + "prompt": prompt, + "n": 1, + "size": _image2_size(ratio), + } + max_attempts = max(1, int(retries) + 1) + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + response = requests.post( + f"{api_base}/images/generations", + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + data=json.dumps(payload), + timeout=240, + ) + if response.status_code == 400 and payload.get("size") != "1024x1024": + # Some image gateways accept only square output for a model. The + # prompt and final ratio-normalization still preserve slot ratio. + payload = dict(payload) + payload["size"] = "1024x1024" + response = requests.post( + f"{api_base}/images/generations", + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + data=json.dumps(payload), + timeout=240, + ) + if response.status_code == 429 or 500 <= response.status_code < 600: + raise RuntimeError(f"image2 request returned HTTP {response.status_code}") + response.raise_for_status() + if _write_image_response(response.json(), output_path): + return True + raise RuntimeError("image2 response did not contain b64_json or url image data") + except Exception as exc: + last_error = exc + if attempt >= max_attempts: + break + time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) + if last_error: + raise last_error + return False + + +def _call_gemini(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: + api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") + url = os.getenv("GEMINI_GEN_IMG_URL") + if not api_key or not url: + raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode gemini") + payload = { + "contents": [{"role": "user", "parts": [{"text": prompt}]}], + "generationConfig": { + "responseModalities": ["IMAGE"], + "imageConfig": {"imageSize": "2K", "aspectRatio": _supported_gemini_ratio(ratio)}, + }, + } + max_attempts = max(1, int(retries) + 1) + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + response = requests.post( + url, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + data=json.dumps(payload), + timeout=180, + ) + if response.status_code == 429 or 500 <= response.status_code < 600: + raise RuntimeError(f"Gemini image request returned HTTP {response.status_code}") + response.raise_for_status() + data = response.json() + for candidate in data.get("candidates", []): + for part in candidate.get("content", {}).get("parts", []): + inline = part.get("inlineData") or part.get("inline_data") + if inline and inline.get("data"): + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(base64.b64decode(inline["data"])) + return True + raise RuntimeError("Gemini image response did not contain inline image data") + except Exception as exc: + last_error = exc + if attempt >= max_attempts: + break + # Back off on rate limits, transient server errors, and network timeouts. + time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) + if last_error: + raise last_error + return False + + +def _enforce_target_canvas_ratio(path: Path, ratio: str) -> None: + """Resize to the exact slot canvas ratio without semantic cropping.""" + target_w, target_h = _target_size(ratio, max_side=1024, min_side=96) + with Image.open(path) as image: + converted = image.convert("RGBA") + if converted.size == (target_w, target_h): + return + resized = converted.resize((target_w, target_h), Image.LANCZOS) + path.parent.mkdir(parents=True, exist_ok=True) + resized.save(path) + + +def _content_bbox(rgba: Image.Image) -> tuple[list[int], float, float]: + width, height = rgba.size + pixels = rgba.load() + background = _median_corner_color(pixels, width, height) + rgba_array = np.asarray(rgba, dtype=np.int16) + rgb = rgba_array[:, :, :3] + alpha = rgba_array[:, :, 3] + background_array = np.asarray(background, dtype=np.float32) + color_distance_sq = np.sum((rgb.astype(np.float32) - background_array) ** 2, axis=2) + mask = (alpha > 12) & ((alpha < 245) | (color_distance_sq > 30.0 ** 2)) + ys, xs = np.nonzero(mask) + if xs.size == 0 or ys.size == 0: + return [0, 0, width, height], 0.0, 100.0 + left, right = int(xs.min()), int(xs.max()) + top, bottom = int(ys.min()), int(ys.max()) + bbox_w = max(1, right - left + 1) + bbox_h = max(1, bottom - top + 1) + content_fill = bbox_w * bbox_h / float(width * height) * 100.0 + margins = ( + left / width * 100.0, + (width - right - 1) / width * 100.0, + top / height * 100.0, + (height - bottom - 1) / height * 100.0, + ) + return [left, top, right + 1, bottom + 1], content_fill, max(margins) + + +def _hex_rgb(value: str, fallback: tuple[int, int, int] = (229, 242, 248)) -> tuple[int, int, int]: + text = str(value or "").strip().lstrip("#") + if len(text) >= 6: + try: + return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) + except Exception: + return fallback + return fallback + + +def _fit_subject_to_canvas(path: Path, min_fill: float = 88.0, max_margin: float = 9.0, support_color: str | None = None) -> None: + """Remove only plain background slack and enlarge the complete subject.""" + with Image.open(path) as image: + rgba = image.convert("RGBA") + width, height = rgba.size + bbox, content_fill, empty_margin = _content_bbox(rgba) + if content_fill >= min_fill and empty_margin <= max_margin: + return + left, top, right, bottom = bbox + pad_x = max(4, int((right - left) * 0.06)) + pad_y = max(4, int((bottom - top) * 0.06)) + left = max(0, left - pad_x) + top = max(0, top - pad_y) + right = min(width, right + pad_x) + bottom = min(height, bottom + pad_y) + if right <= left or bottom <= top: + return + subject = rgba.crop((left, top, right, bottom)) + scale = min(width * 0.94 / max(subject.width, 1), height * 0.94 / max(subject.height, 1)) + if scale <= 1.02: + canvas = rgba + else: + new_w = max(1, min(width, int(subject.width * scale))) + new_h = max(1, min(height, int(subject.height * scale))) + subject = subject.resize((new_w, new_h), Image.LANCZOS) + pixels = rgba.load() + bg = tuple(int(v) for v in _median_corner_color(pixels, width, height)) + (255,) + canvas = Image.new("RGBA", (width, height), bg) + canvas.alpha_composite(subject, ((width - new_w) // 2, (height - new_h) // 2)) + _bbox, final_fill, final_margin = _content_bbox(canvas) + if (final_fill < min_fill or final_margin > max_margin) and support_color: + support = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(support) + r, g, b = _hex_rgb(support_color) + inset_x = max(2, int(width * 0.025)) + inset_y = max(2, int(height * 0.025)) + draw.rounded_rectangle( + (inset_x, inset_y, width - inset_x, height - inset_y), + radius=max(8, min(width, height) // 12), + fill=(r, g, b, 82), + outline=(r, g, b, 150), + width=max(2, min(width, height) // 80), + ) + for i in range(5): + x = int(width * (0.10 + i * 0.18)) + draw.line((x, inset_y + 4, x + int(width * 0.10), height - inset_y - 4), fill=(r, g, b, 58), width=2) + canvas.alpha_composite(support) + canvas.save(path) + + +def _median_corner_color(pixels, width: int, height: int) -> tuple[float, float, float]: + sample = [] + band_x = max(1, min(width // 12, 28)) + band_y = max(1, min(height // 12, 28)) + regions = ( + (0, 0, band_x, band_y), + (width - band_x, 0, width, band_y), + (0, height - band_y, band_x, height), + (width - band_x, height - band_y, width, height), + ) + for x0, y0, x1, y1 in regions: + for y in range(y0, y1): + for x in range(x0, x1): + r, g, b, _a = pixels[x, y] + sample.append((r, g, b)) + if not sample: + return (255.0, 255.0, 255.0) + return ( + float(median([rgb[0] for rgb in sample])), + float(median([rgb[1] for rgb in sample])), + float(median([rgb[2] for rgb in sample])), + ) + + +def _color_distance(a: tuple[int, int, int], b: tuple[float, float, float]) -> float: + return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2) ** 0.5 + + +def _estimate_quality(path: Path, slot: dict, candidate_index: int, source: str) -> dict: + with Image.open(path) as image: + rgba = image.convert("RGBA") + width, height = rgba.size + bbox, content_fill, empty_margin = _content_bbox(rgba) + + image_ratio = width / max(height, 1) + target_ratio = _ratio_value(slot["target_canvas_ratio"]) + ratio_mismatch_percent = abs(image_ratio - target_ratio) / max(target_ratio, 0.001) * 100.0 + + issue_tags = [] + if content_fill < float(slot.get("min_content_fill_percent", 80)): + issue_tags.append("too_much_whitespace") + if empty_margin > float(slot.get("max_empty_margin_percent", 12)): + issue_tags.append("large_blank_canvas") + if content_fill < 65 and empty_margin > 18: + issue_tags.append("tiny_centered_subject") + if ratio_mismatch_percent > 10: + issue_tags.append("ratio_mismatch") + + fill_score = min(content_fill, 95.0) - max(0.0, content_fill - 95.0) * 1.5 + margin_penalty = max(0.0, empty_margin - 12.0) * 4.0 + ratio_penalty = max(0.0, ratio_mismatch_percent - 10.0) * 2.0 + score = fill_score - margin_penalty - ratio_penalty + + return { + "candidate_index": candidate_index, + "path": str(path), + "source": source, + "width": width, + "height": height, + "content_bbox_px": bbox, + "content_fill_percent": round(content_fill, 2), + "min_content_fill_percent": slot["min_content_fill_percent"], + "empty_margin_percent": round(empty_margin, 2), + "max_empty_margin_percent": slot["max_empty_margin_percent"], + "ratio_mismatch_percent": round(ratio_mismatch_percent, 2), + "edge_cutoff_status": "ok", + "ratio_status": "ok" if ratio_mismatch_percent <= 10 else "needs_ratio_regeneration", + "issue_tags": issue_tags, + "selection_score": round(score, 2), + "action": "ok_no_crop" if not issue_tags else "candidate_rejected", + "estimation_method": "corner-background-content-bbox", + } + + +def _simple_allowed_slot(slot: dict) -> bool: + text = " ".join( + str(slot.get(key, "")) + for key in ["id", "paper_concept", "macro_panel", "parent_panel", "complexity_kind", "composition_type"] + ).lower() + return str(slot.get("complexity_kind", "")).lower() == "legend_icon" or any( + term in text + for term in ["legend", "badge", "label", "ocean", "openness", "conscientiousness", "extraversion", "agreeableness", "neuroticism"] + ) + + +def _estimate_component_count(mask: list[list[bool]]) -> int: + if not mask or not mask[0]: + return 0 + height = len(mask) + width = len(mask[0]) + seen = [[False] * width for _ in range(height)] + components = 0 + min_area = max(6, int(width * height * 0.002)) + for y in range(height): + for x in range(width): + if seen[y][x] or not mask[y][x]: + continue + stack = [(x, y)] + seen[y][x] = True + area = 0 + while stack: + px, py = stack.pop() + area += 1 + for nx, ny in ((px + 1, py), (px - 1, py), (px, py + 1), (px, py - 1)): + if 0 <= nx < width and 0 <= ny < height and not seen[ny][nx] and mask[ny][nx]: + seen[ny][nx] = True + stack.append((nx, ny)) + if area >= min_area: + components += 1 + return components + + +def _estimate_complexity(path: Path, slot: dict, quality: dict) -> dict: + with Image.open(path) as image: + rgba = image.convert("RGBA") + width, height = rgba.size + small_w = 96 + small_h = max(16, int(96 * height / max(width, 1))) + if small_h > 128: + small_h = 128 + small_w = max(16, int(128 * width / max(height, 1))) + small = rgba.resize((small_w, small_h), Image.LANCZOS) + pixels = small.load() + background = _median_corner_color(pixels, small_w, small_h) + mask: list[list[bool]] = [] + quantized_colors: set[tuple[int, int, int]] = set() + edge_hits = 0 + comparisons = 0 + for y in range(small_h): + row: list[bool] = [] + for x in range(small_w): + r, g, b, a = pixels[x, y] + active = a > 12 and (a < 245 or _color_distance((r, g, b), background) > 30.0) + row.append(active) + if active: + quantized_colors.add((r // 24, g // 24, b // 24)) + if x > 0: + pr, pg, pb, pa = pixels[x - 1, y] + comparisons += 1 + if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: + edge_hits += 1 + if y > 0: + pr, pg, pb, pa = pixels[x, y - 1] + comparisons += 1 + if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: + edge_hits += 1 + mask.append(row) + component_count = _estimate_component_count(mask) + color_count = len(quantized_colors) + edge_density = edge_hits / max(comparisons, 1) * 100.0 + content_fill = float(quality.get("content_fill_percent", 0)) + spec_objects = len(slot.get("secondary_objects") if isinstance(slot.get("secondary_objects"), list) else []) + spec_micro = len(slot.get("micro_details") if isinstance(slot.get("micro_details"), list) else []) + object_count_estimate = max(component_count, 1 + min(4, spec_objects // 2)) + detail_score = min( + 100.0, + content_fill * 0.30 + + min(color_count, 24) * 1.25 + + min(edge_density, 28.0) * 1.15 + + min(component_count, 8) * 4.0 + + min(spec_objects + spec_micro, 8) * 2.5, + ) + target = float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)) + object_target = int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)) + simple_icon_risk = False + issue_tags: list[str] = [] + if not _simple_allowed_slot(slot): + if detail_score < target: + simple_icon_risk = True + issue_tags.append("too_simple") + if object_count_estimate < object_target: + simple_icon_risk = True + issue_tags.append("generic_icon") + if content_fill >= 85 and color_count <= 4 and edge_density < 3: + simple_icon_risk = True + issue_tags.append("single_object_on_blank_background") + if not str(slot.get("reference_crop_path", "")).strip(): + issue_tags.append("reference_crop_ignored") + if not str(slot.get("reference_style_profile_path", "")).strip(): + issue_tags.append("style_drift") + return { + "detail_score": round(detail_score, 2), + "detail_score_target": target, + "object_count_estimate": object_count_estimate, + "object_count_target": object_target, + "simple_icon_risk": simple_icon_risk, + "reference_crop_match": "planned_reference_crop_grounded" if str(slot.get("reference_crop_path", "")).strip() else "missing_reference_crop", + "style_match": "planned_reference_style_grounded" if str(slot.get("reference_style_profile_path", "")).strip() else "missing_reference_style", + "color_bin_count": color_count, + "edge_density_percent": round(edge_density, 2), + "component_count_estimate": component_count, + "complexity_kind": slot.get("complexity_kind") or "pipeline_module", + "required_visual_complexity": slot.get("required_visual_complexity") or "dense", + "complexity_issue_tags": sorted(set(issue_tags)), + } + + +def _select_candidate(candidate_reports: list[dict]) -> dict: + passing = [item for item in candidate_reports if not item.get("issue_tags")] + pool = passing or candidate_reports + return max(pool, key=lambda item: item["selection_score"]) + + +def _make_contact_sheet(asset_paths: Iterable[Path], output_path: Path, title: str = "selected assets") -> None: + paths = list(asset_paths) + thumbs = [] + cell_w, cell_h = 260, 228 + for path in paths: + img = Image.open(path).convert("RGB") + img.thumbnail((224, 162), Image.LANCZOS) + cell = Image.new("RGB", (cell_w, cell_h), "white") + draw = ImageDraw.Draw(cell) + cell.paste(img, ((cell_w - img.width) // 2, 14)) + draw.text((10, 184), path.stem[:34], fill=(20, 20, 20)) + thumbs.append(cell) + cols = 5 + rows = max(1, (len(thumbs) + cols - 1) // cols) + header_h = 34 + sheet = Image.new("RGB", (cols * cell_w, rows * cell_h + header_h), (245, 247, 248)) + draw = ImageDraw.Draw(sheet) + draw.text((12, 10), title, fill=(20, 20, 20)) + for index, thumb in enumerate(thumbs): + sheet.paste(thumb, ((index % cols) * cell_w, header_h + (index // cols) * cell_h)) + output_path.parent.mkdir(parents=True, exist_ok=True) + sheet.save(output_path) + + +def _generate_one_candidate(slot: dict, style: dict, path: Path, asset_mode: str, candidate_index: int, prompt: str, retries: int = 2) -> str: + support_color = "" + if isinstance(slot.get("reference_dominant_colors"), list) and slot.get("reference_dominant_colors"): + support_color = str(slot["reference_dominant_colors"][0]) + elif isinstance(style.get("reference_palette"), list) and style.get("reference_palette"): + support_color = str(style["reference_palette"][0]) + if asset_mode == "placeholder": + _draw_placeholder(slot, path, style, candidate_index=candidate_index) + _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) + return "explicit_placeholder_candidate" + if asset_mode == "gemini": + ok = _call_gemini(prompt, slot["target_canvas_ratio"], path, retries=retries) + if not ok: + raise RuntimeError(f"Gemini returned no image for slot {slot['id']} candidate {candidate_index}") + _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) + _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) + return "gemini_candidate_generation" + if asset_mode == "image2": + ok = _call_image2(prompt, slot["target_canvas_ratio"], path, retries=retries) + if not ok: + raise RuntimeError(f"image2 returned no image for slot {slot['id']} candidate {candidate_index}") + _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) + _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) + return f"image2_candidate_generation:{_image2_model_name()}" + raise ValueError(f"Unsupported asset_mode: {asset_mode}") + + +def _generation_failure_report(slot: dict, candidate_index: int, path: Path, exc: Exception) -> dict: + return { + "candidate_index": candidate_index, + "path": str(path), + "source": "generation_failed", + "width": 0, + "height": 0, + "content_bbox_px": [0, 0, 0, 0], + "content_fill_percent": 0.0, + "min_content_fill_percent": slot["min_content_fill_percent"], + "empty_margin_percent": 100.0, + "max_empty_margin_percent": slot["max_empty_margin_percent"], + "ratio_mismatch_percent": 100.0, + "edge_cutoff_status": "unknown_generation_failed", + "ratio_status": "unknown_generation_failed", + "issue_tags": ["generation_failed"], + "selection_score": -9999.0, + "detail_score": 0.0, + "detail_score_target": float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)), + "object_count_estimate": 0, + "object_count_target": int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)), + "simple_icon_risk": True, + "reference_crop_match": "unknown_generation_failed", + "style_match": "unknown_generation_failed", + "complexity_issue_tags": ["generation_failed"], + "action": "candidate_rejected", + "error": str(exc), + } + + +def _generate_and_score_candidate(task: dict) -> dict: + slot = task["slot"] + candidate_path = task["path"] + try: + source = _generate_one_candidate( + slot, + task["style"], + candidate_path, + task["asset_mode"], + task["candidate_index"], + task["prompt"], + retries=task["asset_retries"], + ) + quality = _estimate_quality(candidate_path, slot, task["candidate_index"], source) + complexity = _estimate_complexity(candidate_path, slot, quality) + issue_tags = sorted(set(list(quality.get("issue_tags", [])) + list(complexity.get("complexity_issue_tags", [])))) + quality.update(complexity) + quality["issue_tags"] = issue_tags + complexity_penalty = 0.0 + if complexity.get("simple_icon_risk"): + complexity_penalty += 35.0 + complexity_penalty += max(0.0, float(complexity.get("detail_score_target", 0)) - float(complexity.get("detail_score", 0))) * 0.75 + quality["selection_score"] = round(float(quality.get("selection_score", 0)) + float(complexity.get("detail_score", 0)) * 0.35 - complexity_penalty, 2) + quality["action"] = "ok_no_crop" if not issue_tags else "candidate_rejected" + return quality + except Exception as exc: + return _generation_failure_report(slot, task["candidate_index"], candidate_path, exc) + + +def generate_assets( + program: dict, + style: dict, + out_dir: str | Path, + asset_mode: str = "gemini", + candidates_per_slot: int = 3, + asset_workers: int = 1, + asset_retries: int = 2, +) -> dict: + out = Path(out_dir) + assets_dir = ensure_dir(out / "assets") + candidates_root = ensure_dir(out / "asset_candidates") + candidates_per_slot = max(1, min(5, int(candidates_per_slot))) + asset_workers = max(1, min(12, int(asset_workers))) + asset_retries = max(0, min(5, int(asset_retries))) + + prompt_lines = [ + "# Summary", + "Slot-level image prompts generated after paper brief, reference slot analysis, style sheet, layout_plan.json, figure_program.json, and slot_prompt_plan.json.", + "Each prompt inherits the model-planned image_prompt_core for its slot before adding canvas, fill, text-control, and no-crop constraints.", + f"Each slot generates {candidates_per_slot} candidate image block(s) with up to {asset_workers} worker(s); the selected asset is copied into `assets/` without semantic cropping.", + "", + ] + report_items = [] + complexity_items = [] + selected_paths = [] + all_candidate_paths = [] + tasks = [] + ordered_slot_candidates: dict[str, list[dict]] = {} + + for slot in program["slots"]: + slot_candidate_dir = ensure_dir(candidates_root / slot["id"]) + ordered_slot_candidates[slot["id"]] = [] + prompt_lines.append(f"## {slot['id']}") + prompt_lines.append(f"Paper concept: {slot['paper_concept']}") + prompt_lines.append(f"Prompt plan id: {slot.get('prompt_plan_id', 'missing')}") + prompt_lines.append(f"Visual spec id: {slot.get('visual_spec_id', 'missing')}") + prompt_lines.append(f"Complexity kind: {slot.get('complexity_kind', 'pipeline_module')}") + prompt_lines.append(f"Image prompt core: {slot.get('image_prompt_core', '')}") + prompt_lines.append("") + + for candidate_index in range(1, candidates_per_slot + 1): + prompt = build_slot_prompt(slot, style, candidate_index=candidate_index) + candidate_path = slot_candidate_dir / f"candidate_{candidate_index:02d}.png" + prompt_lines.extend([f"### Candidate {candidate_index}", prompt, ""]) + task = { + "slot": slot, + "style": style, + "path": candidate_path, + "asset_mode": asset_mode, + "candidate_index": candidate_index, + "prompt": prompt, + "asset_retries": asset_retries, + } + tasks.append(task) + ordered_slot_candidates[slot["id"]].append(task) + + reports_by_key: dict[tuple[str, int], dict] = {} + if asset_workers == 1 or len(tasks) <= 1: + for task in tasks: + report = _generate_and_score_candidate(task) + reports_by_key[(task["slot"]["id"], task["candidate_index"])] = report + else: + with ThreadPoolExecutor(max_workers=asset_workers) as executor: + futures = {executor.submit(_generate_and_score_candidate, task): task for task in tasks} + for future in as_completed(futures): + task = futures[future] + reports_by_key[(task["slot"]["id"], task["candidate_index"])] = future.result() + + for slot in program["slots"]: + candidate_reports = [ + reports_by_key[(slot["id"], candidate_index)] + for candidate_index in range(1, candidates_per_slot + 1) + ] + successful_reports = [item for item in candidate_reports if Path(item.get("path", "")).exists() and "generation_failed" not in item.get("issue_tags", [])] + all_candidate_paths.extend(Path(item["path"]) for item in successful_reports) + if not successful_reports: + errors = "; ".join(item.get("error", "unknown generation failure") for item in candidate_reports) + raise RuntimeError(f"All candidate generations failed for slot {slot['id']}: {errors}") + selected = _select_candidate(candidate_reports) + selected_path = Path(selected["path"]) + final_asset_path = assets_dir / f"{slot['asset_id']}.png" + shutil.copyfile(selected_path, final_asset_path) + selected_paths.append(final_asset_path) + + selected_action = "ok_no_crop" if not selected.get("issue_tags") else "selected_best_but_needs_regeneration" + selected_reason = ( + "Selected because it had the best combined fill, margin, ratio, and visual-complexity score." + if not selected.get("issue_tags") + else "Selected as the least-bad candidate but validation should fail until regenerated." + ) + report_items.append({ + "asset_id": slot["asset_id"], + "slot_id": slot["id"], + "path": str(final_asset_path), + "selected_candidate_index": selected["candidate_index"], + "selected_candidate_path": selected["path"], + "source": selected["source"], + "content_fill_percent": selected["content_fill_percent"], + "min_content_fill_percent": slot["min_content_fill_percent"], + "empty_margin_percent": selected["empty_margin_percent"], + "max_empty_margin_percent": slot["max_empty_margin_percent"], + "ratio_mismatch_percent": selected["ratio_mismatch_percent"], + "edge_cutoff_status": selected["edge_cutoff_status"], + "ratio_status": selected["ratio_status"], + "issue_tags": selected.get("issue_tags", []), + "action": selected_action, + "detail_score": selected.get("detail_score"), + "detail_score_target": selected.get("detail_score_target"), + "object_count_estimate": selected.get("object_count_estimate"), + "object_count_target": selected.get("object_count_target"), + "simple_icon_risk": selected.get("simple_icon_risk"), + "reference_crop_match": selected.get("reference_crop_match"), + "style_match": selected.get("style_match"), + "complexity_issue_tags": selected.get("complexity_issue_tags", []), + "selected_reason": selected_reason, + "notes": "Selected from candidates; no semantic crop used; inserted into PPT with contain-fit.", + "rejected_candidates": [item for item in candidate_reports if item["candidate_index"] != selected["candidate_index"]], + }) + complexity_items.append({ + "asset_id": slot["asset_id"], + "slot_id": slot["id"], + "selected_candidate_index": selected["candidate_index"], + "selected_candidate_path": selected["path"], + "complexity_kind": selected.get("complexity_kind") or slot.get("complexity_kind"), + "required_visual_complexity": selected.get("required_visual_complexity") or slot.get("required_visual_complexity"), + "detail_score": selected.get("detail_score"), + "detail_score_target": selected.get("detail_score_target"), + "object_count_estimate": selected.get("object_count_estimate"), + "object_count_target": selected.get("object_count_target"), + "simple_icon_risk": selected.get("simple_icon_risk"), + "reference_crop_match": selected.get("reference_crop_match"), + "style_match": selected.get("style_match"), + "complexity_issue_tags": selected.get("complexity_issue_tags", []), + "selected_reason": selected_reason, + "candidate_complexity": [ + { + "candidate_index": item.get("candidate_index"), + "path": item.get("path"), + "detail_score": item.get("detail_score"), + "object_count_estimate": item.get("object_count_estimate"), + "simple_icon_risk": item.get("simple_icon_risk"), + "reference_crop_match": item.get("reference_crop_match"), + "style_match": item.get("style_match"), + "complexity_issue_tags": item.get("complexity_issue_tags", []), + "selection_score": item.get("selection_score"), + } + for item in candidate_reports + ], + }) + + write_text(out / "prompts.md", "\n".join(prompt_lines)) + report = { + "summary": "Asset quality report for multi-candidate slot-level generated blocks.", + "candidate_policy": { + "candidates_per_slot": candidates_per_slot, + "selection": "prefer passing candidates; otherwise choose highest fill/margin/ratio score and fail validation if below thresholds", + "no_crop": True, + }, + "assets": report_items, + } + write_json(out / "asset_quality_report.json", report) + complexity_report = { + "summary": "Asset complexity report for selected slot-level generated blocks.", + "policy": "Reject normal non-legend assets that look like sparse generic icons, ignore the local crop, or lack layered visual detail.", + "assets": complexity_items, + } + write_json(out / "asset_complexity_report.json", complexity_report) + _make_contact_sheet(selected_paths, out / "asset_contact_sheet.png", title="selected slot assets") + _make_contact_sheet(all_candidate_paths, out / "asset_candidate_contact_sheet.png", title="all generated candidates") + return report diff --git a/rfs/generation/rebuild_assets.py b/rfs/generation/rebuild_assets.py new file mode 100644 index 0000000..9336bda --- /dev/null +++ b/rfs/generation/rebuild_assets.py @@ -0,0 +1,167 @@ +"""Canonical boundary for rebuild slot-asset orchestration. + +The public implementation is exposed here while the legacy workflow keeps +its private implementation under a compatibility-only name. The lazy call +keeps import order cycle-free and preserves historical behavior. +""" + +from __future__ import annotations + +import json +import base64 +import mimetypes +import os +import shutil +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from PIL import Image, ImageChops, ImageDraw + +from ..utils import ensure_dir, write_json +from .rebuild_policy import ASSET_THRESHOLDS, _effective_asset_mode_for_spec, economy_acceptance_decision + + +def _crop_reference(reference_path: Path, out_path: Path, bbox_percent: dict) -> Path: + with Image.open(reference_path).convert("RGB") as image: + w, h = image.size + left, top = int(float(bbox_percent["x"]) * w), int(float(bbox_percent["y"]) * h) + right = int((float(bbox_percent["x"]) + float(bbox_percent["w"])) * w) + bottom = int((float(bbox_percent["y"]) + float(bbox_percent["h"])) * h) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.crop((max(0, left), max(0, top), min(w, right), min(h, bottom))).save(out_path) + return out_path + + +def _placeholder_asset(slot: dict, out_path: Path, background: str) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) + size = (512, 288) if ratio >= 1.55 else (288, 512) if ratio <= 0.62 else (384, 384) + image = Image.new("RGB", size, background) + draw = ImageDraw.Draw(image) + pad = int(min(size) * 0.12) + draw.rounded_rectangle((pad, pad, size[0] - pad, size[1] - pad), radius=18, fill="#FFFFFF", outline="#6C7A89", width=3) + draw.line((pad * 1.5, size[1] * 0.45, size[0] - pad * 1.5, size[1] * 0.45), fill="#4A6FA5", width=5) + draw.ellipse((size[0] * 0.42, size[1] * 0.22, size[0] * 0.58, size[1] * 0.38), fill="#E17721") + draw.text((pad * 1.3, size[1] * 0.58), slot["id"], fill="#263747") + image.save(out_path) + + +def _api_generate_asset(spec: dict, crop_path: Path, out_path: Path) -> dict: + import requests + api_key, url = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY"), os.getenv("GEMINI_GEN_IMG_URL") + if not api_key or not url: + raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode api") + payload = {"contents": [{"role": "user", "parts": [{"text": spec["prompt"]}, {"inlineData": {"mimeType": mimetypes.guess_type(crop_path.name)[0] or "image/png", "data": base64.b64encode(crop_path.read_bytes()).decode("ascii")}}]}], "generationConfig": {"responseModalities": ["IMAGE"], "imageConfig": {"imageSize": "1K", "aspectRatio": spec["generation_aspect_ratio"]}}} + response = requests.post(url, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), timeout=240) + response.raise_for_status() + for candidate in response.json().get("candidates", []): + for part in candidate.get("content", {}).get("parts", []): + inline = part.get("inlineData") or part.get("inline_data") + if inline and inline.get("data"): + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(base64.b64decode(inline["data"])) + return {"status": "generated", "api_requests_attempted": 1} + raise RuntimeError("API returned no image data") + + +def _estimate_background(image: Image.Image) -> str: + rgb = image.convert("RGB") + w, h = rgb.size + samples = [rgb.crop(box).resize((1, 1)).getpixel((0, 0)) for box in [(0, 0, w, max(1, h // 12)), (0, h - max(1, h // 12), w, h), (0, 0, max(1, w // 12), h), (w - max(1, w // 12), 0, w, h)]] + avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) + return f"#{avg[0]:02X}{avg[1]:02X}{avg[2]:02X}" + + +def _foreground_metrics(image_path: str | Path, background_hex: str | None = None) -> dict: + try: + image = Image.open(image_path).convert("RGB") + except Exception as exc: + return {"status": "error", "error": str(exc), "foreground_bbox_px": None, "foreground_bbox_fill_percent": 0.0} + w, h = image.size + bg = background_hex or _estimate_background(image) + bg_rgb = tuple(int(bg.strip("#")[i:i + 2], 16) for i in (0, 2, 4)) + diff = ImageChops.difference(image, Image.new("RGB", image.size, bg_rgb)).convert("L") + box = diff.point(lambda p: 255 if p > 18 else 0).getbbox() + if not box: + return {"status": "empty", "foreground_bbox_px": None, "foreground_bbox_fill_percent": 0.0, "margin_left_percent": 0.5, "margin_right_percent": 0.5, "margin_top_percent": 0.5, "margin_bottom_percent": 0.5} + left, top, right, bottom = box + fill = ((right - left) * (bottom - top)) / max(w * h, 1) + return {"status": "ok", "foreground_bbox_px": [left, top, right, bottom], "foreground_bbox_fill_percent": round(fill, 4), "margin_left_percent": round(left / w, 4), "margin_right_percent": round((w - right) / w, 4), "margin_top_percent": round(top / h, 4), "margin_bottom_percent": round((h - bottom) / h, 4)} + + +def _load_accepted_assets(path: Path) -> dict: + if not path.exists(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return data if isinstance(data, dict) else {} + + +def _generate_assets(specs: list[dict], program: dict, out: Path, asset_mode: str, asset_workers: int, asset_retries: int, economy_mode: bool, regenerate_slots: set[str], strict_asset_regeneration: bool, asset_policy: str = "legacy") -> tuple[list[dict], dict]: + """Generate/reuse slot raster assets and write the stable rebuild reports.""" + asset_dir, accepted, reports = ensure_dir(out / "assets"), _load_accepted_assets(out / "accepted_assets.json"), [] + def one(spec: dict) -> dict: + slot_id, out_path = spec["slot_id"], asset_dir / f"{spec['asset_id']}.png" + if spec.get("asset_generation_mode") == "skip_asset" or spec.get("asset_decision") in {"ppt_primitive", "needs_editable_decomposition"}: + return {**spec, "status": str(spec.get("asset_decision") or "skip_asset"), "asset_path": None, "api_requests_attempted": 0, "effective_asset_mode": "skip_asset", "policy_decision_mode": "editable_first", "fallback_used": False} + if spec.get("asset_decision") == "reuse_existing": + source = asset_dir / f"{spec['reuse_source_asset_id']}.png" + if not source.exists(): return {**spec, "status": "reuse_source_missing", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": f"reuse source missing: {source}"} + if source.resolve() != out_path.resolve(): shutil.copyfile(source, out_path) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=False) + return {**spec, **metrics, **decision, "status": "reused_from_group", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_group_asset", "fallback_used": False} + locked = bool(accepted.get(slot_id, {}).get("accepted")) or bool(accepted.get(spec["asset_id"], {}).get("accepted")) + if out_path.exists() and economy_mode and slot_id not in regenerate_slots and not strict_asset_regeneration: + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=False) + if locked or decision["accepted"]: return {**spec, **metrics, **decision, "status": "reused", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_existing_accepted_or_threshold_passed", "fallback_used": False} + crop, last_error = Path(spec["reference_crop_path"]), None + for attempt in range(1, (max(1, int(asset_retries) + 1) if strict_asset_regeneration else 1) + 1): + mode, policy_mode = _effective_asset_mode_for_spec(asset_mode, asset_policy, spec) + try: + if mode == "placeholder": _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]); status, requests = "placeholder", 0 + elif mode == "crop" and asset_policy != "smart-api": shutil.copyfile(crop, out_path); status, requests = "reference_crop", 0 + elif mode == "crop": _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]); status, requests = "crop_disabled_by_smart_api_policy_placeholder", 0 + else: result = _api_generate_asset(spec, crop, out_path); status, requests = result.get("status", "generated"), int(result.get("api_requests_attempted", 1)) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=strict_asset_regeneration) + return {**spec, **metrics, **decision, "status": status, "asset_path": str(out_path), "api_requests_attempted": requests, "attempt": attempt, "economy_decision": "generated_once" if economy_mode else "generated", "effective_asset_mode": mode, "policy_decision_mode": policy_mode, "fallback_used": False} + except Exception as exc: + last_error = str(exc) + try: + if mode == "api" and asset_policy != "smart-api": shutil.copyfile(crop, out_path); status, reason = "api_failed_reference_crop_fallback", "fallback_after_api_error" + elif mode == "api": _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]); status, reason = "api_failed_placeholder_fallback", "placeholder_after_api_error_crop_disabled" + else: continue + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=False) + return {**spec, **metrics, **decision, "status": status, "asset_path": str(out_path), "api_requests_attempted": 1, "attempt": attempt, "economy_decision": reason, "effective_asset_mode": mode, "policy_decision_mode": policy_mode, "fallback_used": True, "error": last_error} + except Exception: pass + return {**spec, "status": "failed", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": last_error} + primary, reuse = [s for s in specs if s.get("asset_decision") != "reuse_existing"], [s for s in specs if s.get("asset_decision") == "reuse_existing"] + with ThreadPoolExecutor(max_workers=max(1, min(int(asset_workers), 12))) as pool: + reports.extend(f.result() for f in as_completed([pool.submit(one, s) for s in primary])) + reports.extend(one(s) for s in reuse); reports.sort(key=lambda item: str(item.get("slot_id"))) + api_requests = sum(int(item.get("api_requests_attempted") or 0) for item in reports); program["assets"] = [{"id": s["asset_id"], "path": f"assets/{s['asset_id']}.png", "source": "slot_asset"} for s in specs] + summary = {"summary": "Asset generation report for reusable editable rebuild workflow.", "asset_mode": asset_mode, "asset_policy": asset_policy, "final_crop_assets_allowed": asset_policy != "smart-api", "economy_mode": economy_mode, "asset_retries": asset_retries, "strict_asset_regeneration": strict_asset_regeneration, "api_requests_attempted": api_requests, "policy_decision_mode": "generation_policy" if asset_mode == "policy" or asset_policy == "generation-policy" else "global_override", "assets": reports} + write_json(out / "asset_generation_report.json", summary) + write_json(out / "asset_economy_report.json", {"summary": "Economy policy decisions.", "api_requests_attempted": api_requests, "accepted_assets_file": str(out / "accepted_assets.json"), "regenerate_slots": sorted(regenerate_slots), "thresholds": {k: {"min": v[0], "max": v[1]} for k, v in ASSET_THRESHOLDS.items()}, "asset_policy": asset_policy, "assets": [{k: item.get(k) for k in ("slot_id", "slot_type", "economy_decision", "selected_reason", "api_requests_attempted", "asset_source_policy", "policy_source", "policy_decision_mode", "fallback_used")} for item in reports]}) + def size(path: str | None): + if not path or not Path(path).exists(): return None + with Image.open(path) as image: return list(image.size) + write_json(out / "asset_ratio_fit_report.json", {"summary": "Aspect-ratio fit report.", "assets": [{"slot_id": item.get("slot_id"), "slot_aspect_ratio": item.get("slot_aspect_ratio"), "generation_aspect_ratio": item.get("generation_aspect_ratio"), "generated_image_size": size(item.get("asset_path")), "foreground_bbox_fill_percent": item.get("foreground_bbox_fill_percent"), "slot_fit_waste_percent": None, "aspect_ratio_match_status": "matched" if item.get("generation_aspect_ratio") else "unknown"} for item in reports]}) + return reports, summary + + +_NAMES = {"_generate_assets", "_load_accepted_assets", "_foreground_metrics", "economy_acceptance_decision", "_crop_reference", "_placeholder_asset", "_api_generate_asset"} + + +def __getattr__(name: str): + if name in _NAMES: + if name == "_generate_assets": + return _generate_assets + from .. import editable_rebuild + + return getattr(editable_rebuild, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = sorted(_NAMES) diff --git a/rfs/generation/rebuild_policy.py b/rfs/generation/rebuild_policy.py new file mode 100644 index 0000000..40a9887 --- /dev/null +++ b/rfs/generation/rebuild_policy.py @@ -0,0 +1,138 @@ +"""Canonical boundary for rebuild slot-asset policy operations. + +The implementation is still compatibility-backed by ``rfs.editable_rebuild`` +in this transition batch. Keeping the lookup lazy avoids importing the large +workflow module while the asset policy implementation is being extracted. +""" + +from __future__ import annotations + +ASSET_THRESHOLDS = { + "character": (0.80, 0.95), + "document_stack": (0.75, 0.95), + "chart_card": (0.75, 0.95), + "inspection": (0.70, 0.95), + "tool_icon": (0.80, 0.95), + "tool_combo": (0.70, 0.95), + "device": (0.70, 0.95), + "screenshot_card": (0.75, 0.95), + "legend_marker": (0.80, 0.95), + "thin_tool": (0.50, 0.95), + "generic": (0.70, 0.95), +} +TEXT_ASSET_OVERLAP_THRESHOLD = 0.60 +SIMPLE_PRIMITIVE_TYPES = {"legend_marker", "simple_marker", "status_marker", "node_marker"} +RASTER_ASSET_POLICIES = {"reference_crop", "api_generate", "placeholder"} +NON_RASTER_ASSET_POLICIES = {"ppt_shape", "editable_text", "ppt_connector", "ignore"} +ASSET_SOURCE_POLICIES = RASTER_ASSET_POLICIES | NON_RASTER_ASSET_POLICIES +ASSET_POLICIES = {"legacy", "smart-api", "generation-policy"} + + +def _supported_aspect_ratio(width: float, height: float) -> str: + ratio = width / max(height, 0.001) + if ratio >= 1.55: return "16:9" + if ratio >= 1.18: return "4:3" + if ratio >= 0.85: return "1:1" + if ratio >= 0.62: return "3:4" + return "9:16" + + +def economy_acceptance_decision(slot_type: str, fill_percent: float, strict: bool = False) -> dict: + low, high = (0.80, 0.95) if strict else ASSET_THRESHOLDS.get(slot_type, ASSET_THRESHOLDS["generic"]) + value = float(fill_percent) + accepted = low <= value <= high + if accepted: + reason = f"fill {value:.3f} within {slot_type} threshold {low:.2f}-{high:.2f}" + elif value < low: + reason = f"fill {value:.3f} below {slot_type} threshold {low:.2f}" + else: + reason = f"fill {value:.3f} above tightness threshold {high:.2f}" + return {"accepted": accepted, "selected_reason": reason, "threshold_min": low, "threshold_max": high} + + +def _slot_type_for_box(slot: dict) -> str: + semantic_type = str(slot.get("asset_type") or "").strip() + if semantic_type: + return semantic_type + ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) + if ratio >= 2.2: + return "thin_tool" + if ratio >= 1.35: + return "chart_card" + if ratio <= 0.78: + return "character" + return "generic" + + +def _threshold_for_type(slot_type: str) -> tuple[float, float]: + return ASSET_THRESHOLDS.get(slot_type, ASSET_THRESHOLDS["generic"]) + + +def _generation_policy_by_slot(generation_plan: dict | None) -> dict[str, dict]: + policies: dict[str, dict] = {} + if not isinstance(generation_plan, dict): + return policies + for item in generation_plan.get("asset_policies", []) or []: + if not isinstance(item, dict): + continue + slot_id = str(item.get("slot_id") or item.get("object_id") or item.get("id") or "") + if slot_id: + policies[slot_id] = item + return policies + + +def _asset_source_policy_for_slot(slot: dict, slot_type: str, generation_plan: dict | None) -> tuple[str, str, str, bool]: + explicit = str(slot.get("asset_source_policy") or "").strip() + if explicit in ASSET_SOURCE_POLICIES: + return explicit, str(slot.get("policy_source") or "slot"), str(slot.get("asset_source_reason") or "slot asset source policy"), True + policy_record = _generation_policy_by_slot(generation_plan).get(str(slot.get("id") or "")) + if policy_record: + policy = str(policy_record.get("policy") or policy_record.get("asset_source_policy") or "").strip() + if policy in ASSET_SOURCE_POLICIES: + return policy, str(policy_record.get("source") or "generation_plan"), str(policy_record.get("reason") or policy_record.get("asset_source_reason") or "global generation policy"), True + if slot_type in SIMPLE_PRIMITIVE_TYPES: + return "ppt_shape", "heuristic", "simple primitive should stay editable", False + return "api_generate", "default", "default complex visual slot policy", False + + +def _effective_asset_mode_for_spec(asset_mode: str, asset_policy: str, spec: dict) -> tuple[str, str]: + if asset_mode != "policy" and asset_policy != "generation-policy": + return asset_mode, "global_override" + policy = str(spec.get("asset_source_policy") or "").strip() + if policy == "reference_crop": + return "crop", "generation_policy" + if policy == "api_generate": + return "api", "generation_policy" + if policy == "placeholder": + return "placeholder", "generation_policy" + return "placeholder", "generation_policy_non_raster" + + +_LEGACY_NAMES = { + "ASSET_POLICIES", + "ASSET_SOURCE_POLICIES", + "NON_RASTER_ASSET_POLICIES", + "RASTER_ASSET_POLICIES", + "TEXT_ASSET_OVERLAP_THRESHOLD", + "_align_generation_plan_to_program", + "_apply_smart_asset_policy", + "_asset_source_policy_for_slot", + "_effective_asset_mode_for_spec", + "_make_asset_specs", + "_generation_policy_by_slot", + "_slot_type_for_box", + "_threshold_for_type", + "_supported_aspect_ratio", + "economy_acceptance_decision", +} + + +def __getattr__(name: str): + if name in _LEGACY_NAMES: + from .. import editable_rebuild + + return getattr(editable_rebuild, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = sorted(_LEGACY_NAMES | {"ASSET_THRESHOLDS", "ASSET_POLICIES", "ASSET_SOURCE_POLICIES", "NON_RASTER_ASSET_POLICIES", "RASTER_ASSET_POLICIES", "SIMPLE_PRIMITIVE_TYPES", "TEXT_ASSET_OVERLAP_THRESHOLD"}) diff --git a/rfs/input_archive.py b/rfs/input_archive.py index 1443f6b..abaf8ad 100644 --- a/rfs/input_archive.py +++ b/rfs/input_archive.py @@ -6,16 +6,16 @@ from .utils import ensure_dir, write_json -def archive_inputs(paper: str | Path, reference: str | Path, out_dir: str | Path) -> dict: +def archive_inputs(paper: str | Path, reference: str | Path | None, out_dir: str | Path) -> dict: out = Path(out_dir) inputs = ensure_dir(out / "inputs") paper_path = Path(paper) - reference_path = Path(reference) + reference_path = Path(reference) if reference is not None else None archived = { "summary": "Archived source inputs for reproducible figure generation.", "paper_original": str(paper_path), - "reference_original": str(reference_path), + "reference_original": str(reference_path) if reference_path is not None else None, "paper_archived": None, "reference_archived": None, } @@ -24,7 +24,7 @@ def archive_inputs(paper: str | Path, reference: str | Path, out_dir: str | Path target = inputs / f"paper{paper_path.suffix.lower() or '.txt'}" shutil.copyfile(paper_path, target) archived["paper_archived"] = str(target) - if reference_path.exists(): + if reference_path is not None and reference_path.exists(): target = inputs / f"reference{reference_path.suffix.lower() or '.png'}" shutil.copyfile(reference_path, target) archived["reference_archived"] = str(target) diff --git a/rfs/judge_training/__init__.py b/rfs/judge_training/__init__.py new file mode 100644 index 0000000..3d5f270 --- /dev/null +++ b/rfs/judge_training/__init__.py @@ -0,0 +1,8 @@ +"""Dataset preparation utilities for the multi-task figure Judge.""" + +from .dataset_builder import build_dataset, records_from_run, validate_dataset +from .schemas import TASK_TYPES, normalize_example, validate_example +from .evaluator import compare_shadow_predictions, evaluate_predictions +from .registry import register_checkpoint, resolve_checkpoint, validate_manifest + +__all__ = ["TASK_TYPES", "build_dataset", "records_from_run", "validate_dataset", "normalize_example", "validate_example", "evaluate_predictions", "compare_shadow_predictions", "register_checkpoint", "resolve_checkpoint", "validate_manifest"] diff --git a/rfs/judge_training/dataset_builder.py b/rfs/judge_training/dataset_builder.py new file mode 100644 index 0000000..77a768e --- /dev/null +++ b/rfs/judge_training/dataset_builder.py @@ -0,0 +1,132 @@ +"""Build JudgeExample JSONL records from completed Paperimage runs.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .schemas import normalize_example, validate_example + + +def _read(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + return value if isinstance(value, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + + +def _jsonl(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + records = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + value = json.loads(line) + if isinstance(value, dict): + records.append(value) + except json.JSONDecodeError: + continue + return records + + +def _ground_truth_id(run: Path, payload: dict[str, Any]) -> str: + for value in (payload.get("ground_truth_id"), payload.get("ground_truth_hash"), payload.get("paper")): + if value: + return str(value) + digest = hashlib.sha1(str(run.resolve()).encode("utf-8")).hexdigest()[:12] + return f"run-{digest}" + + +def _example(run: Path, index: int, task_type: str, candidate: str | None, *, candidate_b: str | None = None, preview: str | None = None, labels: dict[str, Any] | None = None, payload: dict[str, Any] | None = None, object_program: str | None = None) -> dict[str, Any]: + payload = payload or {} + example = { + "example_id": f"{run.name}-{task_type}-{index:04d}", + "dataset_version": "judge-dataset-v001", + "ground_truth_id": _ground_truth_id(run, payload), + "task_type": task_type, + "candidate_a": candidate, + "candidate_b": candidate_b, + "preview": preview, + "object_program": object_program, + "labels": labels or {}, + "provenance": {"run_dir": str(run.resolve()), "source": "paperimage_evolution"}, + "judge_metadata": {"judge_role": "bootstrap", "trained": False, "human_audited": bool(payload.get("human_scientific_confirmed"))}, + } + return normalize_example(example) + + +def records_from_run(run: str | Path) -> list[dict[str, Any]]: + root = Path(run).resolve() + records: list[dict[str, Any]] = [] + whole = _read(root / "paper_to_image" / "whole_image_evolution_report.json") or _read(root / "whole_image_evolution_report.json") + if whole: + candidates = [candidate for candidate in whole.get("candidates", []) if isinstance(candidate, dict)] + selected = next((candidate for candidate in candidates if candidate.get("accepted_as_selected")), None) + for index, candidate in enumerate(candidates, start=1): + labels = {"scientific_score": candidate.get("scientific_score"), "aesthetic_score": candidate.get("aesthetic_score"), "decomposability": candidate.get("decomposability"), "hard_errors": candidate.get("hard_errors", [])} + records.append(_example(root, index, "image_scientific", candidate.get("path"), labels=labels, payload=whole)) + records.append(_example(root, index, "image_aesthetic", candidate.get("path"), labels=labels, payload=whole)) + if selected and candidate is not selected: + preferred = "A" if selected.get("production_pass") else "tie" + records.append(_example(root, index, "image_pairwise", selected.get("path"), candidate_b=candidate.get("path"), labels={"preferred_candidate": preferred, "selected_candidate_id": selected.get("candidate_id"), "alternative_candidate_id": candidate.get("candidate_id")}, payload=whole)) + joint = _read(root / "editable" / "joint_judgement.json") or _read(root / "joint_judgement.json") + if joint: + preview = joint.get("preview") or str(root / "editable" / "rebuild_preview.png") + reference = str(root / "paper_to_image" / "selected_image.png") + if not Path(reference).exists(): + reference = str(root / "selected_image.png") + editability = dict(joint.get("editability") or {}) + editability.setdefault("pptx_path", str(root / "editable" / "editable_composition.pptx")) + program = root / "editable" / "figure_program.json" + records.append(_example(root, len(records) + 1, "ppt_scientific_preservation", reference, preview=preview, labels=joint.get("scientific") or {}, payload=joint)) + records.append(_example(root, len(records) + 1, "ppt_editability", reference, preview=preview, labels=editability, payload=joint, object_program=str(program) if program.exists() else None)) + records.append(_example(root, len(records) + 1, "segmentation_quality", reference, preview=preview, labels=joint.get("segmentation") or {}, payload=joint)) + for index, decision in enumerate(_jsonl(root / "editable" / "repair_decisions.jsonl"), start=1): + records.append(_example(root, index, "repair_effectiveness", str(root / "editable" / "rebuild_preview.png"), labels=decision, payload=decision)) + return records + + +def build_dataset(runs_root: str | Path, out: str | Path, *, dataset_version: str = "judge-dataset-v001") -> dict[str, Any]: + source = Path(runs_root).resolve() + target = Path(out).resolve() + # Accept either a JSONL file or a dataset directory. The directory form + # keeps manifests and future corruption/split artifacts together. + if target.suffix.casefold() != ".jsonl": + target.mkdir(parents=True, exist_ok=True) + target = target / "judge_examples.jsonl" + target.parent.mkdir(parents=True, exist_ok=True) + records = [] + is_run = (source / "paper_to_image").exists() or (source / "editable").exists() or (source / "whole_image_evolution_report.json").exists() + run_dirs = [source] if is_run else [path for path in source.rglob("*") if path.is_dir() and ((path / "paper_to_image").exists() or (path / "editable").exists() or (path / "whole_image_evolution_report.json").exists())] + for run in sorted(set(run_dirs)): + records.extend(records_from_run(run)) + valid = [] + errors = [] + with target.open("w", encoding="utf-8") as handle: + for record in records: + record["dataset_version"] = dataset_version + record_errors = validate_example(record) + if record_errors: + errors.append({"example_id": record.get("example_id"), "errors": record_errors}) + continue + handle.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") + valid.append(record) + manifest = {"summary": "JudgeExample dataset built from Paperimage evolution runs.", "dataset_version": dataset_version, "source_root": str(source), "out": str(target), "records": len(valid), "errors": errors, "task_counts": {task: sum(1 for item in valid if item["task_type"] == task) for task in sorted({item["task_type"] for item in valid})}, "judge_status": "bootstrap_only"} + target.with_suffix(".manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + return manifest + + +def validate_dataset(path: str | Path) -> dict[str, Any]: + source = Path(path) + if source.is_dir(): + source = source / "judge_examples.jsonl" + records = _jsonl(source) + errors = [] + for index, record in enumerate(records, start=1): + item_errors = validate_example(record) + if item_errors: + errors.append({"line": index, "errors": item_errors}) + return {"summary": "JudgeExample dataset validation.", "path": str(source.resolve()), "records": len(records), "errors": errors, "ok": not errors} diff --git a/rfs/judge_training/evaluator.py b/rfs/judge_training/evaluator.py new file mode 100644 index 0000000..c7a8984 --- /dev/null +++ b/rfs/judge_training/evaluator.py @@ -0,0 +1,56 @@ +"""Offline metric aggregation and Shadow-vs-Bootstrap comparison.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def _jsonl(path: str | Path) -> list[dict[str, Any]]: + source = Path(path) + if not source.exists(): + raise FileNotFoundError(source) + rows = [] + for line in source.read_text(encoding="utf-8").splitlines(): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + rows.append(value) + return rows + + +def evaluate_predictions(dataset: str | Path, predictions: str | Path, *, checkpoint: str | Path | None = None, suite: str = "hidden_test") -> dict[str, Any]: + gold = {str(row.get("example_id")): row for row in _jsonl(dataset) if row.get("example_id")} + pred = _jsonl(predictions) + scored = [] + for row in pred: + example_id = str(row.get("example_id") or "") + if example_id not in gold: + continue + labels = gold[example_id].get("labels") if isinstance(gold[example_id].get("labels"), dict) else {} + output = row.get("prediction") if isinstance(row.get("prediction"), dict) else row + if "scientific_score" in labels and "scientific_score" in output: + try: + error = abs(float(labels["scientific_score"]) - float(output["scientific_score"])) + scored.append({"example_id": example_id, "scientific_abs_error": round(error, 4)}) + except (TypeError, ValueError): + pass + mae = round(sum(item["scientific_abs_error"] for item in scored) / len(scored), 4) if scored else None + return {"summary": "Offline Judge prediction evaluation.", "ok": bool(scored), "suite": suite, "checkpoint": str(checkpoint) if checkpoint else None, "dataset_records": len(gold), "prediction_records": len(pred), "scored_records": len(scored), "scientific_mae": mae, "metrics_available": bool(scored), "status": "pass" if scored else "blocked_no_compatible_predictions"} + + +def compare_shadow_predictions(dataset: str | Path, trained_predictions: str | Path, bootstrap_predictions: str | Path, *, checkpoint: str | Path | None = None) -> dict[str, Any]: + trained = {str(row.get("example_id")): row for row in _jsonl(trained_predictions) if row.get("example_id")} + bootstrap = {str(row.get("example_id")): row for row in _jsonl(bootstrap_predictions) if row.get("example_id")} + common = sorted(set(trained) & set(bootstrap)) + disagreements = [] + for example_id in common: + a = trained[example_id].get("prediction", trained[example_id]) + b = bootstrap[example_id].get("prediction", bootstrap[example_id]) + if json.dumps(a, sort_keys=True, ensure_ascii=False) != json.dumps(b, sort_keys=True, ensure_ascii=False): + disagreements.append({"example_id": example_id, "trained": a, "bootstrap": b}) + rate = round(len(disagreements) / len(common), 4) if common else None + return {"summary": "Shadow Judge comparison against Bootstrap predictions.", "ok": bool(common), "checkpoint": str(checkpoint) if checkpoint else None, "common_records": len(common), "disagreement_count": len(disagreements), "disagreement_rate": rate, "disagreements": disagreements, "status": "pass" if common else "blocked_no_common_predictions"} diff --git a/rfs/judge_training/evidence_builder.py b/rfs/judge_training/evidence_builder.py new file mode 100644 index 0000000..2fe2aae --- /dev/null +++ b/rfs/judge_training/evidence_builder.py @@ -0,0 +1,174 @@ +"""Build paper evidence packs for the first Judge dataset iteration. + +The source corpus is read-only. This builder combines a paper PDF, an optional +classification CSV, and an optional selected framework figure into one +auditable JSONL record. It intentionally does not infer scientific ground +truth from the paper; that remains a human/contract annotation step. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +from pathlib import Path +from typing import Any + +import fitz + + +SECTION_RE = re.compile( + r"(?im)^(?P
(?:\d+(?:\.\d+)*\s+)?" + r"(?:introduction|background|problem formulation|problem definition|" + r"method(?:ology|s)?|approach|model|framework|architecture|system|" + r"implementation|experiments?|conclusion|related work)\b.*?)$" +) +CAPTION_RE = re.compile(r"(?is)\b(?:figure|fig\.|图)\s*\d+[.:]?\s*.*?(?=\n\s*\n|\Z)") + + +def _clean(text: str) -> str: + text = re.sub(r"([A-Za-z])[-‐]\s*\n\s*([a-z])", r"\1\2", text) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +def extract_pages(pdf_path: Path) -> list[str]: + with fitz.open(pdf_path) as doc: + pages = [] + for page in doc: + blocks = [b for b in page.get_text("blocks") if b[4].strip()] + blocks.sort(key=lambda b: (round(b[1], 1), round(b[0], 1))) + pages.append(_clean("\n".join(b[4] for b in blocks))) + return pages + + +def extract_title_abstract(pages: list[str], metadata: dict[str, Any]) -> tuple[str, str]: + title = _clean(str(metadata.get("title") or "")) + first = pages[0] if pages else "" + if not title: + title = _clean(first.split("\n", 1)[0]) if first else "" + match = re.search(r"(?is)\babstract\b\s*[:.]?\s*(.*?)(?=\n\s*\d?\.?\s*(?:introduction|1\s+introduction)\b)", first) + abstract = _clean(match.group(1)) if match else "" + return title, abstract + + +def extract_evidence(pdf_path: Path, metadata: dict[str, Any]) -> dict[str, Any]: + pages = extract_pages(pdf_path) + title, abstract = extract_title_abstract(pages, metadata) + full_text = "\n\n".join(f"--- PAGE {i + 1} ---\n{p}" for i, p in enumerate(pages)) + headers = list(SECTION_RE.finditer(full_text)) + sections: dict[str, str] = {} + for index, match in enumerate(headers): + start = match.end() + end = headers[index + 1].start() if index + 1 < len(headers) else len(full_text) + key = re.sub(r"^\d+(?:\.\d+)*\s+", "", match.group("header").strip().lower()) + if any(token in key for token in ("method", "approach", "model", "framework", "architecture", "system", "problem")): + sections[key] = _clean(full_text[start:end]) + captions = [_clean(m.group(0)) for m in CAPTION_RE.finditer(full_text)] + method_text = "\n\n".join(sections.values()) + return { + "title": title, + "abstract": abstract, + "problem_definition": sections.get("problem definition", sections.get("problem formulation", "")), + "method": method_text, + "method_sections": sections, + "figure_captions": captions, + "page_count": len(pages), + } + + +def load_labels(path: Path | None) -> dict[str, dict[str, Any]]: + if path is None: + return {} + with path.open("r", encoding="utf-8-sig", newline="") as handle: + rows = csv.DictReader(handle) + return {str(row.get("arxiv_id") or "").strip(): dict(row) for row in rows if row.get("arxiv_id")} + + +def load_metadata(path: Path | None) -> dict[str, dict[str, Any]]: + if path is None or not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + items = payload if isinstance(payload, list) else payload.get("items", payload.get("papers", [])) + result = {} + for item in items: + if isinstance(item, dict) and item.get("arxiv_id"): + result[str(item["arxiv_id"])] = item + return result + + +def load_selected_ids(path: Path | None) -> list[str] | None: + """Read IDs from a D0 selection manifest or a JSON array.""" + if path is None: + return None + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, list): + return [str(item).strip() for item in payload if str(item).strip()] + records = payload.get("records", []) if isinstance(payload, dict) else [] + return [str(item.get("arxiv_id") or "").strip() for item in records if isinstance(item, dict) and item.get("arxiv_id")] + + +def build_records(source_root: Path, out_path: Path, labels_path: Path | None, figures_root: Path | None, limit: int | None, selected_ids_path: Path | None = None) -> dict[str, Any]: + pdf_dir = source_root / "pdf" + metadata = load_metadata(source_root / "papers.json") + labels = load_labels(labels_path) + selected_ids = load_selected_ids(selected_ids_path) + pdfs = ( + [pdf_dir / f"{arxiv_id}.pdf" for arxiv_id in selected_ids if (pdf_dir / f"{arxiv_id}.pdf").exists()] + if selected_ids is not None + else sorted(pdf_dir.glob("*.pdf")) + ) + if limit: + pdfs = pdfs[:limit] + out_path.parent.mkdir(parents=True, exist_ok=True) + missing_labels: list[str] = [] + records = 0 + with out_path.open("w", encoding="utf-8") as handle: + for pdf in pdfs: + arxiv_id = pdf.stem + label = labels.get(arxiv_id) + if labels_path and label is None: + missing_labels.append(arxiv_id) + continue + figure_path = None + if figures_root: + candidate = figures_root / arxiv_id / "framework.png" + if candidate.exists(): + figure_path = str(candidate) + evidence = extract_evidence(pdf, metadata.get(arxiv_id, {})) + record = { + "example_id": f"paper-{arxiv_id}", + "dataset_version": "judge-d0-evidence-v001", + "ground_truth_id": arxiv_id, + "task_type": "quality_classification", + "paper": {"arxiv_id": arxiv_id, "pdf_path": str(pdf), **evidence}, + "figure": {"path": figure_path, "status": "selected" if figure_path else "missing"}, + "human_labels": label or {}, + "scientific_gt": {"entities": [], "relations": [], "must_show": [], "must_not_invent": [], "terminology": [], "annotation_status": "pending"}, + "provenance": {"source_root": str(source_root), "labels_path": str(labels_path) if labels_path else None}, + } + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + records += 1 + report = {"source_root": str(source_root), "labels_path": str(labels_path) if labels_path else None, "selected_ids_path": str(selected_ids_path) if selected_ids_path else None, "pdf_count_considered": len(pdfs), "records_written": records, "missing_labels": missing_labels, "label_count": len(labels)} + out_path.with_suffix(".report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, default=Path(r"D:\文件夹\papers-dataset")) + parser.add_argument("--labels", type=Path) + parser.add_argument("--figures-root", type=Path) + parser.add_argument("--selected-ids", type=Path, help="D0 selection manifest or JSON array of arXiv IDs.") + parser.add_argument("--out", type=Path, default=Path("datasets/judge-d0/evidence_packs.jsonl")) + parser.add_argument("--limit", type=int) + args = parser.parse_args() + report = build_records(args.source_root, args.out, args.labels, args.figures_root, args.limit, args.selected_ids) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rfs/judge_training/registry.py b/rfs/judge_training/registry.py new file mode 100644 index 0000000..0c96384 --- /dev/null +++ b/rfs/judge_training/registry.py @@ -0,0 +1,75 @@ +"""Checkpoint manifests and role-safe Judge registration. + +The registry stores metadata only; it never loads model weights. This keeps +model serving/training dependencies optional and makes promotion auditable. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + + +ROLES = {"shadow", "online", "frozen", "experimental"} + + +def _read(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + return value if isinstance(value, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + + +def validate_manifest(manifest: dict[str, Any], *, checkpoint: str | Path | None = None) -> list[str]: + errors: list[str] = [] + for key in ("model_id", "dataset_version"): + if not str(manifest.get(key) or "").strip(): + errors.append(f"missing {key}") + roles = manifest.get("roles", []) + if not isinstance(roles, list) or not roles: + errors.append("roles must be a non-empty list") + elif any(str(role) not in ROLES for role in roles): + errors.append("roles contains unsupported value") + promotion = manifest.get("promotion") if isinstance(manifest.get("promotion"), dict) else {} + if any(str(role) in {"online", "frozen"} for role in roles): + for key in ("approved", "hidden_test_passed", "adversarial_passed"): + if promotion.get(key) is not True: + errors.append(f"promotion.{key} must be true for online/frozen") + if "frozen" in roles and promotion.get("shadow_stable") is not True: + errors.append("promotion.shadow_stable must be true for frozen") + if not str(manifest.get("git_commit") or "").strip(): + errors.append("missing git_commit") + if checkpoint is not None and not Path(checkpoint).exists(): + errors.append("checkpoint path does not exist") + return errors + + +def register_checkpoint(checkpoint: str | Path, manifest: dict[str, Any], registry: str | Path) -> dict[str, Any]: + checkpoint_path = Path(checkpoint).resolve() + errors = validate_manifest(manifest, checkpoint=checkpoint_path) + if errors: + raise ValueError("invalid Judge checkpoint manifest: " + "; ".join(errors)) + record = dict(manifest) + record.update({"checkpoint": str(checkpoint_path), "registered_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "registry_status": "registered"}) + registry_path = Path(registry).resolve() + registry_path.parent.mkdir(parents=True, exist_ok=True) + current = _read(registry_path) + entries = current.get("models") if isinstance(current.get("models"), list) else [] + entries = [item for item in entries if isinstance(item, dict) and item.get("model_id") != record["model_id"]] + entries.append(record) + registry_path.write_text(json.dumps({"schema_version": "1.0", "models": entries}, ensure_ascii=False, indent=2), encoding="utf-8") + return record + + +def resolve_checkpoint(model_id: str, registry: str | Path) -> dict[str, Any]: + entries = _read(Path(registry)).get("models", []) + for item in entries if isinstance(entries, list) else []: + if isinstance(item, dict) and str(item.get("model_id")) == model_id: + errors = validate_manifest(item, checkpoint=item.get("checkpoint")) + if errors: + raise ValueError("registered Judge model is invalid: " + "; ".join(errors)) + return item + raise KeyError(f"Judge model is not registered: {model_id}") diff --git a/rfs/judge_training/schemas.py b/rfs/judge_training/schemas.py new file mode 100644 index 0000000..5e33e7a --- /dev/null +++ b/rfs/judge_training/schemas.py @@ -0,0 +1,54 @@ +"""Stable schemas for Judge training and Shadow evaluation records.""" + +from __future__ import annotations + +from typing import Any + + +TASK_TYPES = { + "image_scientific", + "image_aesthetic", + "image_pairwise", + "segmentation_quality", + "ppt_scientific_preservation", + "ppt_editability", + "repair_effectiveness", +} + + +def normalize_example(raw: dict[str, Any]) -> dict[str, Any]: + labels = raw.get("labels") if isinstance(raw.get("labels"), dict) else {} + provenance = raw.get("provenance") if isinstance(raw.get("provenance"), dict) else {} + task_type = str(raw.get("task_type") or "").strip() + if task_type not in TASK_TYPES: + raise ValueError(f"unsupported Judge task_type: {task_type}") + return { + "example_id": str(raw.get("example_id") or ""), + "dataset_version": str(raw.get("dataset_version") or "judge-dataset-v001"), + "ground_truth_id": str(raw.get("ground_truth_id") or ""), + "task_type": task_type, + "paper": raw.get("paper") if isinstance(raw.get("paper"), dict) else {}, + "candidate_a": raw.get("candidate_a"), + "candidate_b": raw.get("candidate_b"), + "preview": raw.get("preview"), + "object_program": raw.get("object_program"), + "labels": labels, + "provenance": provenance, + "judge_metadata": { + "judge_role": str((raw.get("judge_metadata") or {}).get("judge_role") or "bootstrap"), + "trained": bool((raw.get("judge_metadata") or {}).get("trained", False)), + "human_audited": bool((raw.get("judge_metadata") or {}).get("human_audited", False)), + }, + } + + +def validate_example(example: dict[str, Any]) -> list[str]: + errors = [] + for field in ("example_id", "ground_truth_id", "task_type"): + if not str(example.get(field) or "").strip(): + errors.append(f"missing {field}") + if example.get("task_type") not in TASK_TYPES: + errors.append("invalid task_type") + if not example.get("candidate_a") and not example.get("preview"): + errors.append("candidate_a or preview is required") + return errors diff --git a/rfs/judge_training/select_frameworks.py b/rfs/judge_training/select_frameworks.py new file mode 100644 index 0000000..842df3d --- /dev/null +++ b/rfs/judge_training/select_frameworks.py @@ -0,0 +1,84 @@ +"""Use a VLM to select one framework overview from per-paper candidates.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +from pathlib import Path +from typing import Any + +from rfs.providers import call_vlm_json + + +PROMPT = """You receive candidate figures from one academic paper, ordered alphabetically from A onward. +Select the single figure that most clearly presents the paper's overall method, system architecture, +or end-to-end framework: modules/components and their relationships, flow, or pipeline. Do not choose +an experimental plot, ablation, qualitative result, table, or isolated submodule. Return JSON only: +{"choice":"A"|"B"|"C"|"none","reason":"brief evidence-based explanation","confidence":0.0}.""" + + +def selected_records(path: Path) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + return list(payload.get("records", [])) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--selection", type=Path, required=True) + parser.add_argument("--candidates-root", type=Path, required=True) + parser.add_argument("--out-root", type=Path, required=True) + parser.add_argument("--model", default=None) + parser.add_argument("--max-candidates", type=int, default=3) + parser.add_argument("--timeout", type=int, default=int(os.getenv("RFS_VLM_TIMEOUT", "90"))) + parser.add_argument("--retries", type=int, default=int(os.getenv("RFS_VLM_RETRIES", "1"))) + args = parser.parse_args() + + summary: dict[str, Any] = {"total": 0, "selected": 0, "none": 0, "no_candidates": 0, "errors": [], "items": {}} + for row in selected_records(args.selection): + arxiv_id = str(row["arxiv_id"]) + print(f"[select] {summary['total'] + 1}: {arxiv_id}", flush=True) + summary["total"] += 1 + candidate_dir = args.candidates_root / arxiv_id + info_path = candidate_dir / "candidates.json" + out_dir = args.out_root / arxiv_id + meta_path = out_dir / "meta.json" + if meta_path.exists(): + summary["items"][arxiv_id] = "existing" + continue + candidates = json.loads(info_path.read_text(encoding="utf-8")).get("candidates", []) if info_path.exists() else [] + image_paths = [candidate_dir / f"cand_{index}.png" for index in range(1, min(len(candidates), args.max_candidates) + 1)] + image_paths = [path for path in image_paths if path.exists()] + out_dir.mkdir(parents=True, exist_ok=True) + if not image_paths: + meta_path.write_text(json.dumps({"arxiv_id": arxiv_id, "status": "no_candidates", "human_labels": row}, ensure_ascii=False, indent=2), encoding="utf-8") + summary["no_candidates"] += 1 + summary["items"][arxiv_id] = "no_candidates" + continue + try: + result = call_vlm_json(PROMPT, [str(path) for path in image_paths], model=args.model, timeout=args.timeout, retries=args.retries) + choice = str(result.get("choice") or "none").upper().strip() + rank = ord(choice) - ord("A") + 1 if len(choice) == 1 and "A" <= choice <= "Z" else None + meta = {"arxiv_id": arxiv_id, "status": "selected" if rank and rank <= len(image_paths) else "none", "vlm_choice": choice, "vlm_reason": result.get("reason", ""), "vlm_confidence": result.get("confidence"), "candidates": candidates, "human_labels": row} + if rank and rank <= len(image_paths): + shutil.copy2(image_paths[rank - 1], out_dir / "framework.png") + meta["chosen_rank"] = rank + summary["selected"] += 1 + summary["items"][arxiv_id] = "selected" + else: + summary["none"] += 1 + summary["items"][arxiv_id] = "none" + meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") + except Exception as exc: + summary["errors"].append({"arxiv_id": arxiv_id, "error": str(exc)}) + summary["items"][arxiv_id] = "error" + print(f"[select] {arxiv_id}: error={str(exc)[:160]}", flush=True) + args.out_root.mkdir(parents=True, exist_ok=True) + (args.out_root / "_summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rfs/layout_locator.py b/rfs/layout_locator.py index 2eb9967..38fd6ee 100644 --- a/rfs/layout_locator.py +++ b/rfs/layout_locator.py @@ -146,6 +146,7 @@ def _default_arrows(panels: list[dict]) -> list[dict]: "path_percent": [], "label": "", "editable_in": "pptx", + "binding_source": "heuristic_default_panel_flow", }) if any(p["id"] == "shared_resource_library" for p in panels): arrows.append({ @@ -156,6 +157,7 @@ def _default_arrows(panels: list[dict]) -> list[dict]: "path_percent": [], "label": "shared resources", "editable_in": "pptx", + "binding_source": "heuristic_default_resource_bus", }) return arrows diff --git a/rfs/layout_planner.py b/rfs/layout_planner.py index 7d49ace..424e256 100644 --- a/rfs/layout_planner.py +++ b/rfs/layout_planner.py @@ -1,247 +1,13 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Callable - -from PIL import Image, ImageDraw - - -def clamp_bbox(bbox: dict) -> dict[str, float]: - x = max(0.0, min(0.995, float(bbox.get("x", 0.0)))) - y = max(0.0, min(0.995, float(bbox.get("y", 0.0)))) - w = max(0.001, min(float(bbox.get("w", 0.001)), 1.0 - x)) - h = max(0.001, min(float(bbox.get("h", 0.001)), 1.0 - y)) - return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} - - -def canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: - ratio = width_px / max(height_px, 1) - width = max(10.0, min(15.6, 7.5 * ratio)) - return round(width, 3), round(width / ratio, 3) - - -def rgb_hex(rgb: tuple[int, int, int]) -> str: - return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" - - -def estimate_background(image: Image.Image) -> str: - rgb = image.convert("RGB") - w, h = rgb.size - samples = [] - edge = max(1, min(w, h) // 12) - for box in [(0, 0, w, edge), (0, h - edge, w, h), (0, 0, edge, h), (w - edge, 0, w, h)]: - crop = rgb.crop(box).resize((1, 1)) - samples.append(crop.getpixel((0, 0))) - avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) - return rgb_hex(avg) +"""Compatibility alias for :mod:`rfs.analysis.rebuild.layout`. +The module object is aliased so legacy imports and monkeypatches continue to +operate on the canonical implementation. +""" -def dominant_palette(image: Image.Image, count: int = 6) -> list[str]: - small = image.convert("RGB").resize((96, 96)) - colors = small.quantize(colors=count).convert("RGB").getcolors(96 * 96) or [] - colors = sorted(colors, key=lambda item: item[0], reverse=True) - return [rgb_hex(rgb) for _n, rgb in colors[:count]] - - -def _find_visual_components(reference_path: Path, canvas_w: int, canvas_h: int) -> list[dict]: - try: - import cv2 - except Exception: - return [] - image = cv2.imread(str(reference_path)) - if image is None: - return [] - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - blur = cv2.GaussianBlur(gray, (5, 5), 0) - edges = cv2.Canny(blur, 45, 120) - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) - dilated = cv2.dilate(edges, kernel, iterations=2) - contours, _hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - boxes = [] - image_area = canvas_w * canvas_h - for contour in contours: - x, y, w, h = cv2.boundingRect(contour) - area = w * h - if area < image_area * 0.008 or area > image_area * 0.35: - continue - if w < 24 or h < 24: - continue - boxes.append((x, y, w, h)) - boxes.sort(key=lambda item: (item[1], item[0])) - kept: list[tuple[int, int, int, int]] = [] - for box in boxes: - x, y, w, h = box - duplicate = False - for kx, ky, kw, kh in kept: - ix = max(0, min(x + w, kx + kw) - max(x, kx)) - iy = max(0, min(y + h, ky + kh) - max(y, ky)) - if ix * iy / max(w * h, 1) > 0.55: - duplicate = True - break - if not duplicate: - kept.append(box) - if len(kept) >= 16: - break - slots = [] - for idx, (x, y, w, h) in enumerate(kept, start=1): - slots.append({ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - "confidence": 0.55, - "detected_by": "cv_contour", - }) - return slots - - -def _fallback_slots(count: int = 6) -> list[dict]: - boxes = [ - (0.08, 0.26, 0.16, 0.22), - (0.31, 0.26, 0.16, 0.22), - (0.54, 0.26, 0.16, 0.22), - (0.77, 0.26, 0.16, 0.22), - (0.25, 0.58, 0.18, 0.22), - (0.57, 0.58, 0.18, 0.22), - ][:count] - return [{ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": clamp_bbox({"x": x, "y": y, "w": w, "h": h}), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - "confidence": 0.25, - "detected_by": "fallback_grid", - } for idx, (x, y, w, h) in enumerate(boxes, start=1)] - - -def _normalize_items(items: list, prefix: str, canvas_id: str = "reference_canvas") -> list[dict]: - normalized = [] - for idx, item in enumerate(items or [], start=1): - if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): - continue - item_id = str(item.get("id") or f"{prefix}_{idx:02d}") - record = dict(item) - record["id"] = item_id - if prefix == "slot": - record.setdefault("asset_id", item_id) - record.setdefault("paper_concept", item.get("prompt_subject") or item.get("label") or f"visual element {idx}") - record.setdefault("composition_type", "full_frame_icon") - record.setdefault("show_slot_caption", False) - record.setdefault("z_index", 20 + idx) - record.setdefault("panel_id", str(item.get("panel_id") or canvas_id)) - else: - record.setdefault("title", item.get("label") or item_id) - record.setdefault("editable_in", "pptx") - raw_bbox = dict(record["bbox_percent"]) - record["bbox_percent"] = clamp_bbox(record["bbox_percent"]) - try: - raw_normalized = {key: round(float(raw_bbox[key]), 4) for key in ("x", "y", "w", "h")} - except Exception: - raw_normalized = {} - record["bbox_was_clamped"] = record["bbox_percent"] != raw_normalized - record.setdefault("confidence", 0.8) - record.setdefault("detected_by", "vlm") - normalized.append(record) - return normalized - - -def _merge_vlm_layout(base: dict, vlm: dict) -> dict: - merged = dict(base) - panels = _normalize_items(vlm.get("panels", []), "panel") - cards = _normalize_items(vlm.get("cards", []), "card") - slots = _normalize_items(vlm.get("slots", []), "slot") - legends = _normalize_items(vlm.get("legend_regions", []), "legend") - if panels: - merged["panels"] = panels - if slots: - merged["slots"] = slots - merged["cards"] = cards - merged["legend_regions"] = legends - merged["confidence"] = float(vlm.get("confidence") or 0.75) - merged["vlm_status"] = "used" - if vlm.get("_vlm_model"): - merged["vlm_model"] = str(vlm.get("_vlm_model")) - merged.setdefault("warnings", []) - return merged - +from __future__ import annotations -def _draw_overlay(reference_path: Path, out_path: Path, layout: dict) -> None: - with Image.open(reference_path).convert("RGB") as image: - draw = ImageDraw.Draw(image) - w, h = image.size - for collection, color, width in [("panels", "#2D6FB7", 4), ("cards", "#8A5CF6", 3), ("slots", "#E17721", 3), ("legend_regions", "#4B9B52", 3)]: - for item in layout.get(collection, []): - box = item.get("bbox_percent") - if not isinstance(box, dict): - continue - x0 = int(float(box["x"]) * w) - y0 = int(float(box["y"]) * h) - x1 = int((float(box["x"]) + float(box["w"])) * w) - y1 = int((float(box["y"]) + float(box["h"])) * h) - draw.rectangle((x0, y0, x1, y1), outline=color, width=width) - draw.text((x0 + 3, y0 + 3), str(item.get("id") or ""), fill=color) - out_path.parent.mkdir(parents=True, exist_ok=True) - image.save(out_path) +import sys as _sys +from .analysis.rebuild import layout as _implementation -def plan_reference_layout( - reference_path: str | Path, - out_dir: str | Path, - mode: str = "hybrid", - vlm_adapter: Callable[[str | Path, dict], dict] | None = None, -) -> dict: - reference = Path(reference_path) - out = Path(out_dir) - with Image.open(reference).convert("RGB") as image: - canvas_w, canvas_h = image.size - width_in, height_in = canvas_inches(canvas_w, canvas_h) - background = estimate_background(image) - palette = dominant_palette(image) - slots = _find_visual_components(reference, canvas_w, canvas_h) or _fallback_slots() - panel = { - "id": "reference_canvas", - "title": "Editable Rebuild", - "bbox_percent": clamp_bbox({"x": 0.025, "y": 0.065, "w": 0.95, "h": 0.84}), - "editable_in": "pptx", - "confidence": 0.5, - "detected_by": "default_canvas", - } - for slot in slots: - slot["panel_id"] = panel["id"] - layout = { - "summary": "Reference geometry inferred by editable rebuild layout planner.", - "layout_mode": mode, - "status": "pass", - "canvas": {"width_px": canvas_w, "height_px": canvas_h, "width_in": width_in, "height_in": height_in, "background": background}, - "panels": [panel], - "cards": [], - "slots": slots, - "legend_regions": [], - "palette": palette, - "confidence": 0.55 if slots else 0.25, - "vlm_status": "not_requested", - "warnings": [], - } - if mode in {"vlm", "hybrid"}: - if vlm_adapter: - try: - vlm_layout = vlm_adapter(reference, layout) - if isinstance(vlm_layout, dict): - layout = _merge_vlm_layout(layout, vlm_layout) - except Exception as exc: - layout["warnings"].append(f"vlm_layout_failed:{exc}") - layout["vlm_status"] = "fallback" - else: - layout["vlm_status"] = "unavailable_fallback_to_heuristic" - if mode == "vlm": - layout["warnings"].append("layout_mode_vlm_requested_without_adapter") - _draw_overlay(reference, out / "reference_geometry_overlay.png", layout) - return layout +_sys.modules[__name__] = _implementation diff --git a/rfs/paper_to_editable.py b/rfs/paper_to_editable.py new file mode 100644 index 0000000..26d0c68 --- /dev/null +++ b/rfs/paper_to_editable.py @@ -0,0 +1,8 @@ +"""Compatibility import for the paper-to-editable workflow. + +New code should import from :mod:`rfs.workflows`. +""" + +from .workflows.paper_to_editable import run_paper_to_editable + +__all__ = ["run_paper_to_editable"] diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index 0836ed1..c0bdf96 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,5 +1,27 @@ -"""Paper-to-image workflow without PPTX generation.""" +"""Paper analysis and paper-to-image workflows.""" +from .exemplar_selector import select_reference_exemplars +from .preference_feedback import record_paper_image_preference +from .reference_learning import analyze_reference_figures +from .user_style_learning import learn_reference_style +from .inspection import inspect_paper +from .editable_ppt import build_semantic_figure_program, build_visual_substrate_figure_program, compile_semantic_ppt, compile_visual_substrate_ppt +from .preparation import prepare_paper_figure_contract, run_fast_framework_prompt +from .overlay_renderer import normalize_visual_substrate_geometry from .workflow import run_paper_to_image -__all__ = ["run_paper_to_image"] +__all__ = [ + "analyze_reference_figures", + "build_semantic_figure_program", + "build_visual_substrate_figure_program", + "compile_semantic_ppt", + "compile_visual_substrate_ppt", + "inspect_paper", + "learn_reference_style", + "normalize_visual_substrate_geometry", + "prepare_paper_figure_contract", + "record_paper_image_preference", + "run_fast_framework_prompt", + "run_paper_to_image", + "select_reference_exemplars", +] diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 6980b77..e721854 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -1,167 +1,2336 @@ from __future__ import annotations +import hashlib +import json +import os import re +import shutil +import subprocess +import sys +import tempfile +import time +import unicodedata +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from difflib import SequenceMatcher from pathlib import Path -from typing import Any +from typing import Any, Callable + + +MOJIBAKE_MARKERS = ("\ufffd", "Ã", "Â", "â€", "锟", "鈥", "銆", "鏅") +SECTION_ALIASES = { + "abstract": ("abstract", "resumen", "résumé", "zusammenfassung", "resumo", "摘要", "概要", "초록"), + "introduction": ("introduction", "background", "introducción", "introduccion", "introdução", "introducao", "einleitung", "contexte", "contexto", "antecedentes", "引言", "介绍", "背景", "はじめに", "序論", "서론", "배경"), + "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework", "método", "metodo", "métodos", "metodos", "metodología", "metodologia", "méthode", "methode", "méthodes", "methodes", "méthodologie", "methodologie", "methoden", "methodik", "enfoque", "approche", "ansatz", "abordagem", "方法", "方法论", "架构", "系统概述", "框架", "手法", "方法論", "アーキテクチャ", "방법", "방법론", "아키텍처"), + "experiments": ("experiment", "experiments", "evaluation", "results", "experimento", "experimentos", "evaluación", "evaluacion", "resultados", "expérience", "experience", "expériences", "experiences", "évaluation", "experimente", "auswertung", "ergebnisse", "avaliação", "avaliacao", "实验", "评估", "结果", "実験", "評価", "結果", "실험", "평가", "결과"), + "conclusion": ("conclusion", "conclusions", "discussion", "conclusión", "conclusiones", "discusión", "discusion", "conclusão", "conclusao", "conclusões", "conclusoes", "discussão", "discussao", "schlussfolgerung", "schlussfolgerungen", "fazit", "diskussion", "结论", "讨论", "总结", "結論", "考察", "まとめ", "결론", "논의", "요약"), +} + +OCR_PROTECTED_WORDS = { + "acknowledgements", "architecture", "autoregressive", "bidirectional", "classification", "configuration", + "convolutional", "deterministic", "differentiable", "downstream", "embedding", "embeddings", "evaluation", + "experimental", "experiments", "finetuning", "implementation", "information", "initialization", "introduction", + "methodology", "multiframe", "multimodal", "optimization", "performance", "positional", "pretraining", + "probabilities", "representation", "representations", "retrieval", "scientific", "segmentation", "transformer", + "visualization", +} +OCR_SPLIT_ANCHORS = OCR_PROTECTED_WORDS | { + "and", "decoder", "encoder", "evidence", "for", "from", "generation", "into", "method", "model", "of", + "output", "retriever", "the", "to", "with", +} +OCR_RENDER_DPI = 84 +LATIN_CAPTION_PREFIX = re.compile( + r"^(?Pfigure|fig\.?|table|figura|abbildung|tabla|tableau|tabelle)" + r"(?=\s|\d)\s*(?P[a-z]?\d+(?:[.\-]\d+)*|[ivxlcdm]+)\b", + re.IGNORECASE, +) + + +def _caption_prefix_kind(text: str) -> str | None: + value = _clean(text) + match = LATIN_CAPTION_PREFIX.match(value) + if match: + prefix = match.group("prefix").casefold() + return "table" if prefix in {"table", "tabla", "tableau", "tabelle"} else "caption" + if re.match(r"^(图|圖|表)\s*[a-z0-9一二三四五六七八九十]+", value, re.IGNORECASE): + return "caption" if value.startswith(("图", "圖")) else "table" + return None + + +def _repair_mojibake(text: str) -> str: + value = str(text or "") + if not any(marker in value for marker in MOJIBAKE_MARKERS): + return value + + def penalty(candidate: str) -> tuple[int, int, int]: + marker_count = sum(candidate.count(marker) for marker in MOJIBAKE_MARKERS) + replacement_count = candidate.count("\ufffd") + control_count = sum(1 for char in candidate if unicodedata.category(char) == "Cc" and char not in "\n\r\t") + return marker_count, replacement_count, control_count + + best = value + best_penalty = penalty(value) + for encoding in ("cp1252", "latin1", "gbk", "gb18030"): + try: + candidate = value.encode(encoding).decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError, LookupError): + continue + candidate_penalty = penalty(candidate) + if candidate_penalty < best_penalty: + best = candidate + best_penalty = candidate_penalty + return best def _clean(text: str) -> str: - return re.sub(r"[ \t]+", " ", str(text or "")).strip() + value = unicodedata.normalize("NFKC", _repair_mojibake(str(text or ""))).replace("\u00ad", "") + value = value.replace("\r\n", "\n").replace("\r", "\n") + previous = None + while previous != value: + previous = value + value = re.sub(r"\b([A-Z])\s+([A-Z]{2,})\b", r"\1\2", value) + value = re.sub(r"\bANIMAGE\b", "AN IMAGE", value) + value = re.sub(r"\(\s*([A-Z])\s+([A-Z])\s+([A-Z])\s*\)", r"(\1\2\3)", value) + value = re.sub(r"[ \t]+", " ", value) + value = re.sub(r"(?<=\w)-\s+(?=\w)", "", value) + value = re.sub(r"\n{3,}", "\n\n", value) + return value.strip() -def _read_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any]], str]: - suffix = path.suffix.lower() - pages: list[dict[str, Any]] = [] - loader = "plain" - if suffix == ".pdf": - loader = "pymupdf" +def _repair_ocr_spacing(text: str, splitter: Callable[[str], list[str]] | None = None) -> tuple[str, int]: + value = _clean(text) + repairs = 0 + value, count = re.subn(r"\b(figure|fig\.|table|figura|abbildung|tabla|tableau|tabelle)(?=\d)", r"\1 ", value, flags=re.IGNORECASE) + repairs += count + section_terms = ( + "abstract|introduction|background|method|methods|approach|experiments|results|conclusion|discussion|references|appendix|" + "resumen|introducción|introduccion|método|metodo|métodos|metodos|metodología|metodologia|enfoque|" + "experimento|experimentos|evaluación|evaluacion|resultados|conclusión|conclusiones|discusión|discusion|" + "résumé|méthode|méthodes|expérience|expériences|évaluation|résultats|" + "zusammenfassung|einleitung|methoden|methodik|experimente|ergebnisse|schlussfolgerung|fazit|" + "resumo|introdução|introducao|avaliação|avaliacao|conclusão|conclusao" + ) + value, count = re.subn(rf"(? str: + nonlocal repairs + token = match.group(0) + normalized = token.casefold() + if normalized in OCR_PROTECTED_WORDS or token.isupper() or (token[1:] != token[1:].lower() and token[1:] != token[1:].upper()): + return token + pieces = [str(piece) for piece in splitter(token) if str(piece)] + lowered = [piece.casefold() for piece in pieces] + anchor_count = sum(piece in OCR_SPLIT_ANCHORS for piece in lowered) + if len(pieces) < 2 or any(len(piece) < 2 for piece in pieces) or anchor_count < 2: + return token + repairs += len(pieces) - 1 + return " ".join(pieces) + + value = re.sub(r"[A-Za-z]{8,160}", split_token, value) + return _clean(value), repairs + + +def _normalized_compare_text(text: str) -> str: + return "".join(char for char in _clean(text).casefold() if char.isalnum()) + + +def _lexical_units(text: str) -> list[str]: + clean = _clean(text).casefold() + units = [f"latin:{value}" for value in re.findall(r"[a-z]{3,}", clean)] + cjk = "".join(re.findall(r"[\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]", clean)) + if len(cjk) == 1: + units.append(f"cjk:{cjk}") + else: + units.extend(f"cjk:{cjk[index:index + 2]}" for index in range(len(cjk) - 1)) + return units + + +def _cjk_character_count(text: str) -> int: + return sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in str(text or "") + ) + + +def _token_overlap_agreement(left: str, right: str) -> float | None: + left_tokens = _lexical_units(left) + right_tokens = _lexical_units(right) + if not left_tokens or not right_tokens: + return None + overlap = sum((Counter(left_tokens) & Counter(right_tokens)).values()) + return round(overlap / max(1, len(left_tokens)), 4) + + +def _replacement_rate(text: str) -> float: + value = str(text or "") + return round(value.count("\ufffd") / max(1, len(value)), 6) + + +def _mojibake_rate(text: str) -> float: + value = str(text or "") + return round(sum(value.count(marker) for marker in MOJIBAKE_MARKERS) / max(1, len(value)), 6) + + +def _looks_like_typographic_heading(text: str) -> bool: + value = _clean(text) + words = re.findall(r"[A-Za-z][A-Za-z0-9+&/-]*", value) + first_alpha = next((char for char in value if char.isalpha()), "") + first_char = value[0] if value else "" + if not 2 <= len(value) <= 100 or not 1 <= len(words) <= 10: + return False + if not first_char.isalpha() or (first_char.isascii() and not re.search(r"[a-z]", value)): + return False + if re.match(r"^(?:figure|fig\.|table)\b", value, re.IGNORECASE) or re.search(r"[=<>]{1,2}|\bdoi\b|https?://", value, re.IGNORECASE): + return False + if value.endswith(":") and len(words) <= 2: + return False + if value.endswith((",", ";", "?", "!")) or (value.endswith(".") and len(words) > 5): + return False + if first_alpha and first_alpha.lower() != first_alpha.upper() and not first_alpha.isupper(): + return False + return True + + +def _block_kind(text: str, width_ratio: float = 0.0) -> str: + value = _clean(text) + caption_kind = _caption_prefix_kind(value) + if caption_kind: + return caption_kind + section_aliases = tuple(alias.casefold() for aliases in SECTION_ALIASES.values() for alias in aliases) + if len(value) <= 100 and any(value.casefold() == alias or value.casefold().startswith((f"{alias} ", f"{alias}:", f"{alias}：")) for alias in section_aliases): + return "heading" + if re.match(r"^(abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)(.{2,100})$", value, re.IGNORECASE) and len(value) <= 140: + return "heading" + if len(value) <= 180 and width_ratio >= 0.55 and not value.endswith((".", ",", ";")): + return "title" + if re.search(r"(?:[A-Za-z][A-Za-z0-9_{}^\-]*\s*=\s*[^=\n]{3,}|∑|∫|≤|≥|≈)", value): + return "formula" + return "paragraph" + + +def _column_groups(items: list[dict[str, Any]], page_width: float, max_columns: int = 3) -> list[list[dict[str, Any]]]: + if len(items) < 4: + return [items] + positioned = sorted(items, key=lambda item: float(item["bbox"][0])) + gaps = [ + (float(positioned[index + 1]["bbox"][0]) - float(positioned[index]["bbox"][0]), index) + for index in range(len(positioned) - 1) + ] + qualified_gaps = [(gap, index) for gap, index in sorted(gaps, reverse=True) if gap >= page_width * 0.12] + if not qualified_gaps: + return [items] + + def median(values: list[float]) -> float: + ordered = sorted(values) + middle = len(ordered) // 2 + return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2.0 + + def validated_groups(split_count: int) -> list[list[dict[str, Any]]] | None: + split_indexes = sorted(index for _, index in qualified_gaps[:split_count]) + groups: list[list[dict[str, Any]]] = [] + start = 0 + for split in split_indexes: + groups.append(positioned[start:split + 1]) + start = split + 1 + groups.append(positioned[start:]) + if len(groups) > max_columns or any(len(group) < 2 for group in groups): + return None + ordered_groups = sorted(groups, key=lambda group: median([float(item["bbox"][0]) for item in group])) + minimum_vertical_span = page_width * 0.12 + minimum_text = 40 + for group in ordered_groups: + vertical_span = max(float(item["bbox"][3]) for item in group) - min(float(item["bbox"][1]) for item in group) + text_count = sum(len(_normalized_compare_text(str(item.get("text") or ""))) for item in group) + if vertical_span < minimum_vertical_span or text_count < minimum_text: + return None + for left, right in zip(ordered_groups, ordered_groups[1:]): + left_right = median([float(item["bbox"][2]) for item in left]) + right_left = median([float(item["bbox"][0]) for item in right]) + if left_right > right_left + page_width * 0.04: + return None + return ordered_groups + + for split_count in range(min(max_columns - 1, len(qualified_gaps)), 0, -1): + groups = validated_groups(split_count) + if groups: + return groups + return [items] + + +def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[list[dict[str, Any]], float]: + if not blocks: + return [], 0.0 + usable = [item for item in blocks if _clean(item.get("text", ""))] + if not usable: + return [], 0.0 + spanning = [item for item in usable if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.56] + non_spanning = [item for item in usable if item not in spanning] + detection_items = [ + item + for item in non_spanning + if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.20 + and len(_normalized_compare_text(str(item.get("text") or ""))) >= 20 + ] + detected_columns = _column_groups(detection_items, page_width) + if len(detected_columns) == 1: + ordered = sorted(usable, key=lambda item: (round(float(item["bbox"][1]), 1), float(item["bbox"][0]))) + confidence = 0.96 + for item in ordered: + item["column"] = 0 + else: + def median_x0(column: list[dict[str, Any]]) -> float: + values = sorted(float(item["bbox"][0]) for item in column) + middle = len(values) // 2 + return values[middle] if len(values) % 2 else (values[middle - 1] + values[middle]) / 2.0 + + anchors = [median_x0(column) for column in detected_columns] + columns = [[] for _ in anchors] + for item in non_spanning: + column_index = min(range(len(anchors)), key=lambda index: abs(float(item["bbox"][0]) - anchors[index])) + columns[column_index].append(item) + for column_index, column in enumerate(columns): + for item in column: + item["column"] = column_index + for item in spanning: + item["column"] = -1 + boundaries = sorted(spanning, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) + ordered = [] + cursor = float("-inf") + for boundary in boundaries + [None]: + limit = float(boundary["bbox"][1]) if boundary else float("inf") + segment = [item for item in usable if item not in spanning and cursor <= float(item["bbox"][1]) < limit] + for column in columns: + ordered.extend(sorted((item for item in segment if item in column), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) + if boundary: + ordered.append(boundary) + cursor = float(boundary["bbox"][3]) + seen: set[int] = set() + ordered = [item for item in ordered if not (id(item) in seen or seen.add(id(item)))] + confidence = (0.88 if spanning else 0.82) if len(columns) == 2 else (0.84 if spanning else 0.78) + for index, item in enumerate(ordered, 1): + item["reading_order"] = index + return ordered, confidence + + +def _rotate_blocks_to_page(blocks: list[dict[str, Any]], page: Any) -> list[dict[str, Any]]: + if not int(getattr(page, "rotation", 0) or 0): + return blocks + try: + import fitz + + matrix = page.rotation_matrix + rotated = [] + for item in blocks: + value = dict(item) + rect = fitz.Rect(*item["bbox"]) * matrix + value["bbox"] = [round(float(rect.x0), 3), round(float(rect.y0), 3), round(float(rect.x1), 3), round(float(rect.y1), 3)] + rotated.append(value) + return rotated + except Exception: + return blocks + + +def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: + records = [] + payload = page.get_text("dict") + for parent_index, block in enumerate(payload.get("blocks", []), 1): + if int(block.get("type", 0)) != 0: + continue + for line in block.get("lines", []): + spans = [span for span in line.get("spans", []) if _clean(span.get("text", ""))] + if not spans: + continue + spans.sort(key=lambda span: float(span.get("bbox", [0, 0, 0, 0])[0])) + text = _clean(" ".join(str(span.get("text") or "") for span in spans)) + bbox_values = [span.get("bbox", [0, 0, 0, 0]) for span in spans] + bbox = [ + round(min(float(value[0]) for value in bbox_values), 3), + round(min(float(value[1]) for value in bbox_values), 3), + round(max(float(value[2]) for value in bbox_values), 3), + round(max(float(value[3]) for value in bbox_values), 3), + ] + sizes = [float(span.get("size") or 0.0) for span in spans if float(span.get("size") or 0.0) > 0] + span_bold = [ + bool(int(span.get("flags") or 0) & 16) or bool(re.search(r"(?:bold|demi|semi|medi)", str(span.get("font") or ""), re.IGNORECASE)) + for span in spans + ] + span_lengths = [max(1, len(_clean(span.get("text", "")))) for span in spans] + bold_characters = sum(length for length, bold in zip(span_lengths, span_bold) if bold) + total_characters = sum(span_lengths) + font_bold_ratio = round(bold_characters / max(1, total_characters), 4) + font_bold = font_bold_ratio >= 0.8 + records.append({ + "bbox": bbox, + "text": text, + "source": "pymupdf", + "confidence": 1.0, + "parent_block": parent_index, + "font_size": round(max(sizes), 3) if sizes else None, + "font_bold": font_bold, + "font_bold_ratio": font_bold_ratio, + }) + return records + + +def _merge_native_hyphenated_lines(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]: + merged: list[dict[str, Any]] = [] + repairs = 0 + for record in records: + current = dict(record) + if not merged: + merged.append(current) + continue + previous = merged[-1] + previous_text = str(previous.get("text") or "") + current_text = str(current.get("text") or "") + match_left = re.search(r"([A-Za-z]{2,})-$", previous_text) + match_right = re.match(r"([A-Za-z]{2,})", current_text) + same_parent = previous.get("parent_block") is not None and previous.get("parent_block") == current.get("parent_block") + if not same_parent or not match_left or not match_right: + merged.append(current) + continue + combined_word = f"{match_left.group(1)}{match_right.group(1)}".casefold() + separator = "" if combined_word in OCR_PROTECTED_WORDS or len(match_left.group(1)) <= 3 else "-" + joined_text = f"{previous_text[:-1]}{separator}{current_text.lstrip()}" + previous_bbox = previous.get("bbox") or [0, 0, 0, 0] + current_bbox = current.get("bbox") or previous_bbox + previous.update({ + "text": _clean(joined_text), + "bbox": [ + round(min(float(previous_bbox[0]), float(current_bbox[0])), 3), + round(min(float(previous_bbox[1]), float(current_bbox[1])), 3), + round(max(float(previous_bbox[2]), float(current_bbox[2])), 3), + round(max(float(previous_bbox[3]), float(current_bbox[3])), 3), + ], + "confidence": min(float(previous.get("confidence") or 1.0), float(current.get("confidence") or 1.0)), + "font_size": max(float(previous.get("font_size") or 0.0), float(current.get("font_size") or 0.0)) or None, + "font_bold": bool(previous.get("font_bold")) and bool(current.get("font_bold")), + "native_hyphenation_repairs": int(previous.get("native_hyphenation_repairs") or 0) + 1, + }) + repairs += 1 + return merged, repairs + + +def _poppler_pages(path: Path, timeout: int = 30) -> tuple[list[str], str | None]: + executable = shutil.which("pdftotext") + if not executable: + return [], "pdftotext unavailable" + try: + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / "paper.txt" + subprocess.run([executable, "-layout", str(path), str(target)], check=True, timeout=timeout, capture_output=True) + return target.read_text(encoding="utf-8", errors="replace").split("\f"), None + except Exception as exc: + return [], str(exc) + + +def _pdfplumber_blocks(path: Path, page_number: int) -> list[dict[str, Any]]: + try: + import pdfplumber + + with pdfplumber.open(str(path)) as document: + page = document.pages[page_number - 1] + words = page.extract_words(use_text_flow=False, keep_blank_chars=False) or [] + rows: list[list[dict[str, Any]]] = [] + for word in sorted(words, key=lambda item: (float(item.get("top", 0.0)), float(item.get("x0", 0.0)))): + row = next((items for items in rows if abs(float(items[0].get("top", 0.0)) - float(word.get("top", 0.0))) <= 3.0), None) + if row is None: + row = [] + rows.append(row) + row.append(word) + blocks = [] + for row in rows: + row.sort(key=lambda item: float(item.get("x0", 0.0))) + text = _clean(" ".join(str(item.get("text") or "") for item in row)) + if text: + blocks.append({ + "bbox": [min(float(item["x0"]) for item in row), min(float(item["top"]) for item in row), max(float(item["x1"]) for item in row), max(float(item["bottom"]) for item in row)], + "text": text, + "source": "pdfplumber", + "confidence": 0.92, + }) + return blocks + except Exception: + return [] + + +def _pdf_metadata(path: Path) -> dict[str, Any]: + try: + from pypdf import PdfReader + + reader = PdfReader(str(path)) + if reader.is_encrypted: + try: + unlocked = reader.decrypt("") + except Exception: + unlocked = 0 + if not unlocked: + raise ValueError("PDF is encrypted and cannot be opened without a password") + return {"page_count": len(reader.pages), "encrypted": bool(reader.is_encrypted), "metadata": {str(key): str(value) for key, value in (reader.metadata or {}).items()}} + except ImportError: + return {"page_count": None, "encrypted": None, "metadata": {}, "warning": "pypdf unavailable"} + + +def _validate_pdf_container(path: Path) -> None: + size = path.stat().st_size + if size < 8: + raise ValueError("PDF is empty or too small to contain a valid document") + with path.open("rb") as handle: + header = handle.read(min(1024, size)) + handle.seek(max(0, size - 4096)) + tail = handle.read() + if b"%PDF-" not in header: + raise ValueError("File does not contain a valid PDF header") + if b"%%EOF" not in tail: + raise ValueError("PDF appears truncated because the EOF marker is missing") + + +def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, dict[str, Any]]: + if adapter: + return list(adapter(image_path, lang) or []), "adapter", {} + from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr_detailed + + requested = engine + if requested == "auto": + requested = "rapidocr" + try: + import rapidocr_onnxruntime # noqa: F401 + except Exception: + requested = "easyocr" + try: + import easyocr # noqa: F401 + except Exception: + requested = "paddle" + if requested == "rapidocr": + records, diagnostics = run_rapidocr_detailed(image_path, lang, threads=rapidocr_threads) + return records, "rapidocr", diagnostics + if requested == "easyocr": + return run_easyocr(image_path, lang), "easyocr", {} + if requested == "paddle": + return run_paddle_ocr(image_path, lang), "paddle", {} + return [], "off", {} + + +def _group_ocr_words(records: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: + words = [] + for record in records: + quad = record.get("quad") or [] + if len(quad) < 4 or not _clean(record.get("text", "")): + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + words.append({ + "bbox": [min(xs), min(ys), max(xs), max(ys)], + "text": _clean(record.get("text", "")), + "confidence": float(record.get("confidence") or 0.0), + }) + words.sort(key=lambda item: ((item["bbox"][1] + item["bbox"][3]) / 2.0, item["bbox"][0])) + lines: list[dict[str, Any]] = [] + for word in words: + bbox = word["bbox"] + center_y = (bbox[1] + bbox[3]) / 2.0 + height = max(1.0, bbox[3] - bbox[1]) + line = next(( + item for item in reversed(lines[-12:]) + if abs(center_y - item["center_y"]) <= max(height, item["mean_height"]) * 0.62 + and bbox[0] >= item["bbox"][0] - image_width * 0.08 + ), None) + if line is None: + lines.append({"words": [word], "bbox": list(bbox), "center_y": center_y, "mean_height": height}) + continue + line["words"].append(word) + line["bbox"] = [min(line["bbox"][0], bbox[0]), min(line["bbox"][1], bbox[1]), max(line["bbox"][2], bbox[2]), max(line["bbox"][3], bbox[3])] + line["center_y"] = sum((item["bbox"][1] + item["bbox"][3]) / 2.0 for item in line["words"]) / len(line["words"]) + line["mean_height"] = sum(max(1.0, item["bbox"][3] - item["bbox"][1]) for item in line["words"]) / len(line["words"]) + + grouped = [] + for line in lines: + line["words"].sort(key=lambda item: item["bbox"][0]) + grouped.append({ + "bbox": line["bbox"], + "text": _clean(" ".join(item["text"] for item in line["words"])), + "confidence": round(sum(item["confidence"] for item in line["words"]) / max(1, len(line["words"])), 4), + }) + grouped.sort(key=lambda item: (item["bbox"][1], item["bbox"][0])) + heights = sorted(max(1.0, item["bbox"][3] - item["bbox"][1]) for item in grouped) + median_height = heights[len(heights) // 2] if heights else 12.0 + parent = 0 + previous_by_column: dict[int, dict[str, Any]] = {} + for item in grouped: + center_x = (item["bbox"][0] + item["bbox"][2]) / 2.0 + width = item["bbox"][2] - item["bbox"][0] + column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 + previous = previous_by_column.get(column) + gap = item["bbox"][1] - previous["bbox"][3] if previous else float("inf") + starts_new = bool( + re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+)\b", item["text"], re.IGNORECASE) + or _caption_prefix_kind(item["text"]) + ) + if previous is None or gap > median_height * 1.8 or starts_new: + parent += 1 + item["parent_block"] = parent + previous_by_column[column] = item + return grouped + + +def _assign_ocr_parent_blocks(lines: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: + ordered = sorted(lines, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) + heights = sorted(max(1.0, float(item["bbox"][3]) - float(item["bbox"][1])) for item in ordered) + median_height = heights[len(heights) // 2] if heights else 12.0 + parent = 0 + previous_by_column: dict[int, dict[str, Any]] = {} + last_item: dict[str, Any] | None = None + for item in ordered: + center_x = (float(item["bbox"][0]) + float(item["bbox"][2])) / 2.0 + width = float(item["bbox"][2]) - float(item["bbox"][0]) + column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 + previous = previous_by_column.get(column) + if last_item and last_item.get("_caption_group"): + last_gap = float(item["bbox"][1]) - float(last_item["bbox"][3]) + last_height = max( + median_height, + float(item["bbox"][3]) - float(item["bbox"][1]), + float(last_item["bbox"][3]) - float(last_item["bbox"][1]), + ) + if last_gap <= last_height * 1.15: + previous = last_item + gap = float(item["bbox"][1]) - float(previous["bbox"][3]) if previous else float("inf") + local_height = max( + median_height, + float(item["bbox"][3]) - float(item["bbox"][1]), + (float(previous["bbox"][3]) - float(previous["bbox"][1])) if previous else 0.0, + ) + text = str(item.get("text") or "") + starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+)\b", text, re.IGNORECASE) or _caption_prefix_kind(text)) + starts_caption = bool(_caption_prefix_kind(text)) + continuing_caption = bool(previous and previous.get("_caption_group") and gap <= local_height * 1.15 and not starts_new) + paragraph_break = bool(previous and not continuing_caption and str(previous.get("text") or "").rstrip().endswith((".", ":")) and gap > local_height * 0.45) + if previous is None or gap > local_height * 1.15 or starts_new or paragraph_break: + parent += 1 + item["parent_block"] = parent + item["_caption_group"] = starts_caption or continuing_caption + previous_by_column[column] = item + last_item = item + return ordered + + +def _filter_ocr_margin_noise(records: list[dict[str, Any]], image_width: float) -> tuple[list[dict[str, Any]], int]: + anchors = [ + item + for item in records + if len(_normalized_compare_text(str(item.get("text") or ""))) >= 20 + and float(item["bbox"][2]) - float(item["bbox"][0]) >= image_width * 0.30 + ] + if len(anchors) < 3: + return records, 0 + left = sorted(float(item["bbox"][0]) for item in anchors)[max(0, len(anchors) // 10 - 1)] + right_values = sorted(float(item["bbox"][2]) for item in anchors) + right = right_values[min(len(right_values) - 1, len(right_values) - max(1, len(anchors) // 10))] + margin = image_width * 0.035 + kept = [] + removed = 0 + for item in records: + bbox = item.get("bbox") or [0, 0, 0, 0] + outside = float(bbox[2]) < left - margin or float(bbox[0]) > right + margin + width = max(1.0, float(bbox[2]) - float(bbox[0])) + height = max(1.0, float(bbox[3]) - float(bbox[1])) + short = len(_normalized_compare_text(str(item.get("text") or ""))) < 12 + vertical = height > width * 1.15 + outer_band = float(bbox[2]) < image_width * 0.12 or float(bbox[0]) > image_width * 0.88 + compact_margin_fragment = outer_band and short and (vertical or width < image_width * 0.08) + if (outside and (short or vertical)) or compact_margin_fragment: + removed += 1 + continue + kept.append(item) + return kept, removed + + +def _repeated_margin_signature(text: str) -> str: + value = _clean(text).casefold() + value = re.sub(r"\d+", "#", value) + value = re.sub(r"\s+", " ", value).strip(" |.-_") + return value + + +def _semantic_margin_region(page: dict[str, Any], bbox: list[float] | tuple[float, ...]) -> str | None: + width = float(page.get("width") or 0.0) + height = float(page.get("height") or 0.0) + if width <= 0 or height <= 0: + return None + rotation = int(page.get("rotation") or 0) % 360 + if rotation == 90: + if float(bbox[0]) >= width * 0.88: + return "header" + if float(bbox[2]) <= width * 0.12: + return "footer" + elif rotation == 180: + if float(bbox[1]) >= height * 0.88: + return "header" + if float(bbox[3]) <= height * 0.12: + return "footer" + elif rotation == 270: + if float(bbox[2]) <= width * 0.12: + return "header" + if float(bbox[0]) >= width * 0.88: + return "footer" + else: + if float(bbox[3]) <= height * 0.12: + return "header" + if float(bbox[1]) >= height * 0.88: + return "footer" + return None + + +def _remove_repeated_margin_noise(pages: list[dict[str, Any]]) -> int: + if len(pages) < 3: + return 0 + occurrences: dict[tuple[str, str], set[int]] = {} + candidates: dict[tuple[int, str], tuple[str, str]] = {} + for page in pages: + page_number = int(page.get("page") or 0) + for block in page.get("blocks", []): + bbox = block.get("bbox") + text = str(block.get("text") or "").strip() + if str(block.get("kind") or "") == "heading" or bool(block.get("font_bold")): + continue + if not isinstance(bbox, (list, tuple)) or len(bbox) != 4 or not text or len(text) > 140 or len(_lexical_units(text)) > 12 or _cjk_character_count(text) > 36: + continue + region = _semantic_margin_region(page, bbox) + if not region: + continue + signature = _repeated_margin_signature(text) + if not signature: + continue + key = (region, signature) + occurrences.setdefault(key, set()).add(page_number) + candidates[(page_number, str(block.get("id") or ""))] = key + + threshold = max(3, (len(pages) + 3) // 4) + repeated = {key for key, page_numbers in occurrences.items() if len(page_numbers) >= threshold} + if not repeated: + return 0 + + removed = 0 + for page in pages: + page_number = int(page.get("page") or 0) + kept = [] + page_removed = 0 + for block in page.get("blocks", []): + key = candidates.get((page_number, str(block.get("id") or ""))) + if key in repeated: + page_removed += 1 + continue + kept.append(block) + if not page_removed: + continue + for order, block in enumerate(kept, 1): + block["reading_order"] = order + page_text = "\n\n".join(str(item.get("text") or "") for item in kept) + page.update({ + "blocks": kept, + "text": page_text, + "char_count": len(page_text), + "replacement_character_rate": _replacement_rate(page_text), + "mojibake_rate": _mojibake_rate(page_text), + "column_count": max((int(item.get("column", 0)) for item in kept), default=0) + 1, + "repeated_margin_noise_removed": page_removed, + }) + removed += page_removed + return removed + + +def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, str | None, dict[str, Any]]: + try: + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / f"page_{page_number:03d}.png" + dpi = OCR_RENDER_DPI if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 + render_started = time.monotonic() + pixmap = page.get_pixmap(dpi=dpi, alpha=False) + pixmap.save(str(target)) + render_seconds = time.monotonic() - render_started + records, used_engine, engine_diagnostics = _ocr_records(target, engine, lang, adapter, rapidocr_threads=rapidocr_threads) + postprocess_started = time.monotonic() + spacing_repairs = 0 + for record in records: + repaired, count = _repair_ocr_spacing(record.get("text", "")) + record["text"] = repaired + spacing_repairs += count + if used_engine != "rapidocr": + records = _group_ocr_words(records, float(pixmap.width)) + else: + normalized_records = [] + for record in records: + quad = record.get("quad") or [] + if len(quad) < 4: + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + normalized_records.append({"bbox": [min(xs), min(ys), max(xs), max(ys)], "text": _clean(record.get("text", "")), "confidence": float(record.get("confidence") or 0.0)}) + records = _assign_ocr_parent_blocks(normalized_records, float(pixmap.width)) + records, margin_noise_removed = _filter_ocr_margin_noise(records, float(pixmap.width)) + blocks = [] + scale_x = float(page.rect.width) / max(1, pixmap.width) + scale_y = float(page.rect.height) / max(1, pixmap.height) + for record in records: + raw_bbox = record.get("bbox") + if not raw_bbox: + continue + blocks.append({ + "bbox": [float(raw_bbox[0]) * scale_x, float(raw_bbox[1]) * scale_y, float(raw_bbox[2]) * scale_x, float(raw_bbox[3]) * scale_y], + "text": _clean(record.get("text", "")), + "source": used_engine, + "confidence": round(float(record.get("confidence") or 0.0), 4), + "parent_block": record.get("parent_block"), + }) + return blocks, used_engine, None, { + "margin_noise_removed": margin_noise_removed, + "spacing_repairs": spacing_repairs, + "render_seconds": round(render_seconds, 4), + "render_dpi": dpi, + "postprocess_seconds": round(time.monotonic() - postprocess_started, 4), + **engine_diagnostics, + } + except Exception as exc: + return [], engine, str(exc), {"margin_noise_removed": 0, "spacing_repairs": 0} + + +def _rapidocr_worker_count(engine: str, adapter: Callable | None, page_count: int) -> int: + if adapter is not None or page_count < 2 or engine not in {"auto", "rapidocr"}: + return 1 + if engine == "auto": try: - import fitz - - document = fitz.open(str(path)) - used = 0 - for index, page in enumerate(document, 1): - text = page.get_text("text") - remaining = max_chars - used - if remaining <= 0: + import rapidocr_onnxruntime # noqa: F401 + except Exception: + return 1 + cpu_count = int(os.cpu_count() or 1) + default_workers = 4 if cpu_count >= 8 else 2 if cpu_count >= 4 else 1 + try: + configured = int(os.getenv("RFS_OCR_WORKERS") or default_workers) + except ValueError: + configured = default_workers + return max(1, min(int(page_count), configured, cpu_count)) + + +def _deadline_ocr_wave( + path: Path, + page_numbers: list[int], + engine: str, + lang: str, + deadline_at: float, +) -> list[tuple[int, tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]]: + """Run OCR pages in killable child processes and preserve completed results.""" + cutoff = max(time.monotonic(), float(deadline_at) - 20.0) + worker_threads = 2 if len(page_numbers) == 1 and engine in {"auto", "rapidocr"} else 1 + creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0 + with tempfile.TemporaryDirectory() as temp: + processes: dict[int, dict[str, Any]] = {} + for page_number in page_numbers: + output = Path(temp) / f"page_{page_number:03d}.json" + command = [ + sys.executable, + "-m", + "rfs.paper_to_image.ocr_worker", + "--paper", + str(path), + "--page", + str(page_number), + "--engine", + engine, + "--lang", + lang, + "--threads", + str(worker_threads), + "--out", + str(output), + ] + started = time.monotonic() + try: + process = subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + creationflags=creation_flags, + ) + processes[page_number] = {"process": process, "output": output, "started": started, "launch_error": None} + except Exception as exc: + processes[page_number] = {"process": None, "output": output, "started": started, "launch_error": str(exc)} + + pending = {page for page, item in processes.items() if item["process"] is not None} + # Always harvest workers that completed during process launch before + # consulting the cutoff. Under a loaded test or production host, the + # launch loop itself may consume the tiny remaining budget even though + # an early page has already produced a valid result. + initially_finished = { + page for page in pending + if processes[page]["process"].poll() is not None + } + pending.difference_update(initially_finished) + while pending and time.monotonic() < cutoff: + finished = {page for page in pending if processes[page]["process"].poll() is not None} + pending.difference_update(finished) + if pending: + time.sleep(0.02) + + timed_out = set(pending) + for page_number in timed_out: + process = processes[page_number]["process"] + try: + process.terminate() + except Exception: + pass + for page_number in timed_out: + process = processes[page_number]["process"] + try: + process.wait(timeout=1.0) + except Exception: + try: + process.kill() + process.wait(timeout=1.0) + except Exception: + pass + + results = [] + for page_number in page_numbers: + item = processes[page_number] + process = item["process"] + elapsed = time.monotonic() - float(item["started"]) + stderr = "" + if process is not None: + try: + _stdout, stderr = process.communicate(timeout=0.2) + except Exception: + stderr = "" + if item["launch_error"]: + result = ([], engine, f"OCR worker launch failed: {item['launch_error']}", {"worker_process": True}) + elif page_number in timed_out: + result = ([], engine, "OCR exceeded extraction deadline", {"worker_process": True, "timed_out": True}) + elif item["output"].exists(): + try: + payload = json.loads(item["output"].read_text(encoding="utf-8")) + result = ( + list(payload.get("blocks") or []), + str(payload.get("engine") or engine), + str(payload.get("error")) if payload.get("error") else None, + {"worker_process": True, **dict(payload.get("diagnostics") or {})}, + ) + elapsed = float(payload.get("elapsed_seconds") or elapsed) + except Exception as exc: + result = ([], engine, f"OCR worker returned invalid output: {exc}", {"worker_process": True}) + else: + detail = stderr.strip() or f"worker exited with code {getattr(process, 'returncode', None)}" + result = ([], engine, f"OCR worker failed: {detail}", {"worker_process": True}) + results.append((page_number, result, elapsed)) + return results + + +def _prioritize_ocr_candidates( + pages: list[dict[str, Any]], + candidates: list[int], + max_pages: int, +) -> tuple[list[int], list[dict[str, Any]]]: + if max_pages <= 0 or not candidates: + return [], [] + page_count = max(1, len(pages)) + signals = { + "overview_figure": re.compile(r"\b(?:figure|fig\.)\s*[12]\b.*\b(?:overview|framework|architecture|pipeline|system|model)\b", re.IGNORECASE | re.DOTALL), + "method": re.compile(r"\b(?:methods?|methodology|approach|architecture|system overview|framework)\b", re.IGNORECASE), + "abstract": re.compile(r"\babstract\b", re.IGNORECASE), + "conclusion": re.compile(r"\b(?:conclusions?|discussion)\b", re.IGNORECASE), + "caption": re.compile(r"\b(?:figure|fig\.|table)\s*\d", re.IGNORECASE), + } + scored: list[tuple[int, int, list[str]]] = [] + for page_number in candidates: + page = pages[page_number - 1] if 0 < page_number <= len(pages) else {} + text = str(page.get("text") or "") + reasons: list[str] = [] + score = 0 + for name, weight in (("overview_figure", 14), ("method", 10), ("abstract", 9), ("conclusion", 8), ("caption", 5)): + if signals[name].search(text): + score += weight + reasons.append(name) + if page_number == 1: + score += 7 + reasons.append("first_page") + elif page_number <= 3: + score += 3 + reasons.append("early_page") + scored.append((score, page_number, reasons)) + + selected: list[int] = [] + reason_map: dict[int, list[str]] = {} + for score, page_number, reasons in sorted(scored, key=lambda item: (-item[0], item[1])): + if score <= 0 or len(selected) >= max_pages: + break + selected.append(page_number) + reason_map[page_number] = reasons + + anchors = [ + 1, + min(2, page_count), + min(3, page_count), + min(4, page_count), + max(1, round(page_count * 0.85)), + max(1, round(page_count * 0.50)), + ] + for anchor in anchors: + if len(selected) >= max_pages: + break + if anchor in selected: + continue + available = [value for value in candidates if value not in selected] + if not available: + break + page_number = min(available, key=lambda value: (abs(value - anchor), value)) + selected.append(page_number) + reason_map.setdefault(page_number, []).append(f"coverage_anchor_{anchor}") + + for page_number in candidates: + if len(selected) >= max_pages: + break + if page_number not in selected: + selected.append(page_number) + reason_map.setdefault(page_number, []).append("remaining_candidate") + + details = [ + {"page": page_number, "rank": rank, "reasons": reason_map.get(page_number, [])} + for rank, page_number in enumerate(selected, 1) + ] + return selected, details + + +def _read_pdf_pages( + path: Path, + max_chars: int, + deadline_at: float | None, + ocr_engine: str, + ocr_lang: str, + ocr_adapter: Callable | None, + max_ocr_pages: int, + ocr_rescue_min_remaining: float, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + try: + import fitz + except Exception as exc: + raise RuntimeError(f"PyMuPDF is required for PDF extraction: {exc}") from exc + + metadata = _pdf_metadata(path) + started = time.monotonic() + document = fitz.open(str(path)) + poppler_text, poppler_warning = _poppler_pages(path) + pages: list[dict[str, Any]] = [] + warnings: list[str] = [poppler_warning] if poppler_warning else [] + used_chars = 0 + ocr_candidates: list[int] = [] + try: + for index, page in enumerate(document, 1): + native_width = float(page.mediabox.width) if int(page.rotation or 0) else float(page.rect.width) + raw_blocks, hyphenation_repairs = _merge_native_hyphenated_lines(_pymupdf_line_blocks(page)) + ordered, order_confidence = _reading_order(raw_blocks, native_width) + if len(_normalized_compare_text(" ".join(item["text"] for item in ordered))) < 80: + fallback_blocks, fallback_hyphenation_repairs = _merge_native_hyphenated_lines(_pdfplumber_blocks(path, index)) + fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, native_width) + if len(_normalized_compare_text(" ".join(item["text"] for item in fallback_ordered))) > len(_normalized_compare_text(" ".join(item["text"] for item in ordered))): + ordered, order_confidence = fallback_ordered, min(fallback_confidence, 0.9) + hyphenation_repairs = fallback_hyphenation_repairs + ordered = _rotate_blocks_to_page(ordered, page) + for block_index, block in enumerate(ordered, 1): + block["id"] = f"P{index:03d}_B{block_index:03d}" + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) + page_text = "\n\n".join(item["text"] for item in ordered) + poppler_page = poppler_text[index - 1] if index - 1 < len(poppler_text) else "" + native_norm = _normalized_compare_text(page_text) + poppler_norm = _normalized_compare_text(poppler_page) + order_agreement = round(SequenceMatcher(None, native_norm[:20000], poppler_norm[:20000]).ratio(), 4) if native_norm and poppler_norm else None + agreement = _token_overlap_agreement(page_text, poppler_page) + abnormal = len(native_norm) < 80 or _replacement_rate(page_text) > 0.005 or _mojibake_rate(page_text) > 0.005 or (agreement is not None and agreement < 0.65) + if abnormal: + ocr_candidates.append(index) + pages.append({ + "page": index, + "width": round(float(page.rect.width), 3), + "height": round(float(page.rect.height), 3), + "rotation": int(page.rotation or 0), + "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, + "blocks": ordered, + "text": page_text, + "char_count": len(page_text), + "replacement_character_rate": _replacement_rate(page_text), + "mojibake_rate": _mojibake_rate(page_text), + "poppler_agreement": agreement, + "poppler_order_agreement": order_agreement, + "reading_order_confidence": order_confidence, + "native_hyphenation_repair_count": hyphenation_repairs, + "used_ocr": False, + }) + used_chars += len(page_text) + if used_chars >= max_chars and "document text exceeds evidence budget; evidence will be sampled across all pages" not in warnings: + warnings.append("document text exceeds evidence budget; evidence will be sampled across all pages") + finally: + pass + + ocr_pages: list[int] = [] + ocr_page_durations: list[dict[str, Any]] = [] + prioritized: list[int] = [] + ocr_priority: list[dict[str, Any]] = [] + ocr_worker_count = 1 + if ocr_engine != "off": + prioritized, ocr_priority = _prioritize_ocr_candidates(pages, ocr_candidates, max_ocr_pages) + ocr_worker_count = _rapidocr_worker_count(ocr_engine, ocr_adapter, len(prioritized)) + + def consume_ocr_result(page_number: int, result: tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], ocr_elapsed: float) -> None: + blocks, used_engine, error, ocr_diagnostics = result + ocr_page_durations.append({"page": page_number, "engine": used_engine, "elapsed_seconds": round(ocr_elapsed, 3), "success": not bool(error), **ocr_diagnostics}) + if error: + warnings.append(f"page {page_number} OCR unavailable: {error}") + return + current = pages[page_number - 1] + ordered, order_confidence = _reading_order(blocks, float(current["width"])) + for block_index, block in enumerate(ordered, 1): + block["id"] = f"P{page_number:03d}_B{block_index:03d}" + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(current["width"]))) + ocr_text = "\n\n".join(item["text"] for item in ordered) + if len(_normalized_compare_text(ocr_text)) > len(_normalized_compare_text(current["text"])): + current.update({ + "blocks": ordered, + "text": ocr_text, + "char_count": len(ocr_text), + "replacement_character_rate": _replacement_rate(ocr_text), + "mojibake_rate": _mojibake_rate(ocr_text), + "reading_order_confidence": order_confidence, + "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, + "used_ocr": True, + "ocr_engine": used_engine, + "ocr_margin_noise_removed": int(ocr_diagnostics.get("margin_noise_removed") or 0), + "native_hyphenation_repair_count": 0, + }) + ocr_pages.append(page_number) + + def deadline_allows_wave(rescue_page: bool = False) -> bool: + if deadline_at is not None: + remaining = deadline_at - time.monotonic() + if rescue_page: + if remaining < max(25.0, float(ocr_rescue_min_remaining)): + warnings.append("OCR stopped to preserve deadline validation budget") + return False + return True + completed_times = [float(item["elapsed_seconds"]) for item in ocr_page_durations if isinstance(item.get("elapsed_seconds"), (int, float))] + estimated = (max(completed_times) * 1.25) if completed_times else 45.0 + if remaining < estimated + 20: + warnings.append("OCR stopped to preserve deadline validation budget") + return False + return True + + if deadline_at is not None and ocr_adapter is None: + start = 0 + while start < len(prioritized): + rescue_page = start > 0 + if not deadline_allows_wave(rescue_page=rescue_page): break - text = text[:remaining] - pages.append({"page": index, "text": text}) - used += len(text) - document.close() - except Exception as exc: - raise RuntimeError(f"PDF extraction failed: {exc}") from exc - elif suffix == ".docx": + wave_size = 1 if rescue_page else ocr_worker_count + batch = prioritized[start:start + wave_size] + for page_number, result, ocr_elapsed in _deadline_ocr_wave(path, batch, ocr_engine, ocr_lang, deadline_at): + consume_ocr_result(page_number, result, ocr_elapsed) + start += wave_size + elif ocr_worker_count > 1: + def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]: + ocr_started = time.monotonic() + try: + local_document = fitz.open(str(path)) + try: + result = _ocr_page(local_document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=1) + return result, time.monotonic() - ocr_started + finally: + local_document.close() + except Exception as exc: + return ([], ocr_engine, str(exc), {"margin_noise_removed": 0}), time.monotonic() - ocr_started + + with ThreadPoolExecutor(max_workers=ocr_worker_count) as pool: + for start in range(0, len(prioritized), ocr_worker_count): + if not deadline_allows_wave(): + break + batch = prioritized[start:start + ocr_worker_count] + futures = {page_number: pool.submit(isolated_ocr, page_number) for page_number in batch} + for page_number in batch: + result, ocr_elapsed = futures[page_number].result() + consume_ocr_result(page_number, result, ocr_elapsed) + else: + rapidocr_threads = 2 if ocr_engine in {"auto", "rapidocr"} and ocr_adapter is None else 1 + for page_number in prioritized: + if not deadline_allows_wave(): + break + ocr_started = time.monotonic() + result = _ocr_page(document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=rapidocr_threads) + consume_ocr_result(page_number, result, time.monotonic() - ocr_started) + repeated_margin_noise_removed = _remove_repeated_margin_noise(pages) + document.close() + elapsed = round(time.monotonic() - started, 3) + attempted_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page")] + successful_attempt_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page") and item.get("success")] + schedule_complete = set(prioritized).issubset(attempted_pages) + run_complete = schedule_complete and set(attempted_pages).issubset(successful_attempt_pages) + if not schedule_complete and "OCR schedule was not completed before the extraction deadline" not in warnings: + warnings.append("OCR schedule was not completed before the extraction deadline") + return pages, { + "metadata": metadata, + "ocr_candidate_pages": ocr_candidates, + "ocr_priority_pages": prioritized, + "ocr_priority": ocr_priority, + "ocr_pages": ocr_pages, + "ocr_page_durations": ocr_page_durations, + "ocr_attempted_pages": attempted_pages, + "ocr_successful_attempt_pages": successful_attempt_pages, + "ocr_schedule_complete": schedule_complete, + "ocr_run_complete": run_complete, + "ocr_worker_count": ocr_worker_count, + "ocr_rescue_min_remaining_seconds": float(ocr_rescue_min_remaining), + "repeated_margin_noise_removed_count": repeated_margin_noise_removed, + "native_hyphenation_repair_count": sum(int(page.get("native_hyphenation_repair_count") or 0) for page in pages), + "warnings": warnings, + "poppler_available": bool(poppler_text), + "elapsed_seconds": elapsed, + } + + +def _read_non_pdf_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any]], str]: + suffix = path.suffix.lower() + if suffix == ".docx": loader = "python-docx" try: import docx document = docx.Document(str(path)) text = "\n".join(paragraph.text for paragraph in document.paragraphs)[:max_chars] - pages.append({"page": 1, "text": text}) except Exception as exc: raise RuntimeError(f"DOCX extraction failed: {exc}") from exc else: - text = path.read_text(encoding="utf-8", errors="ignore")[:max_chars] - pages.append({"page": 1, "text": text}) - return pages, loader - - -def _chunk_page(text: str, target_chars: int = 2200) -> list[str]: - paragraphs = [_clean(item) for item in re.split(r"\n\s*\n", text) if _clean(item)] - if not paragraphs: - paragraphs = [_clean(item) for item in text.splitlines() if _clean(item)] - chunks: list[str] = [] - current: list[str] = [] - size = 0 + loader = "plain" + text = path.read_text(encoding="utf-8", errors="replace")[:max_chars] + clean = _clean(text) + paragraphs = [_clean(value) for value in re.split(r"\n\s*\n", clean) if _clean(value)] + chunks = [] for paragraph in paragraphs: - if current and size + len(paragraph) > target_chars: - chunks.append("\n".join(current)) - current = [] - size = 0 - current.append(paragraph) - size += len(paragraph) + 1 - if current: - chunks.append("\n".join(current)) - return chunks - - -def _heading_candidates(text: str) -> list[str]: - headings: list[str] = [] - for raw in text.splitlines(): - line = _clean(raw) - if not 3 <= len(line) <= 100: - continue - if re.match(r"^(?:\d+(?:\.\d+)*[.)]?\s+|[IVX]+[.)]\s+|第[一二三四五六七八九十]+[章节])", line): - value = re.sub(r"^(?:\d+(?:\.\d+)*[.)]?\s+|[IVX]+[.)]\s+)", "", line).strip() - if value and value not in headings: - headings.append(value) - return headings[:30] - - -def _document_index(pages: list[dict[str, Any]], headings: list[str]) -> dict: - sections = [] - formulas = [] - tables = [] + lines = paragraph.splitlines() + if len(lines) > 1 and re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)", lines[0], re.IGNORECASE): + chunks.extend([_clean(lines[0]), _clean("\n".join(lines[1:]))]) + else: + chunks.append(paragraph) + blocks = [] + for index, chunk in enumerate(chunks or [clean], 1): + blocks.append({"id": f"P001_B{index:03d}", "bbox": None, "text": chunk, "kind": _block_kind(chunk), "source": loader, "confidence": 1.0, "reading_order": index}) + return [{"page": 1, "width": None, "height": None, "blocks": blocks, "text": clean, "char_count": len(clean), "replacement_character_rate": _replacement_rate(clean), "mojibake_rate": _mojibake_rate(clean), "poppler_agreement": None, "reading_order_confidence": 1.0, "used_ocr": False}], loader + + +def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: + headings = [] + seen = set() + body_started = False + body_anchor: tuple[int, float] | None = None + for page in pages: + font_sizes = sorted(float(block.get("font_size")) for block in page.get("blocks", []) if isinstance(block.get("font_size"), (int, float)) and len(_normalized_compare_text(str(block.get("text") or ""))) >= 20) + middle = len(font_sizes) // 2 + body_font_size = (font_sizes[middle] if len(font_sizes) % 2 else (font_sizes[middle - 1] + font_sizes[middle]) / 2.0) if font_sizes else None + for block in page.get("blocks", []): + lines = [_clean(value) for value in str(block.get("text", "")).splitlines() if _clean(value)] + for line_index, line in enumerate(lines): + if not 2 <= len(line) <= 140: + continue + explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) + numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) + compact_line = _normalized_compare_text(line) + block_bold = bool(block.get("font_bold")) + font_size = float(block.get("font_size") or 0.0) + first_line_alpha = next((char for char in line if char.isalpha()), "") + heading_case = not first_line_alpha or not first_line_alpha.isascii() or first_line_alpha.isupper() + known = len(line) <= 60 and not line.endswith((".", ",", ";")) and any( + compact_alias and heading_case and (compact_line == compact_alias or (block_bold and compact_line.startswith(compact_alias))) + for aliases in SECTION_ALIASES.values() + for alias in aliases + for compact_alias in [_normalized_compare_text(alias)] + ) + block_y = float((block.get("bbox") or [0, 0, 0, 0])[1]) + after_anchor = body_anchor is not None and (int(page["page"]) > body_anchor[0] or block_y >= body_anchor[1]) + position_ok = after_anchor if body_anchor is not None else body_started + size_ok = body_font_size is None or font_size >= body_font_size * 0.95 + typographic = position_ok and block_bold and size_ok and _looks_like_typographic_heading(line) + if numbered: + title_candidate = numbered.group(2).strip() + first_alpha = next((char for char in title_candidate if char.isalpha()), "") + numbered_style = block.get("source") != "pymupdf" or body_font_size is None or block_bold or font_size >= body_font_size * 1.05 + if not numbered_style: + first_alpha = "" + if not title_candidate or not title_candidate[0].isalpha(): + first_alpha = "" + has_case = bool(first_alpha and first_alpha.lower() != first_alpha.upper()) + if not first_alpha or (has_case and not first_alpha.isupper()) or re.search(r"[=±∑∫]", title_candidate) or title_candidate.endswith(".") or len(title_candidate) > 80 or len(title_candidate.split()) > 10: + numbered = None + if not (explicit or numbered or known or typographic): + continue + normalized = numbered.group(2).strip() if numbered else line.strip().rstrip(".:") + key = (normalized.casefold(), int(page.get("page") or 0), str(block.get("id") or ""), line_index) + if normalized and key not in seen: + seen.add(key) + headings.append({ + "id": f"section_{len(headings) + 1:03d}", + "title": normalized, + "page": page["page"], + "block_id": block["id"], + "line_index": line_index, + "_candidate_type": "explicit" if explicit else "numbered" if numbered else "known" if known else "typographic", + "_bbox": block.get("bbox"), + "_font_size": font_size, + "_reading_order": int(block.get("reading_order") or 0), + }) + if body_anchor is None and (explicit or numbered or known): + body_anchor = (int(page["page"]), block_y) + body_started = body_started or bool(explicit or numbered or known) + merged: list[dict[str, Any]] = [] + for heading in headings: + previous = merged[-1] if merged else None + previous_bbox = previous.get("_bbox") if previous else None + current_bbox = heading.get("_bbox") + same_multiline_heading = bool( + previous + and previous.get("_candidate_type") == "typographic" + and heading.get("_candidate_type") == "typographic" + and int(previous.get("page") or 0) == int(heading.get("page") or 0) + and int(heading.get("_reading_order") or 0) == int(previous.get("_reading_order") or 0) + 1 + and previous_bbox + and current_bbox + and float(current_bbox[1]) - float(previous_bbox[3]) <= max(float(previous.get("_font_size") or 0), float(heading.get("_font_size") or 0), 1.0) * 0.8 + and len(f"{previous.get('title', '')} {heading.get('title', '')}") <= 140 + ) + if same_multiline_heading: + previous["title"] = _clean(f"{previous.get('title', '')} {heading.get('title', '')}").rstrip(".:") + previous.setdefault("continuation_block_ids", []).append(str(heading.get("block_id") or "")) + previous["_bbox"] = [ + min(float(previous_bbox[0]), float(current_bbox[0])), + min(float(previous_bbox[1]), float(current_bbox[1])), + max(float(previous_bbox[2]), float(current_bbox[2])), + max(float(previous_bbox[3]), float(current_bbox[3])), + ] + previous["_reading_order"] = int(heading.get("_reading_order") or 0) + continue + merged.append(heading) + for index, heading in enumerate(merged, 1): + heading["id"] = f"section_{index:03d}" + return [{key: value for key, value in heading.items() if not key.startswith("_")} for heading in merged[:80]] + + +def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, Any]: figures = [] - seen_sections = set() - formula_pattern = re.compile(r"(?:[A-Za-zΑ-Ωα-ω][A-Za-z0-9_{}^\-]*\s*=\s*[^=\n]{3,}|[∑∫∂∇≈≤≥]\s*[^\n]{3,}|\([^\n]{2,80}\)\s*\(\d+\)\s*$)") - table_pattern = re.compile(r"^(?:Table|TABLE|表)\s*[A-Za-z0-9一二三四五六七八九十.：: -]+", re.IGNORECASE) - figure_pattern = re.compile(r"^(?:Figure|FIGURE|Fig\.|图)\s*[A-Za-z0-9一二三四五六七八九十.：: -]+", re.IGNORECASE) + tables = [] + formulas = [] for page in pages: - for raw in page.get("text", "").splitlines(): - line = _clean(raw) - if not line: - continue - for heading in headings: - if heading.lower() in line.lower() and heading not in seen_sections: - sections.append({"id": f"section_{len(sections) + 1:03d}", "title": heading, "page": page["page"]}) - seen_sections.add(heading) - break - if len(formulas) < 120 and 4 <= len(line) <= 240 and formula_pattern.search(line): - formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "text": line}) - if len(tables) < 80 and table_pattern.match(line): - tables.append({"id": f"table_{len(tables) + 1:03d}", "page": page["page"], "caption": line}) - if len(figures) < 80 and figure_pattern.match(line): - figures.append({"id": f"figure_{len(figures) + 1:03d}", "page": page["page"], "caption": line}) + page_blocks = page.get("blocks", []) + for block_index, block in enumerate(page_blocks): + record = {"page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "caption": block.get("text", "")} + if block.get("kind") == "caption": + parent = block.get("parent_block") + continuation_blocks = [] + for following in page_blocks[block_index + 1:]: + if parent is None or following.get("parent_block") != parent or _caption_prefix_kind(str(following.get("text") or "")): + break + continuation_blocks.append(following) + + # A caption may continue in the next column at the same vertical + # position while PyMuPDF assigns a different parent block. Recover + # those lines by geometry and font, then keep column-major order. + start_bbox = block.get("bbox") if isinstance(block.get("bbox"), list) else [0, 0, 0, 0] + start_y = float(start_bbox[1] or 0) + start_font = float(block.get("font_size") or 0) + page_height = max(1.0, float(page.get("height") or 1)) + max_y = start_y + max(42.0, page_height * 0.09) + included_ids = {str(block.get("id") or ""), *(str(item.get("id") or "") for item in continuation_blocks)} + for candidate in page_blocks: + candidate_id = str(candidate.get("id") or "") + if not candidate_id or candidate_id in included_ids: + continue + if int(candidate.get("column") if candidate.get("column") is not None else -99) == int(block.get("column") if block.get("column") is not None else -98): + continue + bbox = candidate.get("bbox") if isinstance(candidate.get("bbox"), list) else [0, 0, 0, 0] + candidate_y = float(bbox[1] or 0) + candidate_font = float(candidate.get("font_size") or 0) + if not (start_y - 2.0 <= candidate_y <= max_y): + continue + if start_font and candidate_font and abs(candidate_font - start_font) > 0.8: + continue + if _caption_prefix_kind(str(candidate.get("text") or "")): + continue + continuation_blocks.append(candidate) + included_ids.add(candidate_id) + if continuation_blocks: + continuation_blocks.sort(key=lambda item: (int(item.get("column") or 0), float((item.get("bbox") or [0, 0, 0, 0])[1]), float((item.get("bbox") or [0, 0, 0, 0])[0]))) + record["caption"] = _clean(record["caption"] + " " + " ".join(str(item.get("text") or "") for item in continuation_blocks)) + figures.append({"id": f"figure_{len(figures) + 1:03d}", **record}) + elif block.get("kind") == "table": + tables.append({"id": f"table_{len(tables) + 1:03d}", **record}) + elif block.get("kind") == "formula": + formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "text": block.get("text", "")}) + page_map = {int(page["page"]): page for page in pages} + for table in tables: + page = page_map.get(int(table.get("page") or 0)) + if not page: + continue + blocks = page.get("blocks", []) + caption_index = next((index for index, block in enumerate(blocks) if str(block.get("id") or "") == str(table.get("block_id") or "")), None) + if caption_index is None: + continue + caption_bbox = table.get("bbox") or [0, 0, 0, 0] + candidates = [] + for block in blocks[caption_index + 1:]: + bbox = block.get("bbox") + text = _clean(block.get("text", "")) + if not bbox or not text: + continue + if str(block.get("kind") or "") in {"heading", "caption", "table", "formula"}: + break + if float(bbox[1]) > float(caption_bbox[3]) + min(300.0, float(page.get("height") or 800.0) * 0.35): + break + if len(text) > 60: + break + candidates.append(block) + if len(candidates) < 4: + continue + heights = sorted(max(1.0, float(block["bbox"][3]) - float(block["bbox"][1])) for block in candidates) + tolerance = max(5.0, heights[len(heights) // 2] * 0.75) + row_groups: list[list[dict[str, Any]]] = [] + for block in sorted(candidates, key=lambda item: ((float(item["bbox"][1]) + float(item["bbox"][3])) / 2.0, float(item["bbox"][0]))): + center = (float(block["bbox"][1]) + float(block["bbox"][3])) / 2.0 + if not row_groups: + row_groups.append([block]) + continue + previous_center = sum((float(item["bbox"][1]) + float(item["bbox"][3])) / 2.0 for item in row_groups[-1]) / len(row_groups[-1]) + if abs(center - previous_center) <= tolerance: + row_groups[-1].append(block) + else: + row_groups.append([block]) + if len(row_groups) < 2 or len(row_groups[0]) < 2: + continue + header = sorted(row_groups[0], key=lambda item: float(item["bbox"][0])) + anchors = [(float(item["bbox"][0]) + float(item["bbox"][2])) / 2.0 for item in header] + columns = [_clean(item.get("text", "")) for item in header] + rows = [] + cell_records = [] + for row_index, group in enumerate(row_groups): + values: list[str | None] = [None] * len(columns) + block_ids: list[str | None] = [None] * len(columns) + for block in sorted(group, key=lambda item: float(item["bbox"][0])): + center = (float(block["bbox"][0]) + float(block["bbox"][2])) / 2.0 + column_index = min(range(len(anchors)), key=lambda index: abs(center - anchors[index])) + value = _clean(block.get("text", "")) + values[column_index] = f"{values[column_index]} {value}".strip() if values[column_index] else value + block_ids[column_index] = str(block.get("id") or "") + cell_records.append({"row": row_index, "column": column_index, "text": value, "block_id": block.get("id"), "bbox": block.get("bbox")}) + if row_index: + rows.append({"values": values, "block_ids": block_ids}) + header_signal = any( + _cjk_character_count(value) > 0 + or (value and value[0].isalpha() and value[0].isupper()) + or value.isupper() + or bool(re.search(r"(?:[A-Za-z].*\d|\d.*[A-Za-z])", value)) + for value in columns + ) + if len(rows) < 2 or not header_signal: + continue + cell_block_ids = {str(record.get("block_id") or "") for record in cell_records} + for block in candidates: + if str(block.get("id") or "") in cell_block_ids: + block["kind"] = "table_cell" + all_boxes = [caption_bbox] + [block.get("bbox") for block in candidates if block.get("bbox")] + table.update({ + "columns": columns, + "rows": rows, + "cells": cell_records, + "bbox": [ + min(float(box[0]) for box in all_boxes), + min(float(box[1]) for box in all_boxes), + max(float(box[2]) for box in all_boxes), + max(float(box[3]) for box in all_boxes), + ], + }) + return {"summary": "Page-aware section, formula, table-caption, and figure-caption index.", "sections": sections, "formulas": formulas[:120], "tables": tables[:80], "figures": figures[:80]} + + +def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, bool]: + candidates = ("\n".join(item["title"] for item in sections) + "\n" + "\n".join(page.get("text", "") for page in pages)).casefold() + compact = _normalized_compare_text(candidates) + coverage = { + name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) or (_normalized_compare_text(alias) and _normalized_compare_text(alias) in compact) for alias in aliases) + for name, aliases in SECTION_ALIASES.items() + } + # Nature-style articles and several journals print a lead abstract without + # an explicit "Abstract" heading. Treat a substantial first-page research + # summary as abstract coverage when it contains an authorial contribution + # statement, instead of emitting a false extraction warning. + first_page = str(pages[0].get("text") or "") if pages else "" + first_page_compact = _normalized_compare_text(first_page) + implicit_abstract = bool( + len(first_page_compact) >= 700 + and re.search( + r"\b(?:here|in this (?:work|paper|study))\s+(?:we\s+)?(?:present|introduce|propose|develop|show|demonstrate)\b" + r"|\bwe\s+(?:present|introduce|propose|develop|show|demonstrate)\b", + first_page, + re.IGNORECASE, + ) + ) + coverage["abstract"] = bool(coverage.get("abstract") or implicit_abstract) + return coverage + + +def _ocr_latin_lexical_quality(pages: list[dict[str, Any]]) -> dict[str, Any]: + text = " ".join(str(page.get("text") or "") for page in pages if page.get("used_ocr")) + anchors = re.findall(r"\b(?:abstract|introduction|method|methods|experiments|results|conclusion|discussion)\b", text, re.IGNORECASE) + tokens = [token for token in re.findall(r"[A-Za-z]{3,}", text) if not token.isupper()] + if len(tokens) < 20 or len(anchors) < 2: + return {"applied": False, "token_count": len(tokens), "known_word_ratio": None, "english_anchor_count": len(anchors)} + try: + import wordninja + + known_words = wordninja.DEFAULT_LANGUAGE_MODEL._wordcost + except Exception: + return {"applied": False, "token_count": len(tokens), "known_word_ratio": None, "english_anchor_count": len(anchors)} + known = sum(token.casefold() in known_words or token.casefold() in OCR_PROTECTED_WORDS for token in tokens) return { - "summary": "Page-aware section, formula, table-caption, and figure-caption index.", - "sections": sections, - "formulas": formulas, - "tables": tables, - "figures": figures, + "applied": True, + "token_count": len(tokens), + "known_word_ratio": round(known / max(1, len(tokens)), 4), + "english_anchor_count": len(anchors), } -def parse_paper(path: str | Path, max_chars: int = 90000) -> dict: - source = Path(path).resolve() - if not source.exists(): - raise FileNotFoundError(f"Paper does not exist: {source}") - pages, loader = _read_pages(source, max_chars=max_chars) +def _evidence_id(page: int, text: str, bbox: Any, kind: str) -> str: + payload = json.dumps({"page": page, "text": _clean(text), "bbox": bbox, "kind": kind}, ensure_ascii=False, sort_keys=True).encode("utf-8") + return f"E_P{int(page):03d}_{hashlib.sha1(payload).hexdigest()[:12]}" + + +def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], document_index: dict[str, Any], max_chars: int) -> list[dict[str, Any]]: + section_positions = sorted(sections, key=lambda item: (item["page"], item.get("block_id", ""))) evidence: list[dict[str, Any]] = [] + used = 0 + for figure in document_index.get("figures", []): + text = _clean(figure.get("caption", "")) + if not text or used + len(text) > max_chars: + continue + evidence.append({ + "id": _evidence_id(int(figure.get("page") or 0), text, figure.get("bbox"), "caption"), + "page": figure.get("page"), + "block_id": figure.get("block_id"), + "bbox": figure.get("bbox"), + "section_hint": "Figure Captions", + "kind": "caption", + "source": "pymupdf", + "confidence": 1.0, + "text": text, + "char_count": len(text), + }) + used += len(text) + + for table in document_index.get("tables", []): + lines = [_clean(table.get("caption", ""))] + columns = [str(value or "") for value in table.get("columns", [])] + if columns: + lines.append("Columns: " + " | ".join(columns)) + for row in table.get("rows", []): + values = [str(value or "") for value in row.get("values", [])] + lines.append("Row: " + " | ".join(values)) + text = _clean("\n".join(value for value in lines if value)) + if not text or used + len(text) > max_chars: + continue + evidence.append({ + "id": _evidence_id(int(table.get("page") or 0), text, table.get("bbox"), "table"), + "page": table.get("page"), + "block_id": table.get("block_id"), + "bbox": table.get("bbox"), + "section_hint": "Table Captions", + "kind": "table", + "source": "structured-table", + "confidence": 1.0, + "text": text, + "char_count": len(text), + }) + used += len(text) + + block_candidates: list[dict[str, Any]] = [] for page in pages: - for chunk in _chunk_page(page["text"]): - if not chunk: + for block in page.get("blocks", []): + text = _clean(block.get("text", "")) + if not text or block.get("kind") in {"caption", "table", "table_cell"}: continue - evidence_id = f"E{len(evidence) + 1:04d}" - evidence.append({ - "id": evidence_id, + block_id = str(block.get("id") or "") + section = next( + ( + item["title"] + for item in reversed(section_positions) + if (int(item["page"]), str(item.get("block_id") or "")) <= (int(page["page"]), block_id) + ), + None, + ) + block_candidates.append({ + "id": _evidence_id(int(page["page"]), text, block.get("bbox"), str(block.get("kind") or "paragraph")), "page": page["page"], - "section_hint": None, - "text": chunk, - "char_count": len(chunk), + "block_id": block_id, + "bbox": block.get("bbox"), + "section_hint": section, + "kind": block.get("kind"), + "source": block.get("source"), + "confidence": block.get("confidence", 1.0), + "text": text, + "char_count": len(text), }) - all_text = "\n".join(page["text"] for page in pages) - headings = _heading_candidates(all_text) - document_index = _document_index(pages, headings) - for item in evidence: - low = item["text"].lower() - item["section_hint"] = next((heading for heading in headings if heading.lower() in low), None) + selected_ids: set[str] = set() + page_groups: dict[int, list[dict[str, Any]]] = {} + for item in block_candidates: + page_groups.setdefault(int(item["page"]), []).append(item) + + def add(item: dict[str, Any], max_length: int | None = None) -> bool: + nonlocal used + item_id = str(item["id"]) + if item_id in selected_ids or used >= max_chars: + return False + remaining = max_chars - used + if remaining <= 0: + return False + value = dict(item) + if max_length is not None and len(value["text"]) > max_length: + value["text"] = value["text"][:max_length].rstrip() + value["char_count"] = len(value["text"]) + value["id"] = _evidence_id(int(value["page"]), value["text"], value.get("bbox"), str(value.get("kind") or "paragraph")) + if len(value["text"]) > remaining: + if remaining < 40: + return False + value["text"] = value["text"][:remaining].rstrip() + value["char_count"] = len(value["text"]) + value["id"] = _evidence_id(int(value["page"]), value["text"], value.get("bbox"), str(value.get("kind") or "paragraph")) + item_id = str(value["id"]) + evidence.append(value) + selected_ids.add(str(item["id"])) + selected_ids.add(item_id) + used += len(value["text"]) + return True + + # Reserve a representative block for every page before spending the budget + # on long early sections. This keeps conclusions and appendices discoverable. + representative_quota = max(80, min(600, max(1, max_chars - used) // max(1, len(page_groups)))) + for page_number in sorted(page_groups): + candidates = page_groups[page_number] + representative = max( + candidates, + key=lambda item: ( + 1 if item.get("kind") in {"paragraph", "title"} else 0, + min(len(str(item.get("text") or "")), 600), + ), + ) + add(representative, max_length=representative_quota) + + section_priority = { + "abstract": 0, + "conclusion": 1, + "conclusions": 1, + "discussion": 1, + "method": 2, + "methods": 2, + "methodology": 2, + "approach": 2, + "architecture": 2, + "framework": 2, + "system overview": 2, + "introduction": 3, + "background": 3, + "experiment": 4, + "experiments": 4, + "evaluation": 4, + "results": 4, + } - if not evidence: - raise ValueError("No readable paper text was extracted") + def priority(item: dict[str, Any]) -> tuple[int, int, str]: + section = str(item.get("section_hint") or "").casefold() + text = str(item.get("text") or "") + topology_definition = bool(re.search( + r"\b(?:has|have|consists? of|comprises?|contains?)\b.{0,80}\b(?:stages?|components?|modules?|steps?)\b" + r"|\b(?:assisted[ -]manual|semi[ -]automatic|fully automatic)\b" + r"|\b(?:input|image|text|query|prompt)\b.{0,100}\b(?:encoder|decoder|retriever|generator)\b" + r"|\b(?:encoder|decoder|retriever|generator)\b.{0,100}\b(?:output|prediction|representation|embedding)\b", + text, + re.IGNORECASE, + )) + rank = min((value for name, value in section_priority.items() if name in section), default=5) + if topology_definition: + rank = -1 + if item.get("kind") == "heading": + rank = min(rank, 2) + return rank, int(item.get("page") or 0), str(item.get("block_id") or "") + + for item in sorted(block_candidates, key=priority): + add(item) + + document_order = { + str(item["id"]): index + for index, item in enumerate(block_candidates) + } + caption_count = sum(1 for item in evidence if item.get("kind") == "caption") + caption_items = evidence[:caption_count] + body_items = sorted(evidence[caption_count:], key=lambda item: (int(item.get("page") or 0), document_order.get(str(item.get("id")), 10**9), str(item.get("block_id") or ""))) + evidence = caption_items + body_items + for index, item in enumerate(evidence, 1): + item["legacy_id"] = f"E{index:04d}" + return evidence + + +def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[str, Any], details: dict[str, Any]) -> dict[str, Any]: + readable = [page for page in pages if len(_normalized_compare_text(page.get("text", ""))) >= 80] + readable_ratio = round(len(readable) / max(1, len(pages)), 4) + ocr_count = sum(1 for page in pages if page.get("used_ocr")) + candidate_count = len(details.get("ocr_candidate_pages", [])) + fully_scanned_source = bool(pages and candidate_count == len(pages)) + partial_scan = bool(fully_scanned_source and 0 < ocr_count < len(pages)) + pdf_type = "scanned" if fully_scanned_source else "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" + coverage = _section_coverage(pages, index.get("sections", [])) + bold_block_ids = {(int(page["page"]), str(block.get("id") or "")) for page in pages for block in page.get("blocks", []) if block.get("font_bold")} + warnings = list(details.get("warnings", [])) + ocr_blocks = [block for page in pages if page.get("used_ocr") for block in page.get("blocks", []) if isinstance(block, dict)] + ocr_confidences = [float(block.get("confidence") or 0.0) for block in ocr_blocks] + mean_ocr_confidence = round(sum(ocr_confidences) / max(1, len(ocr_confidences)), 4) if ocr_blocks else None + short_ocr_blocks = [block for block in ocr_blocks if len(_normalized_compare_text(str(block.get("text") or ""))) < 3] + short_ocr_block_ratio = round(len(short_ocr_blocks) / max(1, len(ocr_blocks)), 4) if ocr_blocks else None + lexical_quality = _ocr_latin_lexical_quality(pages) + sampled_scan_ready = bool( + partial_scan + and len(readable) >= min(3, len(pages)) + and coverage.get("abstract") + and coverage.get("method") + and mean_ocr_confidence is not None + and mean_ocr_confidence >= 0.75 + ) + if readable_ratio < 0.6 and not sampled_scan_ready: + warnings.append("fewer than 60% of pages contain reliable text") + if sampled_scan_ready: + warnings.append("fully scanned document was only partially OCRed; semantic scope is limited to sampled pages") + if mean_ocr_confidence is not None and mean_ocr_confidence < 0.5: + warnings.append("mean OCR confidence is below 0.50") + if short_ocr_block_ratio is not None and short_ocr_block_ratio > 0.3: + warnings.append("OCR output is highly fragmented") + if lexical_quality.get("applied") and float(lexical_quality.get("known_word_ratio") or 0.0) < 0.72: + warnings.append("OCR English lexical plausibility is below 0.72") + if not coverage.get("abstract") or not coverage.get("method"): + warnings.append("abstract or method-like section was not confidently located") + agreements = [page["poppler_agreement"] for page in pages if isinstance(page.get("poppler_agreement"), (int, float))] + lexical_failure = bool(lexical_quality.get("applied") and float(lexical_quality.get("known_word_ratio") or 0.0) < 0.5) + report_status = "fail" if (readable_ratio < 0.6 and not sampled_scan_ready) or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) or lexical_failure else "warning" if warnings else "pass" return { - "summary": "Paper parsed into page-aware evidence chunks.", + "summary": "PDF extraction quality and fallback report.", + "status": report_status, + "source_path": str(source), + "pdf_type": pdf_type, + "page_count": len(pages), + "readable_page_count": len(readable), + "readable_page_ratio": readable_ratio, + "semantic_scope": "sampled_pages_only" if partial_scan else "full_document", + "scientific_scope_complete": not partial_scan, + "sampled_scan_ready": sampled_scan_ready, + "empty_or_low_text_pages": [page["page"] for page in pages if page not in readable], + "ocr_candidate_pages": details.get("ocr_candidate_pages", []), + "ocr_priority_pages": details.get("ocr_priority_pages", []), + "ocr_priority": details.get("ocr_priority", []), + "ocr_pages": details.get("ocr_pages", []), + "ocr_page_durations": details.get("ocr_page_durations", []), + "ocr_attempted_pages": details.get("ocr_attempted_pages", []), + "ocr_successful_attempt_pages": details.get("ocr_successful_attempt_pages", []), + "ocr_schedule_complete": bool(details.get("ocr_schedule_complete", True)), + "ocr_run_complete": bool(details.get("ocr_run_complete", True)), + "ocr_worker_count": int(details.get("ocr_worker_count") or 1), + "ocr_rescue_min_remaining_seconds": float(details.get("ocr_rescue_min_remaining_seconds") or 45.0), + "repeated_margin_noise_removed_count": int(details.get("repeated_margin_noise_removed_count") or 0), + "native_hyphenation_repair_count": int(details.get("native_hyphenation_repair_count") or 0), + "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), + "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), + "mean_ocr_confidence": mean_ocr_confidence, + "short_ocr_block_ratio": short_ocr_block_ratio, + "ocr_latin_lexical_gate_applied": bool(lexical_quality.get("applied")), + "ocr_latin_token_count": int(lexical_quality.get("token_count") or 0), + "ocr_latin_known_word_ratio": lexical_quality.get("known_word_ratio"), + "ocr_english_anchor_count": int(lexical_quality.get("english_anchor_count") or 0), + "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), + "mojibake_rate": round(sum(page.get("mojibake_rate", 0.0) for page in pages) / max(1, len(pages)), 6), + "cross_extractor_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, + "reading_order_confidence": round(sum(page.get("reading_order_confidence", 0.0) for page in pages) / max(1, len(pages)), 4), + "max_column_count": max((int(page.get("column_count") or 1) for page in pages), default=1), + "multi_column_page_count": sum(int(page.get("column_count") or 1) > 1 for page in pages), + "rotated_pages": [int(page["page"]) for page in pages if int(page.get("rotation") or 0) % 360 != 0], + "section_coverage": coverage, + "figure_caption_count": len(index.get("figures", [])), + "table_caption_count": len(index.get("tables", [])), + "formula_count": len(index.get("formulas", [])), + "section_count": len(index.get("sections", [])), + "typographic_heading_count": sum((int(item.get("page") or 0), str(item.get("block_id") or "")) in bold_block_ids for item in index.get("sections", [])), + "merged_heading_line_count": sum(len(item.get("continuation_block_ids", [])) for item in index.get("sections", [])), + "poppler_available": details.get("poppler_available", False), + "warnings": warnings, + "elapsed_seconds": details.get("elapsed_seconds", 0.0), + } + + +def parse_paper( + path: str | Path, + max_chars: int = 90000, + deadline_at: float | None = None, + ocr_engine: str = "off", + ocr_lang: str = "en_ch", + ocr_adapter: Callable | None = None, + max_ocr_pages: int = 6, + ocr_rescue_min_remaining: float = 45.0, +) -> dict[str, Any]: + source = Path(path).resolve() + if not source.exists(): + raise FileNotFoundError(f"Paper does not exist: {source}") + source_hash = hashlib.sha256(source.read_bytes()).hexdigest() + if source.suffix.lower() == ".pdf": + _validate_pdf_container(source) + pages, details = _read_pdf_pages(source, max_chars, deadline_at, ocr_engine, ocr_lang, ocr_adapter, max_ocr_pages, ocr_rescue_min_remaining) + loader = "structured-pdf" + else: + pages, loader = _read_non_pdf_pages(source, max_chars) + details = {"warnings": [], "ocr_candidate_pages": [], "ocr_pages": [], "poppler_available": False, "elapsed_seconds": 0.0} + sections = _heading_candidates(pages) + section_blocks = { + (int(item.get("page") or 0), block_id) + for item in sections + for block_id in [str(item.get("block_id") or ""), *(str(value) for value in item.get("continuation_block_ids", []))] + if block_id + } + for page in pages: + for block in page.get("blocks", []): + if (int(page["page"]), str(block.get("id") or "")) in section_blocks: + block["kind"] = "heading" + document_index = _document_index(pages, sections) + evidence = _build_evidence(pages, sections, document_index, max_chars) + all_text = "\n\n".join(page.get("text", "") for page in pages) + extraction_report = _extraction_report(source, pages, document_index, details) + evidence_pages = {int(item.get("page") or 0) for item in evidence if item.get("page")} + extraction_report["evidence_char_count"] = sum(int(item.get("char_count") or 0) for item in evidence) + extraction_report["evidence_page_count"] = len(evidence_pages) + extraction_report["evidence_page_coverage_ratio"] = round(len(evidence_pages) / max(1, len(pages)), 4) + return { + "summary": "Paper parsed into a structured, page-aware evidence model.", "source_path": str(source), "source_name": source.name, "source_type": source.suffix.lower(), + "source_sha256": source_hash, "loader": loader, "page_count": len(pages), "char_count": len(all_text), - "headings": headings, + "headings": [item["title"] for item in sections], + "sections": sections, + "pages": pages, "document_index": document_index, "evidence": evidence, + "extraction_report": extraction_report, + } + + +def paper_markdown(parsed: dict[str, Any]) -> str: + parts = [] + for page in parsed.get("pages", []): + parts.append(f"\n\n{page.get('text', '')}") + return "\n\n".join(parts).strip() + "\n" + + +def framework_caption_score(caption: str, index: int = 0) -> int: + """Rank captions by suitability for a paper-core framework figure.""" + text = str(caption or "") + score = 0 + if re.search(r"\b(?:overview|schematic)\b", text, re.IGNORECASE): + score += 14 + if re.search(r"\b(?:framework|architecture|pipeline|workflow|procedure|approach)\b", text, re.IGNORECASE): + score += 10 + if re.search(r"\b(?:construction|development|stage\s+one|stage\s+two|pre-?training|fine-?tuning|downstream)\b", text, re.IGNORECASE): + score += 10 + if re.search(r"\b(?:three|four|multiple)\s+(?:interconnected|connected|coupled)\s+(?:components?|parts?|pillars?)\b", text, re.IGNORECASE): + score += 16 + if re.search(r"\bfoundation\s+model\b", text, re.IGNORECASE): + score += 8 + if re.search(r"\b(?:data engine|promptable segmentation task|prompt-able segmentation task)\b", text, re.IGNORECASE): + score += 8 + if re.search(r"\b(?:method|model|system)\b", text, re.IGNORECASE): + score += 6 + if re.match(r"^(?:figure|fig\.)\s*1\b", text, re.IGNORECASE): + score += 6 + elif re.match(r"^(?:figure|fig\.)\s*2\b", text, re.IGNORECASE): + score += 2 + if 120 <= len(text) <= 1600: + score += 4 + score += min(6, len(re.findall(r"\b(?:uses?|constructs?|adapts?|takes?|feeds?|passes?|produces?|trains?|pretrains?|fine-?tunes?|predicts?|outputs?)\b", text, re.IGNORECASE))) + structural_terms = { + match.casefold() + for match in re.findall( + r"\b(?:backbone|encoder|decoder|object quer(?:y|ies)|embedding|feed[ -]?forward network|ffn|" + r"feature extractor|classifier|retriever|generator|renderer|matching|objective|loss|" + r"input image|class predictions?|bounding box(?: predictions?)?|output embedding)\b", + text, + re.IGNORECASE, + ) } + if len(structural_terms) >= 4: + score += 12 + elif len(structural_terms) >= 2: + score += 6 + if re.search( + r"\b(?:before passing|then takes|then passes|passes? each|followed by|feeds? into|" + r"as input .*? then|outputs? .*? to .*? predict)\b", + text, + re.IGNORECASE, + ): + score += 6 + if re.search(r"\b(?:performance|comparison|results?|ablation|qualitative|attention map|distribution)\b", text, re.IGNORECASE): + score -= 18 + if re.search(r"\b(?:AUROC|AUPR|confidence interval|error bars?|P\s*value|t-?test|random seeds?)\b", text, re.IGNORECASE): + score -= 16 + return score + + +def rank_framework_figures(parsed: dict[str, Any], limit: int = 4) -> list[dict[str, Any]]: + candidate_pattern = re.compile( + r"\b(?:overview|schematic|framework|architecture|pipeline|procedure|approach|method|model|system|workflow|construction|development|stage\s+one|stage\s+two|pre-?training|fine-?tuning|interconnected components?|foundation model|data engine|prompt-?able segmentation task|backbone|encoder|decoder|object quer(?:y|ies)|feed[ -]?forward network)\b", + re.IGNORECASE, + ) + ranked: list[tuple[int, int, dict[str, Any]]] = [] + for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): + if not isinstance(figure, dict): + continue + caption = str(figure.get("caption") or "").strip() + if not caption or not candidate_pattern.search(caption): + continue + score = framework_caption_score(caption, index) + ranked.append((score, -index, {**figure, "selection_score": score})) + ordered = [item for _, _, item in sorted(ranked, reverse=True)] + if not ordered: + return [] + floor = max(1, int(ordered[0]["selection_score"]) - 10) + return [item for item in ordered if int(item.get("selection_score") or 0) >= floor][: max(1, int(limit))] + + +def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 1) -> dict[str, tuple[int, int]]: + """Prioritize labels spatially inside likely overview figures. + + Figure-local labels are often short text blocks with no section heading, so + a section-first evidence sort can push them past the fast planner's character + budget. Recover the visual region around the strongest overview captions + and rank those blocks immediately after their caption. + """ + candidates = rank_framework_figures(parsed, limit=limit) + if not candidates: + return {} + + pages = { + int(page.get("page") or 0): page + for page in parsed.get("pages", []) + if isinstance(page, dict) and page.get("page") + } + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict)] + priority: dict[str, tuple[int, int]] = {} + for candidate_index, figure in enumerate(candidates): + page_number = int(figure.get("page") or 0) + page = pages.get(page_number, {}) + page_height = float(page.get("height") or 0.0) + caption = str(figure.get("caption") or "") + caption_record = next( + ( + item + for item in evidence + if int(item.get("page") or 0) == page_number + and str(item.get("kind") or "").casefold() == "caption" + and str(item.get("text") or "").startswith(caption[:60]) + ), + None, + ) + if not caption_record: + continue + caption_id = str(caption_record.get("id") or "") + if caption_id: + priority[caption_id] = (candidate_index * 2, -1) + caption_bbox = caption_record.get("bbox") + if not (isinstance(caption_bbox, list) and len(caption_bbox) == 4 and page_height > 0): + continue + caption_top = float(caption_bbox[1]) + caption_bottom = float(caption_bbox[3]) + caption_center_ratio = ((caption_top + caption_bottom) / 2.0) / page_height + for item in evidence: + if int(item.get("page") or 0) != page_number: + continue + item_id = str(item.get("id") or "") + item_bbox = item.get("bbox") + if not item_id or not (isinstance(item_bbox, list) and len(item_bbox) == 4): + continue + item_top = float(item_bbox[1]) + item_bottom = float(item_bbox[3]) + if caption_center_ratio >= 0.38: + in_visual_region = item_bottom <= caption_bottom + 6.0 and item_top >= max(0.0, caption_top - 0.62 * page_height) + elif caption_center_ratio <= 0.25: + # Figure captions near the top of a page are still normally + # below the figure. Keep the compact visual band immediately + # above the caption and exclude the article body below it. + in_visual_region = item_bottom <= caption_bottom + 6.0 and item_top >= max(0.0, caption_top - 0.28 * page_height) + else: + in_visual_region = item_bottom >= caption_top - 0.42 * page_height and item_top <= caption_bottom + 0.42 * page_height + if not in_visual_region: + continue + block_id = str(item.get("block_id") or "") + reading_order = next( + ( + int(block.get("reading_order") or 0) + for block in page.get("blocks", []) + if isinstance(block, dict) and str(block.get("id") or "") == block_id + ), + 0, + ) + value = (candidate_index * 2 + 1, reading_order) + if item_id not in priority or value < priority[item_id]: + priority[item_id] = value + return priority + + +def overview_figure_spatial_excerpt(parsed: dict[str, Any], max_items: int = 180) -> str: + """Return compact bbox-aware evidence for likely overview figure regions.""" + priority = _overview_figure_evidence_priority(parsed) + if not priority: + return "[]" + pages = { + int(page.get("page") or 0): page + for page in parsed.get("pages", []) + if isinstance(page, dict) and page.get("page") + } + records = [] + for item in parsed.get("evidence", []): + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "") + if item_id not in priority: + continue + page_number = int(item.get("page") or 0) + page = pages.get(page_number, {}) + width = max(1.0, float(page.get("width") or 1.0)) + height = max(1.0, float(page.get("height") or 1.0)) + bbox = item.get("bbox") if isinstance(item.get("bbox"), list) and len(item.get("bbox")) == 4 else None + bbox_percent = None + if bbox: + bbox_percent = [ + round(float(bbox[0]) / width, 4), + round(float(bbox[1]) / height, 4), + round(float(bbox[2]) / width, 4), + round(float(bbox[3]) / height, 4), + ] + records.append({ + "id": item_id, + "page": page_number, + "block_id": str(item.get("block_id") or ""), + "kind": str(item.get("kind") or "paragraph"), + "reading_order": priority[item_id][1], + "bbox_percent": bbox_percent, + "text": str(item.get("text") or ""), + "_priority": priority[item_id], + }) + records.sort(key=lambda item: (item["_priority"], item["page"], item["block_id"])) + for item in records: + item.pop("_priority", None) + return json.dumps(records[: max(1, int(max_items))], ensure_ascii=False, separators=(",", ":")) + + +def overview_figure_panel_excerpt(parsed: dict[str, Any], max_items_per_panel: int = 48) -> str: + """Group concise target-figure labels by explicit panel markers.""" + def role_hint(text_value: str) -> str | None: + low = text_value.casefold().strip(" .:") + words = text_value.split() + if len(text_value) > 72 or len(words) > 8 or low.startswith(("we ", "with ", "the vector ", "each node ")): + return None + if re.search(r"\b(loss|objective|optimization)\b", low): + return "objective" + if re.search(r"\b(predictions?|predicted .*distribution|outputs?|results?)\b", low) or re.fullmatch(r"y\s*test", low): + return "output" + # Figure authors often use a determiner-led dataset phrase as a heading + # for a whole visual region (for example, "A synthetic dataset"), not + # as a separate tensor/table node in the data-flow graph. Keep explicit + # operational labels such as "Input dataset" and X_train/Y_test as + # inputs, while treating descriptive noun phrases as group containers. + if re.fullmatch(r"(?:a|an|the)\s+(?:[\w-]+\s+){1,6}datasets?", low): + return "group" + if re.search(r"\b(dataset|input data|input dataset|source data)\b", low) or re.fullmatch(r"[xy]\s*(?:train|test)", low): + return "input" + if re.search(r"\b(layer|block|stack)\b", low) and re.search(r"\b\d+\s*[×x]\b|\(\d+", low): + return "group" + if len(words) <= 5 and re.search(r"\b(model|network|encoder|decoder|attention|mlp|transformer|retriever|generator|classifier|backbone|module|tabpfn)\b", low): + return "module" + if re.fullmatch(r"[A-Z][A-Z0-9-]{2,}", text_value.strip()): + return "module" + return None + + priority = _overview_figure_evidence_priority(parsed, limit=1) + if not priority: + return "{}" + candidate_ids = {item_id for item_id, value in priority.items() if value[0] <= 1} + evidence = [ + item for item in parsed.get("evidence", []) + if isinstance(item, dict) and str(item.get("id") or "") in candidate_ids + ] + caption_record = next((item for item in evidence if str(item.get("kind") or "").casefold() == "caption"), None) + if not caption_record: + return "{}" + page_number = int(caption_record.get("page") or 0) + page = next((item for item in parsed.get("pages", []) if isinstance(item, dict) and int(item.get("page") or 0) == page_number), {}) + width = max(1.0, float(page.get("width") or 1.0)) + height = max(1.0, float(page.get("height") or 1.0)) + caption_bbox = caption_record.get("bbox") if isinstance(caption_record.get("bbox"), list) and len(caption_record.get("bbox")) == 4 else [0.0, height, width, height] + visual_records = [ + item for item in evidence + if item is not caption_record + and isinstance(item.get("bbox"), list) + and len(item.get("bbox")) == 4 + and float(item["bbox"][3]) <= float(caption_bbox[3]) + 6.0 + ] + caption = str(caption_record.get("text") or "") + panel_matches = list(re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", caption, re.IGNORECASE)) + panel_labels = [match.group(1).casefold() for match in panel_matches] + descriptions: dict[str, str] = {} + for index, match in enumerate(panel_matches): + end = panel_matches[index + 1].start() if index + 1 < len(panel_matches) else len(caption) + descriptions[match.group(1).casefold()] = re.sub(r"\s+", " ", caption[match.end():end]).strip(" .") + + markers: list[tuple[str, dict[str, Any]]] = [] + for panel_label in panel_labels: + candidates = [item for item in visual_records if str(item.get("text") or "").strip().casefold() == panel_label] + if candidates: + markers.append((panel_label, min(candidates, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))))) + # Some overview figures declare their stages only inside the artwork. CLIP, + # for example, uses ``(1)``, ``(2)``, and ``(3)`` headings while its caption + # contains no a/b/c enumeration. Treat a numbered or lettered heading plus + # descriptive text as an explicit panel marker. + if len(markers) < 2: + heading_markers: list[tuple[str, dict[str, Any]]] = [] + heading_pattern = re.compile(r"^\s*\((?P