From d5e9e53b5154944a3c0e954811542738b49a44b2 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 16:04:37 +0800 Subject: [PATCH 01/74] Add smart API asset policy for editable rebuilds --- .gitignore | 2 + README.md | 12 +- docs/gpt-reproduction-workflow.md | 12 +- docs/paper-to-image.md | 4 + docs/rebuild-editable.md | 12 +- rfs/cli.py | 6 +- rfs/editable_rebuild.py | 233 +++++++++++++++++++++++++++++- rfs/paper_to_image/workflow.py | 4 +- rfs/professional_rebuild.py | 19 ++- rfs/reference_analyzer.py | 2 - tests/test_rebuild_editable.py | 58 ++++++++ 11 files changed, 342 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 3250ca3..413d876 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ research_figure_studio.egg-info/ venv/ env/ tmp/translation_env/ +tmp/codex-presentations/ +tmp/pdfs/*/ *.key *.pem diff --git a/README.md b/README.md index cad8544..fb27c69 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,7 @@ rfs rebuild-editable-pro ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild_pro" ` --asset-mode api ` + --asset-policy smart-api ` --repair-rounds 2 ` --export-preview ``` @@ -235,14 +236,21 @@ write `professional_gap_report.json` against a specialized-script output. Use `--repair-mode vlm` only when you want the VLM to apply constrained DSL patches after preview comparison. -Lower-cost mode, using VLM structure planning but keeping reference crops as -slot assets: +In `smart-api` policy, reference crops are only used as image-generation context. +They are not inserted as final PPT assets. Text-like slots are filtered into +editable text, duplicate complex icons reuse one generated asset, and API +failures fall back to placeholders instead of reference crops. + +Lower-cost structure check. With `smart-api`, this does not keep reference crops +as final assets; it writes placeholders for visual slots while still producing +the DSL, overlays, and reports: ```powershell rfs rebuild-editable ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild_crop" ` --asset-mode crop ` + --asset-policy smart-api ` --layout-mode hybrid ` --control-mode hybrid ``` diff --git a/docs/gpt-reproduction-workflow.md b/docs/gpt-reproduction-workflow.md index e80ba79..e36e4e9 100644 --- a/docs/gpt-reproduction-workflow.md +++ b/docs/gpt-reproduction-workflow.md @@ -21,7 +21,7 @@ rfs rebuild-editable --reference input.png --out output\demo --asset-mode api -- For higher-quality reference-only reproduction, use the scripted professional workflow. It asks the VLM to generate a controlled Figure DSL that mimics the best one-off rebuild scripts, then compiles that DSL safely: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --repair-rounds 2 --export-preview +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --asset-policy smart-api --repair-rounds 2 --export-preview ``` The evaluation command is: @@ -99,7 +99,7 @@ Never write API keys into source files, docs, fixtures, or committed outputs. ### 1. Start With A Cost-Safe Evaluation -Use crop assets first. This validates layout/control/semantic VLM quality without spending image-generation credits: +Use crop/placeholder assets first only as an engineering check. The evaluator may use crop for cost-safe structure comparison; production/pro runs should use `smart-api` so crop is not inserted as a final PPT asset: ```powershell rfs rebuild-editable-eval ` @@ -136,6 +136,7 @@ rfs rebuild-editable ` --reference "C:\path\figure.png" ` --out "output\editable_rebuild" ` --asset-mode api ` + --asset-policy smart-api ` --layout-mode hybrid ` --control-mode hybrid ` --text-mode ocr ` @@ -215,6 +216,7 @@ A result is acceptable only if: - Complex visuals are slot-level assets under `assets/`. - `rebuild_vlm_validation_report.json` has `status: pass` or only explainable warnings. - `asset_generation_report.json` records API request counts and fallbacks. +- `asset_decision_report.json`, `text_asset_filter_report.json`, and `api_asset_plan.json` explain which slots are API-generated, reused, or rejected as text. - `reference_geometry_overlay.png` and `reference_controls_overlay.png` visually match the reference closely enough for the current stage. For `rfs rebuild-editable-pro`, additionally check: @@ -258,10 +260,10 @@ Use these modes deliberately: rfs rebuild-editable --reference input.png --out output\placeholder --asset-mode placeholder --layout-mode heuristic --control-mode heuristic # VLM structure test without image generation cost. -rfs rebuild-editable --reference input.png --out output\crop --asset-mode crop --layout-mode hybrid --control-mode hybrid +rfs rebuild-editable --reference input.png --out output\crop --asset-mode crop --asset-policy smart-api --layout-mode hybrid --control-mode hybrid # Full quality run. -rfs rebuild-editable --reference input.png --out output\api --asset-mode api --layout-mode hybrid --control-mode hybrid +rfs rebuild-editable --reference input.png --out output\api --asset-mode api --asset-policy smart-api --layout-mode hybrid --control-mode hybrid ``` Default economy behavior: @@ -325,7 +327,7 @@ Action: - Inspect `asset_generation_specs.json` for `generation_aspect_ratio`, `main_subject_fill_target`, and `prompt_subject`. - Regenerate only failed slots with `--regenerate-slots`. -- Do not globally crop assets unless the user explicitly approves a controlled trim experiment. +- Do not use reference crops as final PPT assets in smart-api/pro runs. Crops are context for API generation only. ## Test Commands diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 190b341..be7ef90 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -48,6 +48,10 @@ Positive references are classified as `arbor`, `linear`, `tripanel`, or `dense-m The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. +When `--aspect-ratio auto` is used, the template keeps its internal normalized +geometry while the generation canvas uses the nearest native Image2 ratio +(`3:2`, `2:3`, or `1:1`). This avoids semantic cropping after generation. + ## Production Gates Every Image2 candidate must pass: diff --git a/docs/rebuild-editable.md b/docs/rebuild-editable.md index aac01ac..2d9a42a 100644 --- a/docs/rebuild-editable.md +++ b/docs/rebuild-editable.md @@ -31,13 +31,13 @@ rfs rebuild-editable --reference input.png --out output\demo --asset-mode api -- Professional scripted rebuild: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --repair-rounds 2 --export-preview +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode api --asset-policy smart-api --repair-rounds 2 --export-preview ``` Compare against a known high-quality specialized rebuild: ```powershell -rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode crop --benchmark-out output\autofigure_architecture_ai_rebuild +rfs rebuild-editable-pro --reference input.png --out output\demo_pro --asset-mode crop --asset-policy smart-api --benchmark-out output\autofigure_architecture_ai_rebuild ``` Offline professional smoke test: @@ -52,6 +52,7 @@ rfs rebuild-editable-pro --reference input.png --out output\demo_pro_placeholder --reference Input reference image. --out Output directory. --asset-mode api | crop | placeholder. Default: api. +--asset-policy legacy | smart-api. Default: smart-api. Use legacy only for old crop-fallback behavior. --asset-workers Parallel asset workers. Default: 4. --asset-retries Retries used by strict regeneration. Default: 1. --economy-mode Reuse accepted/passing assets. Enabled by default. @@ -77,6 +78,8 @@ rebuild-editable-pro Use the controlled professional Figure DSL workflow. --compile-only Recompile from professional_rebuild_script.dsl.json without VLM planning or asset API calls. ``` +`smart-api` disables final reference-crop assets. Local crops are still saved and passed as API reference context, but final PPT assets are API-generated, reused from another generated slot, or skipped when the region is primarily editable text. If `--asset-mode crop --asset-policy smart-api` is used for a low-cost dry run, the final visual asset is a placeholder and the report records `crop_disabled_by_smart_api_policy_placeholder`. + ## API Environment `--asset-mode api` uses the same image-generation route as the specialized rebuild scripts: @@ -112,7 +115,7 @@ $env:GEMINI_API_KEY=$env:API_KEY $env:GEMINI_GEN_IMG_URL='https://your-provider/v1beta/models/your-image-model:generateContent' ``` -If the API call fails for a slot, v1 falls back to the reference crop for that slot and records the failure in `asset_generation_report.json`. +With `--asset-policy legacy`, if the API call fails for a slot, v1 falls back to the reference crop for that slot and records the failure in `asset_generation_report.json`. With `--asset-policy smart-api`, crop fallback is disabled; API failure falls back to a placeholder and records `api_failed_placeholder_fallback`. ## Output Files @@ -131,6 +134,9 @@ asset_generation_specs.json asset_generation_report.json asset_economy_report.json asset_ratio_fit_report.json +asset_decision_report.json +text_asset_filter_report.json +api_asset_plan.json figure_program.json composition_quality_report.json rebuild_vlm_validation_report.json diff --git a/rfs/cli.py b/rfs/cli.py index 7839330..6b93653 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -151,6 +151,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild.add_argument("--reference", required=True, help="Reference image path.") rebuild.add_argument("--out", required=True, help="Output directory.") rebuild.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") + rebuild.add_argument("--asset-policy", choices=["legacy", "smart-api"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") rebuild.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") rebuild.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") rebuild.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") @@ -179,6 +180,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild_pro.add_argument("--reference", required=True, help="Reference image path.") rebuild_pro.add_argument("--out", required=True, help="Output directory.") rebuild_pro.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") + rebuild_pro.add_argument("--asset-policy", choices=["legacy", "smart-api"], default="smart-api", help="Slot asset decision policy. Default smart-api disables final crop assets, filters text slots, and reuses duplicate API assets.") rebuild_pro.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") rebuild_pro.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") rebuild_pro.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") @@ -230,7 +232,7 @@ def build_parser() -> argparse.ArgumentParser: def _print_human(data: dict) -> None: if "ok" in data: print(f"ok: {data['ok']}") - for key in ["out_dir", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "control_mode", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: + for key in ["out_dir", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_policy", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "control_mode", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: if key in data: print(f"{key}: {data[key]}") if data.get("presentations_qa"): @@ -341,6 +343,7 @@ def main(argv: list[str] | None = None) -> int: vlm_layout_adapter=rebuild_adapters["layout"] if args.layout_mode in {"vlm", "hybrid"} else None, control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, + asset_policy=args.asset_policy, ) elif args.command == "rebuild-editable-eval": result = evaluate_rebuild_vlm( @@ -374,6 +377,7 @@ def main(argv: list[str] | None = None) -> int: semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, repair_adapter=vlm_professional_repair_adapter if args.repair_mode == "vlm" else None, benchmark_out=args.benchmark_out, + asset_policy=args.asset_policy, ) elif args.command == "coevolve-image": result = run_image_coevolution( diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 54b571e..87ecc72 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -36,6 +36,10 @@ } +TEXT_ASSET_OVERLAP_THRESHOLD = 0.60 +SIMPLE_PRIMITIVE_TYPES = {"legend_marker", "simple_marker", "status_marker", "node_marker"} + + def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: x = max(0.0, min(0.995, x)) y = max(0.0, min(0.995, y)) @@ -385,7 +389,157 @@ def _load_accepted_assets(path: Path) -> dict: return {} -def _make_asset_specs(program: dict, reference_path: Path, out: Path) -> list[dict]: +def _bbox_overlap_ratio(a: dict, b: dict) -> float: + ax1, ay1 = float(a["x"]), float(a["y"]) + ax2, ay2 = ax1 + float(a["w"]), ay1 + float(a["h"]) + bx1, by1 = float(b["x"]), float(b["y"]) + bx2, by2 = bx1 + float(b["w"]), by1 + float(b["h"]) + ix = max(0.0, min(ax2, bx2) - max(ax1, bx1)) + iy = max(0.0, min(ay2, by2) - max(ay1, by1)) + return (ix * iy) / max(float(a["w"]) * float(a["h"]), 0.000001) + + +def _text_program_boxes(program: dict) -> list[dict]: + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} + boxes = [] + for item in text_program.get("items", []) or []: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + if not str(item.get("text") or "").strip(): + continue + boxes.append(item) + return boxes + + +def _normalized_reuse_key(spec: dict) -> str: + import re + + subject = str(spec.get("prompt_subject") or spec.get("semantic_role") or spec.get("slot_id") or "").lower() + subject = re.sub(r"\b(executor|branch|step|slot|icon)\s*[a-z0-9]*\b", " ", subject) + subject = re.sub(r"\b[a-d]\b|\b[0-9]+\b", " ", subject) + subject = re.sub(r"[^a-z0-9]+", " ", subject).strip() + if not subject: + subject = str(spec.get("asset_type") or spec.get("slot_type") or "generic") + return f"{spec.get('asset_type') or spec.get('slot_type') or 'generic'}::{subject}" + + +def _apply_smart_asset_policy(specs: list[dict], program: dict, out: Path, asset_policy: str) -> list[dict]: + text_items = _text_program_boxes(program) + filtered: list[dict] = [] + text_filter_report = [] + decision_report = [] + api_plan = { + "summary": "Smart API asset plan. Reference crops are only prompt/context inputs; they are not final PPT assets.", + "asset_policy": asset_policy, + "unique_api_assets": 0, + "reused_slots": 0, + "skipped_text_regions": 0, + "ppt_primitive_slots": 0, + "estimated_api_requests": 0, + "reuse_groups": [], + "recommended_api_slots": [], + } + + for spec in specs: + slot_id = str(spec["slot_id"]) + bbox = spec["slot_bbox_percent"] + overlaps = [ + { + "text_id": item.get("id"), + "text": item.get("text"), + "overlap_ratio": round(_bbox_overlap_ratio(bbox, item["bbox_percent"]), 4), + } + for item in text_items + ] + best = max(overlaps, key=lambda item: item["overlap_ratio"], default=None) + if best and float(best["overlap_ratio"]) >= TEXT_ASSET_OVERLAP_THRESHOLD: + spec["asset_decision"] = "text_region" + spec["asset_generation_mode"] = "skip_asset" + spec["asset_decision_reason"] = f"OCR/DSL text overlap {best['overlap_ratio']:.2f}; region is primarily editable text." + text_filter_report.append({ + "candidate_id": slot_id, + "rejected_as_asset": True, + "converted_to": "existing_editable_text", + "reason": spec["asset_decision_reason"], + "matched_text_id": best.get("text_id"), + "matched_text": best.get("text"), + "ocr_or_text_overlap": best.get("overlap_ratio"), + }) + decision_report.append({ + "slot_id": slot_id, + "decision": "text_region", + "needs_api": False, + "reason": spec["asset_decision_reason"], + }) + api_plan["skipped_text_regions"] += 1 + continue + if str(spec.get("asset_type") or spec.get("slot_type") or "").lower() in SIMPLE_PRIMITIVE_TYPES: + spec["asset_decision"] = "ppt_primitive" + spec["asset_generation_mode"] = "skip_asset" + spec["asset_decision_reason"] = "Simple marker should be represented by editable PPT primitives, not an image asset." + decision_report.append({ + "slot_id": slot_id, + "decision": "ppt_primitive", + "needs_api": False, + "reason": spec["asset_decision_reason"], + }) + api_plan["ppt_primitive_slots"] += 1 + continue + spec["asset_decision"] = "api_generated" + spec["asset_generation_mode"] = "api" + spec["asset_decision_reason"] = "Complex non-text visual slot; final asset should be API-generated from reference crop context." + spec["reuse_group_key"] = _normalized_reuse_key(spec) + filtered.append(spec) + + primary_by_key: dict[str, dict] = {} + reuse_groups: dict[str, list[str]] = {} + for spec in filtered: + key = str(spec["reuse_group_key"]) + reuse_groups.setdefault(key, []).append(str(spec["slot_id"])) + if key not in primary_by_key: + primary_by_key[key] = spec + decision = "api_generated" + api_plan["unique_api_assets"] += 1 + api_plan["recommended_api_slots"].append(str(spec["slot_id"])) + else: + primary = primary_by_key[key] + spec["asset_decision"] = "reuse_existing" + spec["asset_generation_mode"] = "reuse_existing" + spec["reuse_source_slot_id"] = primary["slot_id"] + spec["reuse_source_asset_id"] = primary["asset_id"] + decision = "reuse_existing" + api_plan["reused_slots"] += 1 + decision_report.append({ + "slot_id": spec["slot_id"], + "decision": decision, + "reused_asset_id": spec.get("reuse_source_asset_id"), + "reuse_group_key": key, + "needs_api": decision == "api_generated", + "reason": spec.get("asset_decision_reason"), + }) + api_plan["estimated_api_requests"] = api_plan["unique_api_assets"] + api_plan["reuse_groups"] = [ + {"reuse_group_key": key, "slots": slots} + for key, slots in sorted(reuse_groups.items()) + if len(slots) > 1 + ] + kept_slot_ids = {str(spec["slot_id"]) for spec in filtered} + program["slots"] = [slot for slot in program.get("slots", []) if str(slot.get("id")) in kept_slot_ids] + write_json(out / "asset_decision_report.json", { + "summary": "Slot-level asset decisions. In smart-api mode, final crop assets are disabled; crops are only API reference inputs.", + "asset_policy": asset_policy, + "decisions": decision_report, + }) + write_json(out / "text_asset_filter_report.json", { + "summary": "Text-like candidate slots rejected before asset generation.", + "overlap_threshold": TEXT_ASSET_OVERLAP_THRESHOLD, + "items": text_filter_report, + }) + write_json(out / "api_asset_plan.json", api_plan) + return filtered + + +def _make_asset_specs(program: dict, reference_path: Path, out: Path, asset_policy: str = "legacy") -> list[dict]: specs = [] background = str(program["canvas"].get("background") or "#FFFFFF") for slot in program["slots"]: @@ -426,6 +580,22 @@ def _make_asset_specs(program: dict, reference_path: Path, out: Path) -> list[di "no_crop_postprocess": True, } specs.append(spec) + if asset_policy == "smart-api": + return _apply_smart_asset_policy(specs, program, out, asset_policy) + write_json(out / "asset_decision_report.json", { + "summary": "Legacy asset decision report.", + "asset_policy": asset_policy, + "decisions": [{"slot_id": spec["slot_id"], "decision": asset_policy, "needs_api": asset_policy == "api"} for spec in specs], + }) + write_json(out / "text_asset_filter_report.json", { + "summary": "Text asset filter disabled in legacy asset policy.", + "items": [], + }) + write_json(out / "api_asset_plan.json", { + "summary": "API asset plan disabled in legacy asset policy.", + "asset_policy": asset_policy, + "estimated_api_requests": len(specs) if asset_policy == "api" else 0, + }) return specs @@ -439,6 +609,7 @@ def _generate_assets( economy_mode: bool, regenerate_slots: set[str], strict_asset_regeneration: bool, + asset_policy: str = "legacy", ) -> tuple[list[dict], dict]: asset_dir = ensure_dir(out / "assets") accepted = _load_accepted_assets(out / "accepted_assets.json") @@ -448,6 +619,14 @@ def _generate_assets( def run_one(spec: dict) -> dict: slot_id = spec["slot_id"] out_path = asset_dir / f"{spec['asset_id']}.png" + if spec.get("asset_decision") == "reuse_existing": + source_asset = asset_dir / f"{spec['reuse_source_asset_id']}.png" + if source_asset.exists(): + shutil.copyfile(source_asset, out_path) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]) + decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) + return {**spec, **metrics, **decision, "status": "reused_from_group", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_group_asset", "fallback_used": False} + return {**spec, "status": "reuse_source_missing", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": f"reuse source missing: {source_asset}"} locked = bool(accepted.get(slot_id, {}).get("accepted")) or bool(accepted.get(spec["asset_id"], {}).get("accepted")) if out_path.exists() and economy_mode and slot_id not in regenerate_slots and not strict_asset_regeneration: metrics = _foreground_metrics(out_path, spec["background_color_hex"]) @@ -463,10 +642,14 @@ def run_one(spec: dict) -> dict: _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) request_count = 0 status = "placeholder" - elif asset_mode == "crop": + elif asset_mode == "crop" and asset_policy != "smart-api": shutil.copyfile(crop_path, out_path) request_count = 0 status = "reference_crop" + elif asset_mode == "crop" and asset_policy == "smart-api": + _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) + request_count = 0 + status = "crop_disabled_by_smart_api_policy_placeholder" else: result = _api_generate_asset(spec, crop_path, out_path) request_count = int(result.get("api_requests_attempted", 1)) @@ -486,7 +669,7 @@ def run_one(spec: dict) -> dict: } except Exception as exc: last_error = str(exc) - if asset_mode == "api": + if asset_mode == "api" and asset_policy != "smart-api": try: shutil.copyfile(crop_path, out_path) metrics = _foreground_metrics(out_path, spec["background_color_hex"]) @@ -505,20 +688,47 @@ def run_one(spec: dict) -> dict: } except Exception: pass + if asset_mode == "api" and asset_policy == "smart-api": + try: + _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]) + decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) + return { + **spec, + **metrics, + **decision, + "status": "api_failed_placeholder_fallback", + "asset_path": str(out_path), + "api_requests_attempted": 1, + "attempt": attempt, + "economy_decision": "placeholder_after_api_error_crop_disabled", + "fallback_used": True, + "error": last_error, + } + except Exception: + pass return {**spec, "status": "failed", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": last_error} + primary_specs = [spec for spec in specs if spec.get("asset_decision") != "reuse_existing"] + reuse_specs = [spec for spec in specs if spec.get("asset_decision") == "reuse_existing"] workers = max(1, min(int(asset_workers), 12)) with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(run_one, spec) for spec in specs] + futures = [pool.submit(run_one, spec) for spec in primary_specs] for future in as_completed(futures): report = future.result() reports.append(report) api_requests += int(report.get("api_requests_attempted") or 0) + for spec in reuse_specs: + report = run_one(spec) + reports.append(report) + api_requests += int(report.get("api_requests_attempted") or 0) reports.sort(key=lambda item: str(item.get("slot_id"))) program["assets"] = [{"id": spec["asset_id"], "path": f"assets/{spec['asset_id']}.png", "source": "slot_asset"} for spec in specs] summary = { "summary": "Asset generation report for reusable editable rebuild workflow.", "asset_mode": asset_mode, + "asset_policy": asset_policy, + "final_crop_assets_allowed": asset_policy != "smart-api", "economy_mode": economy_mode, "asset_retries": asset_retries, "strict_asset_regeneration": strict_asset_regeneration, @@ -531,6 +741,7 @@ def run_one(spec: dict) -> dict: "accepted_assets_file": str(out / "accepted_assets.json"), "regenerate_slots": sorted(regenerate_slots), "thresholds": {key: {"min": value[0], "max": value[1]} for key, value in ASSET_THRESHOLDS.items()}, + "asset_policy": asset_policy, "assets": [{ "slot_id": item.get("slot_id"), "slot_type": item.get("slot_type"), @@ -710,6 +921,7 @@ def rebuild_editable( vlm_layout_adapter: Callable | None = None, control_adapter: Callable | None = None, semantic_adapter: Callable | None = None, + asset_policy: str = "legacy", ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -728,6 +940,7 @@ def rebuild_editable( "text_mode": text_mode, "control_mode": control_mode, "layout_mode": layout_mode, + "asset_policy": asset_policy, "skip_analysis": skip_analysis, "compile_only": compile_only, }) @@ -789,14 +1002,18 @@ def rebuild_editable( write_json(out_path / "slot_semantic_report.json", semantic_report) write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) - specs = _make_asset_specs(program, archived_reference, out_path) + specs = _make_asset_specs(program, archived_reference, out_path, asset_policy=asset_policy) + if asset_policy == "smart-api": + reference_geometry["slots"] = program.get("slots", []) + write_json(out_path / "reference_geometry.json", reference_geometry) + write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) write_json(out_path / "asset_generation_specs.json", {"summary": "Slot-level asset generation specs.", "asset_mode": asset_mode, "specs": specs}) regen = set() if isinstance(regenerate_slots, str): regen = {item.strip() for item in regenerate_slots.split(",") if item.strip()} elif isinstance(regenerate_slots, list): regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} - asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration) + asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration, asset_policy=asset_policy) vlm_validation_report = build_rebuild_vlm_validation_report(out_path, reference_geometry, reference_controls, semantic_report, asset_summary) write_json(out_path / "figure_program.json", program) @@ -828,6 +1045,7 @@ def rebuild_editable( "asset_workers": asset_workers, "asset_retries": asset_retries, "economy_mode": economy_mode, + "asset_policy": asset_policy, "api_requests_attempted": asset_summary.get("api_requests_attempted", 0), "asset_count": len(asset_reports), "slot_count": len(program.get("slots", [])), @@ -847,6 +1065,9 @@ def rebuild_editable( "asset_generation_specs": str(out_path / "asset_generation_specs.json"), "asset_generation_report": str(out_path / "asset_generation_report.json"), "asset_economy_report": str(out_path / "asset_economy_report.json"), + "asset_decision_report": str(out_path / "asset_decision_report.json"), + "text_asset_filter_report": str(out_path / "text_asset_filter_report.json"), + "api_asset_plan": str(out_path / "api_asset_plan.json"), "figure_program": str(out_path / "figure_program.json"), "composition_quality_report": str(out_path / "composition_quality_report.json"), }, diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 35c9a4c..9690f4c 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -114,7 +114,9 @@ def run_paper_to_image( selected_template = select_template(template_profiles, paper_review, requested=template, target_ratio=str(preferences.get("aspect_ratio") or "auto")) if str(preferences.get("aspect_ratio") or "auto") == "auto": ratio = float(selected_template.get("source_aspect_ratio") or selected_template.get("aspect_ratio") or 16 / 9) - preferences["aspect_ratio"] = f"{ratio:.3f}:1.000" + preferences["template_source_aspect_ratio"] = round(ratio, 6) + preferences["aspect_ratio"] = "3:2" if ratio >= 1.25 else "2:3" if ratio <= 0.80 else "1:1" + preferences["generation_ratio_policy"] = "nearest_native_image2_canvas; preserve_template_internal_geometry; no_semantic_crop" write_json(root / "preferences.json", preferences) write_json(root / "selected_template.json", {"summary": "Automatically or explicitly selected content-free architecture template.", **selected_template}) blueprint_report = render_layout_blueprint(selected_template, root / "layout_blueprint.png", target_ratio=preferences["aspect_ratio"]) diff --git a/rfs/professional_rebuild.py b/rfs/professional_rebuild.py index d9d433b..8aa37fc 100644 --- a/rfs/professional_rebuild.py +++ b/rfs/professional_rebuild.py @@ -108,6 +108,7 @@ def _compile_professional( generate_assets: bool = True, baseline_program: dict | None = None, benchmark_out: str | Path | None = None, + asset_policy: str = "smart-api", ) -> dict: normalized, validation = validate_and_normalize_dsl(dsl, archived_reference, out) write_json(out / "professional_rebuild_script.dsl.json", normalized) @@ -134,7 +135,10 @@ def _compile_professional( _draw_geometry_overlay(archived_reference, out, program) _draw_controls_overlay(archived_reference, out, program.get("arrows", [])) - specs = _make_asset_specs(program, archived_reference, out) + specs = _make_asset_specs(program, archived_reference, out, asset_policy=asset_policy) + if asset_policy == "smart-api": + _write_contracts_from_program(out, program, planner_report) + _draw_geometry_overlay(archived_reference, out, program) write_json(out / "asset_generation_specs.json", {"summary": "Professional DSL slot-level asset generation specs.", "asset_mode": asset_mode, "specs": specs}) regen: set[str] = set() if isinstance(regenerate_slots, str): @@ -142,7 +146,7 @@ def _compile_professional( elif isinstance(regenerate_slots, list): regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} if generate_assets: - asset_reports, asset_summary = _generate_assets(specs, program, out, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration) + asset_reports, asset_summary = _generate_assets(specs, program, out, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration, asset_policy=asset_policy) else: program["assets"] = [{"id": spec["asset_id"], "path": f"assets/{spec['asset_id']}.png", "source": "slot_asset"} for spec in specs] existing_report = _load_json_or_empty(out / "asset_generation_report.json") @@ -168,6 +172,7 @@ def _compile_professional( "rebuild_editable_summary": counts, "dsl_validation": validation, "asset_count": len(asset_reports), + "asset_policy": asset_policy, "professional_gap_report": gap_report, "no_full_image_policy": { "status": "pass", @@ -183,6 +188,7 @@ def _compile_professional( "pptx": str(pptx_path), "preview": str(preview) if preview else None, "asset_mode": asset_mode, + "asset_policy": asset_policy, "asset_workers": asset_workers, "asset_retries": asset_retries, "economy_mode": economy_mode, @@ -202,6 +208,9 @@ def _compile_professional( "figure_program": str(out / "figure_program.json"), "composition_quality_report": str(out / "composition_quality_report.json"), "professional_gap_report": str(out / "professional_gap_report.json"), + "asset_decision_report": str(out / "asset_decision_report.json"), + "text_asset_filter_report": str(out / "text_asset_filter_report.json"), + "api_asset_plan": str(out / "api_asset_plan.json"), }, } @@ -229,6 +238,7 @@ def rebuild_editable_pro( planner_adapter: Callable | None = None, repair_adapter: Callable | None = None, benchmark_out: str | Path | None = None, + asset_policy: str = "smart-api", ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -243,6 +253,7 @@ def rebuild_editable_pro( "reference": str(reference_path), "archived_reference": str(archived_reference), "asset_mode": asset_mode, + "asset_policy": asset_policy, "text_mode": text_mode, "control_mode": control_mode, "layout_mode": layout_mode, @@ -273,6 +284,7 @@ def rebuild_editable_pro( generate_assets=False, baseline_program=None, benchmark_out=benchmark_out, + asset_policy=asset_policy, ) result["compile_only"] = True write_json(out_path / "rebuild_result.json", result) @@ -298,6 +310,7 @@ def rebuild_editable_pro( vlm_layout_adapter=vlm_layout_adapter, control_adapter=control_adapter, semantic_adapter=semantic_adapter, + asset_policy="legacy", ) baseline_program = _load_json_or_empty(out_path / "figure_program.json") if not baseline_program: @@ -324,6 +337,7 @@ def rebuild_editable_pro( export_preview, baseline_program=baseline_program, benchmark_out=benchmark_out, + asset_policy=asset_policy, ) repair_reports = run_professional_repair_rounds(archived_reference, out_path, dsl, repair_rounds=repair_rounds, repair_adapter=repair_adapter) if any(int(item.get("applied_count") or 0) > 0 for item in repair_reports): @@ -343,6 +357,7 @@ def rebuild_editable_pro( export_preview, baseline_program=baseline_program, benchmark_out=benchmark_out, + asset_policy=asset_policy, ) result["repair_rounds"] = len(repair_reports) result["planner_status"] = planner_report.get("status") diff --git a/rfs/reference_analyzer.py b/rfs/reference_analyzer.py index 6bf72af..811a7c3 100644 --- a/rfs/reference_analyzer.py +++ b/rfs/reference_analyzer.py @@ -1385,14 +1385,12 @@ def analyze_reference( control_warnings.append("hybrid_downgraded_to_heuristic:no_api_credentials") allow_legacy_templates = False try: - import os allow_legacy_templates = os.getenv("RFS_ALLOW_LEGACY_TEMPLATES", "").lower() in {"1", "true", "yes"} except Exception: allow_legacy_templates = False reference_slot_source = str(slot_source or "").lower() or "paper" try: - import os reference_slot_source = str(slot_source or os.getenv("RFS_SLOT_SOURCE", "paper")).lower() except Exception: reference_slot_source = str(slot_source or "paper").lower() diff --git a/tests/test_rebuild_editable.py b/tests/test_rebuild_editable.py index 9349be9..f7a142d 100644 --- a/tests/test_rebuild_editable.py +++ b/tests/test_rebuild_editable.py @@ -501,6 +501,64 @@ def fake_repair(_reference, _preview, _dsl, _round): repair = json.loads((out / "professional_repair_round_1.json").read_text(encoding="utf-8")) self.assertEqual(repair["applied_count"], 1) + def test_smart_api_policy_filters_text_slots_and_disables_final_crop(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "pro" + + def fake_planner(_reference, _baseline, _text_geometry): + return { + "canvas": {"width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + "objects": [ + {"type": "canvas", "id": "canvas", "width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + {"type": "text", "id": "text_label", "text": "Output", "bbox_percent": {"x": 0.55, "y": 0.20, "w": 0.20, "h": 0.08}, "font_size_pt": 12}, + {"type": "asset_slot", "id": "misread_text_as_icon", "asset_id": "misread_text_as_icon", "bbox_percent": {"x": 0.55, "y": 0.20, "w": 0.20, "h": 0.08}, "asset_type": "generic", "prompt_subject": "Output text"}, + {"type": "asset_slot", "id": "robot_icon", "asset_id": "robot_icon", "bbox_percent": {"x": 0.20, "y": 0.25, "w": 0.16, "h": 0.24}, "asset_type": "character", "prompt_subject": "friendly robot"}, + ], + } + + result = rebuild_editable_pro(reference, out, asset_mode="crop", asset_policy="smart-api", text_mode="off", repair_rounds=0, planner_adapter=fake_planner) + self.assertTrue(result["ok"]) + self.assertEqual(result["asset_policy"], "smart-api") + report = json.loads((out / "asset_generation_report.json").read_text(encoding="utf-8")) + self.assertFalse(report["final_crop_assets_allowed"]) + statuses = {item["slot_id"]: item["status"] for item in report["assets"]} + self.assertEqual(statuses["robot_icon"], "crop_disabled_by_smart_api_policy_placeholder") + self.assertNotIn("misread_text_as_icon", statuses) + text_filter = json.loads((out / "text_asset_filter_report.json").read_text(encoding="utf-8")) + self.assertEqual(text_filter["items"][0]["candidate_id"], "misread_text_as_icon") + program = json.loads((out / "figure_program.json").read_text(encoding="utf-8")) + self.assertEqual([slot["id"] for slot in program["slots"]], ["robot_icon"]) + + def test_smart_api_policy_reuses_duplicate_complex_icons(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "pro" + + def fake_planner(_reference, _baseline, _text_geometry): + return { + "canvas": {"width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + "objects": [ + {"type": "canvas", "id": "canvas", "width_px": 640, "height_px": 360, "width_in": 12.0, "height_in": 6.75, "background": "#F7F3EA"}, + {"type": "asset_slot", "id": "executor_a_icon", "asset_id": "executor_a_icon", "bbox_percent": {"x": 0.20, "y": 0.25, "w": 0.12, "h": 0.18}, "asset_type": "tool_combo", "prompt_subject": "terminal and database icon"}, + {"type": "asset_slot", "id": "executor_b_icon", "asset_id": "executor_b_icon", "bbox_percent": {"x": 0.40, "y": 0.25, "w": 0.12, "h": 0.18}, "asset_type": "tool_combo", "prompt_subject": "terminal and database icon"}, + ], + } + + with patch.dict("os.environ", {"API_BASE": "", "API_KEY": "", "GEMINI_API_KEY": "", "GEMINI_GEN_IMG_URL": ""}, clear=False): + result = rebuild_editable_pro(reference, out, asset_mode="api", asset_policy="smart-api", text_mode="off", repair_rounds=0, planner_adapter=fake_planner) + self.assertTrue(result["ok"]) + report = json.loads((out / "asset_generation_report.json").read_text(encoding="utf-8")) + self.assertEqual(report["api_requests_attempted"], 1) + statuses = {item["slot_id"]: item["status"] for item in report["assets"]} + self.assertEqual(statuses["executor_a_icon"], "api_failed_placeholder_fallback") + self.assertEqual(statuses["executor_b_icon"], "reused_from_group") + plan = json.loads((out / "api_asset_plan.json").read_text(encoding="utf-8")) + self.assertEqual(plan["unique_api_assets"], 1) + self.assertEqual(plan["reused_slots"], 1) + if __name__ == "__main__": unittest.main() From 410acf60be883383a07e4bcd6cc88bcfc59410e1 Mon Sep 17 00:00:00 2001 From: illion Date: Mon, 20 Jul 2026 01:28:11 +0800 Subject: [PATCH 02/74] Add global rebuild design planning --- rfs/cli.py | 5 + rfs/editable_rebuild.py | 30 +++ rfs/rebuild_design_planner.py | 360 +++++++++++++++++++++++++++++++++ rfs/rebuild_vlm_adapters.py | 30 ++- tests/test_rebuild_editable.py | 26 +++ 5 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 rfs/rebuild_design_planner.py diff --git a/rfs/cli.py b/rfs/cli.py index 6b93653..4c5b1f2 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -157,6 +157,8 @@ def build_parser() -> argparse.ArgumentParser: rebuild.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") rebuild.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") rebuild.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") + rebuild.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") + rebuild.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") rebuild.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") rebuild.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") rebuild.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") @@ -344,6 +346,9 @@ def main(argv: list[str] | None = None) -> int: control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, asset_policy=args.asset_policy, + design_plan_mode=args.design_plan_mode, + design_plan_model=args.design_plan_model, + design_adapter=rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, ) elif args.command == "rebuild-editable-eval": result = evaluate_rebuild_vlm( diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 87ecc72..79757eb 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -16,6 +16,7 @@ from .layout_planner import dominant_palette, estimate_background, plan_reference_layout from .layout_semantic_planner import plan_slot_semantics from .ppt_compiler import compile_ppt +from .rebuild_design_planner import plan_rebuild_design from .rebuild_vlm_validation import build_rebuild_vlm_validation_report from .text_layer import build_text_layer from .utils import ensure_dir, write_json, write_text @@ -922,6 +923,9 @@ def rebuild_editable( control_adapter: Callable | None = None, semantic_adapter: Callable | None = None, asset_policy: str = "legacy", + design_plan_mode: str = "vlm", + design_plan_model: str | None = None, + design_adapter: Callable | None = None, ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -941,10 +945,29 @@ def rebuild_editable( "control_mode": control_mode, "layout_mode": layout_mode, "asset_policy": asset_policy, + "design_plan_mode": design_plan_mode, + "design_plan_model": design_plan_model, "skip_analysis": skip_analysis, "compile_only": compile_only, }) + if skip_analysis and (out_path / "reference_logic_plan.json").exists(): + design_bundle = { + "logic": _load_json_or_empty(out_path / "reference_logic_plan.json"), + "layer_plan": _load_json_or_empty(out_path / "reference_layer_plan.json"), + "generation_plan": _load_json_or_empty(out_path / "reference_generation_plan.json"), + "flow_graph": _load_json_or_empty(out_path / "reference_flow_graph.json"), + } + else: + design_bundle = plan_rebuild_design( + archived_reference, + out_path, + mode=design_plan_mode, + model=design_plan_model, + adapter=design_adapter, + fallback_on_error=True, + ) + if skip_analysis and (out_path / "reference_geometry.json").exists(): program = _program_from_contracts(out_path) reference_geometry = _load_json_or_empty(out_path / "reference_geometry.json") @@ -1046,6 +1069,9 @@ def rebuild_editable( "asset_retries": asset_retries, "economy_mode": economy_mode, "asset_policy": asset_policy, + "design_plan_mode": design_plan_mode, + "design_plan_effective_mode": design_bundle.get("logic", {}).get("effective_mode"), + "design_plan_model": design_bundle.get("logic", {}).get("model"), "api_requests_attempted": asset_summary.get("api_requests_attempted", 0), "asset_count": len(asset_reports), "slot_count": len(program.get("slots", [])), @@ -1056,6 +1082,10 @@ def rebuild_editable( "layout_mode": layout_mode, "reports": { "input_manifest": str(out_path / "input_manifest.json"), + "reference_logic_plan": str(out_path / "reference_logic_plan.json"), + "reference_layer_plan": str(out_path / "reference_layer_plan.json"), + "reference_generation_plan": str(out_path / "reference_generation_plan.json"), + "reference_flow_graph": str(out_path / "reference_flow_graph.json"), "reference_geometry": str(out_path / "reference_geometry.json"), "reference_text_geometry": str(out_path / "reference_text_geometry.json"), "reference_controls": str(out_path / "reference_controls.json"), diff --git a/rfs/rebuild_design_planner.py b/rfs/rebuild_design_planner.py new file mode 100644 index 0000000..d756c17 --- /dev/null +++ b/rfs/rebuild_design_planner.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from PIL import Image + +from .layout_planner import canvas_inches, dominant_palette, estimate_background +from .utils import write_json, write_text +from .vlm_client import resolve_vlm_model + + +LAYER_KINDS = {"background", "panel", "card", "visual_slot", "text", "connector", "legend", "ignore"} +ASSET_POLICIES = {"reference_crop", "api_generate", "placeholder", "ppt_shape", "editable_text", "ppt_connector", "ignore"} + + +def _clamp_bbox(bbox: dict | None) -> dict[str, float]: + bbox = bbox if isinstance(bbox, dict) else {} + x = max(0.0, min(0.995, float(bbox.get("x", 0.0)))) + y = max(0.0, min(0.995, float(bbox.get("y", 0.0)))) + w = max(0.001, min(float(bbox.get("w", 0.001)), 1.0 - x)) + h = max(0.001, min(float(bbox.get("h", 0.001)), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _clean_kind(value: object, default: str = "visual_slot") -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "slot": "visual_slot", + "image": "visual_slot", + "visual": "visual_slot", + "arrow": "connector", + "control": "connector", + "box": "card", + "background_panel": "panel", + } + text = aliases.get(text, text) + return text if text in LAYER_KINDS else default + + +def _clean_policy(value: object, kind: str) -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "crop": "reference_crop", + "screenshot": "reference_crop", + "api": "api_generate", + "generate": "api_generate", + "generated": "api_generate", + "shape": "ppt_shape", + "text": "editable_text", + "arrow": "ppt_connector", + "connector": "ppt_connector", + } + text = aliases.get(text, text) + if text in ASSET_POLICIES: + return text + if kind in {"background", "panel", "card"}: + return "ppt_shape" + if kind == "text": + return "editable_text" + if kind == "connector": + return "ppt_connector" + if kind in {"legend", "ignore"}: + return "ignore" + return "api_generate" + + +def _normalize_layers(raw_layers: list | None) -> list[dict]: + layers = [] + for idx, raw in enumerate(raw_layers or [], start=1): + if not isinstance(raw, dict): + continue + kind = _clean_kind(raw.get("kind") or raw.get("layer_kind") or raw.get("type")) + object_id = str(raw.get("id") or raw.get("object_id") or f"{kind}_{idx:02d}").strip() + if not object_id: + object_id = f"{kind}_{idx:02d}" + policy = _clean_policy(raw.get("asset_source_policy") or raw.get("policy") or raw.get("render_as"), kind) + record = { + "id": object_id, + "kind": kind, + "label": str(raw.get("label") or raw.get("title") or raw.get("prompt_subject") or object_id), + "bbox_percent": _clamp_bbox(raw.get("bbox_percent")), + "asset_source_policy": policy, + "asset_source_reason": str(raw.get("asset_source_reason") or raw.get("reason") or f"{kind}_default_policy"), + "z_order_hint": str(raw.get("z_order_hint") or raw.get("z_layer") or ""), + "confidence": _confidence(raw.get("confidence"), default=0.65), + } + if raw.get("panel_id"): + record["panel_id"] = str(raw.get("panel_id")) + if raw.get("prompt_subject"): + record["prompt_subject"] = str(raw.get("prompt_subject")) + if raw.get("semantic_role"): + record["semantic_role"] = str(raw.get("semantic_role")) + if raw.get("asset_type"): + record["asset_type"] = str(raw.get("asset_type")) + layers.append({key: value for key, value in record.items() if value not in ("", None)}) + return layers + + +def _confidence(value: object, default: float = 0.5) -> float: + try: + return round(max(0.0, min(1.0, float(value))), 4) + except Exception: + return default + + +def _normalize_asset_policies(raw_policies: list | None, layers: list[dict]) -> list[dict]: + by_id = {str(layer["id"]): layer for layer in layers} + policies = [] + for layer in layers: + if layer["kind"] == "visual_slot": + policies.append({ + "slot_id": layer["id"], + "object_id": layer["id"], + "policy": layer["asset_source_policy"], + "reason": layer.get("asset_source_reason") or "layer_policy", + "source": "layer_plan", + "confidence": layer.get("confidence", 0.65), + }) + for raw in raw_policies or []: + if not isinstance(raw, dict): + continue + object_id = str(raw.get("slot_id") or raw.get("object_id") or raw.get("id") or "") + if not object_id: + continue + kind = by_id.get(object_id, {}).get("kind", "visual_slot") + policies.append({ + "slot_id": object_id, + "object_id": object_id, + "policy": _clean_policy(raw.get("policy") or raw.get("asset_source_policy"), kind), + "reason": str(raw.get("reason") or raw.get("asset_source_reason") or "global_generation_policy"), + "source": "design_plan", + "confidence": _confidence(raw.get("confidence"), default=0.7), + }) + deduped = {} + for policy in policies: + deduped[str(policy["slot_id"])] = policy + return list(deduped.values()) + + +def _normalize_flow_graph(raw_flow: dict | None, layers: list[dict]) -> dict: + raw_flow = raw_flow if isinstance(raw_flow, dict) else {} + layer_ids = {str(layer["id"]) for layer in layers} + nodes = [] + seen = set() + for raw in raw_flow.get("nodes", []) if isinstance(raw_flow.get("nodes"), list) else []: + if not isinstance(raw, dict): + continue + node_id = str(raw.get("id") or raw.get("object_id") or "") + if not node_id: + continue + nodes.append({ + "id": node_id, + "label": str(raw.get("label") or node_id), + "kind": _clean_kind(raw.get("kind"), default="visual_slot"), + "confidence": _confidence(raw.get("confidence"), default=0.65), + }) + seen.add(node_id) + for layer in layers: + if layer["kind"] == "visual_slot" and layer["id"] not in seen: + nodes.append({"id": layer["id"], "label": layer.get("label", layer["id"]), "kind": "visual_slot", "confidence": layer.get("confidence", 0.65)}) + seen.add(layer["id"]) + edges = [] + invalid_count = 0 + for idx, raw in enumerate(raw_flow.get("edges", []) if isinstance(raw_flow.get("edges"), list) else [], start=1): + if not isinstance(raw, dict): + continue + source = str(raw.get("source_id") or raw.get("source") or "") + target = str(raw.get("target_id") or raw.get("target") or "") + if not source or not target: + invalid_count += 1 + continue + edges.append({ + "id": str(raw.get("id") or f"flow_{idx:02d}"), + "source_id": source, + "target_id": target, + "relation": str(raw.get("relation") or "flows_to"), + "expected_connector": str(raw.get("expected_connector") or raw.get("connector") or "solid_arrow"), + "confidence": _confidence(raw.get("confidence"), default=0.7), + "source_known_in_layer_plan": source in layer_ids, + "target_known_in_layer_plan": target in layer_ids, + }) + return { + "summary": "Reference flow graph inferred by global rebuild design planner.", + "nodes": nodes, + "edges": edges, + "invalid_edge_count": invalid_count, + } + + +def _heuristic_layers(reference_path: Path) -> tuple[dict, list[dict], list[dict], dict]: + with Image.open(reference_path).convert("RGB") as image: + width_px, height_px = image.size + width_in, height_in = canvas_inches(width_px, height_px) + background = estimate_background(image) + palette = dominant_palette(image) + layers = [ + { + "id": "reference_background", + "kind": "background", + "label": "Reference background", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + "asset_source_policy": "ppt_shape", + "asset_source_reason": "full canvas background remains editable PPT shape", + "z_order_hint": "background", + "confidence": 0.35, + }, + { + "id": "reference_canvas", + "kind": "panel", + "label": "Reference canvas", + "bbox_percent": {"x": 0.025, "y": 0.065, "w": 0.95, "h": 0.84}, + "asset_source_policy": "ppt_shape", + "asset_source_reason": "default panel remains editable PPT shape", + "z_order_hint": "panel", + "confidence": 0.35, + }, + ] + logic = { + "summary": "Reference logic plan inferred by heuristic fallback.", + "mode": "heuristic", + "effective_mode": "heuristic", + "status": "fallback" if False else "pass", + "model": None, + "narrative": { + "figure_type": "workflow", + "main_story": "Reference-only editable rebuild of a scientific diagram.", + "key_message": "Preserve layout, editable text, connectors, and slot-level visual assets.", + }, + "reading_order": ["reference_canvas"], + "canvas": { + "width_px": width_px, + "height_px": height_px, + "width_in": width_in, + "height_in": height_in, + "background": background, + "palette": palette, + }, + "layers": layers, + "warnings": [], + } + generation = { + "summary": "Reference generation plan inferred by heuristic fallback.", + "mode": "heuristic", + "effective_mode": "heuristic", + "asset_policies": _normalize_asset_policies([], layers), + } + flow = _normalize_flow_graph({}, layers) + return logic, layers, generation["asset_policies"], flow + + +def _normalize_design_result(raw: dict, reference_path: Path, mode: str, model: str | None, fallback_reason: str | None = None) -> tuple[dict, dict, dict, dict]: + heuristic_logic, heuristic_layers, _heuristic_policies, heuristic_flow = _heuristic_layers(reference_path) + layers = _normalize_layers(raw.get("layers")) or heuristic_layers + policies = _normalize_asset_policies(raw.get("asset_policies"), layers) + flow = _normalize_flow_graph(raw.get("flow_graph"), layers) + effective_mode = "heuristic" if fallback_reason else ("off" if mode == "off" else mode) + status = "fallback_to_heuristic" if fallback_reason else ("skipped" if mode == "off" else "pass") + logic = { + "summary": "Global reference logic plan for editable rebuild.", + "mode": mode, + "effective_mode": effective_mode, + "status": status, + "model": raw.get("_vlm_model") or model, + "fallback_reason": fallback_reason, + "narrative": raw.get("narrative") if isinstance(raw.get("narrative"), dict) else heuristic_logic["narrative"], + "reading_order": raw.get("reading_order") if isinstance(raw.get("reading_order"), list) else heuristic_logic["reading_order"], + "canvas": heuristic_logic["canvas"], + "layers": layers, + "warnings": [fallback_reason] if fallback_reason else [], + } + layer_plan = { + "summary": "Reference layer plan for editable rebuild.", + "mode": mode, + "effective_mode": effective_mode, + "status": status, + "layer_count": len(layers), + "layers": layers, + "warnings": logic["warnings"], + } + generation = { + "summary": "Reference generation policy plan for editable rebuild.", + "mode": mode, + "effective_mode": effective_mode, + "status": status, + "asset_policies": policies, + "warnings": logic["warnings"], + } + flow["mode"] = mode + flow["effective_mode"] = effective_mode + flow["status"] = status + flow["warnings"] = logic["warnings"] + return logic, layer_plan, generation, flow + + +def _write_markdown(out: Path, logic: dict, layer_plan: dict, generation: dict, flow: dict) -> None: + lines = [ + "# Summary", + str(logic.get("summary") or "Global reference logic plan."), + "", + f"- Mode: {logic.get('mode')}", + f"- Effective mode: {logic.get('effective_mode')}", + f"- Status: {logic.get('status')}", + f"- Model: {logic.get('model')}", + f"- Layer count: {layer_plan.get('layer_count')}", + f"- Asset policy count: {len(generation.get('asset_policies', []))}", + f"- Flow edge count: {len(flow.get('edges', []))}", + ] + narrative = logic.get("narrative") if isinstance(logic.get("narrative"), dict) else {} + if narrative: + lines.extend(["", "## Narrative"]) + for key in ("figure_type", "main_story", "key_message"): + if narrative.get(key): + lines.append(f"- {key}: {narrative[key]}") + write_text(out / "reference_logic_plan.md", "\n".join(lines) + "\n") + + +def plan_rebuild_design( + reference_path: str | Path, + out_dir: str | Path, + *, + mode: str = "vlm", + model: str | None = None, + adapter: Callable[[str | Path, str | None], dict] | None = None, + fallback_on_error: bool = True, +) -> dict: + reference = Path(reference_path) + out = Path(out_dir) + requested = str(mode or "off").lower() + if requested == "off": + raw = {} + logic, layer_plan, generation, flow = _normalize_design_result(raw, reference, "off", None) + elif requested == "heuristic": + raw = {} + logic, layer_plan, generation, flow = _normalize_design_result(raw, reference, "heuristic", None) + elif requested == "vlm": + resolved_model = resolve_vlm_model("RFS_REBUILD_DESIGN_MODEL", "RFS_REBUILD_LAYOUT_MODEL", explicit_model=model) + try: + raw = adapter(reference, resolved_model) if adapter else {} + if not adapter: + raise RuntimeError("design VLM adapter unavailable") + logic, layer_plan, generation, flow = _normalize_design_result(raw, reference, "vlm", resolved_model) + except Exception as exc: + if not fallback_on_error: + raise + logic, layer_plan, generation, flow = _normalize_design_result({}, reference, "vlm", resolved_model, fallback_reason=f"vlm_design_planning_failed:{exc}") + else: + raise ValueError(f"Unsupported design plan mode: {mode}") + + write_json(out / "reference_logic_plan.json", logic) + write_json(out / "reference_layer_plan.json", layer_plan) + write_json(out / "reference_generation_plan.json", generation) + write_json(out / "reference_flow_graph.json", flow) + _write_markdown(out, logic, layer_plan, generation, flow) + return { + "logic": logic, + "layer_plan": layer_plan, + "generation_plan": generation, + "flow_graph": flow, + } diff --git a/rfs/rebuild_vlm_adapters.py b/rfs/rebuild_vlm_adapters.py index f392c39..296ef90 100644 --- a/rfs/rebuild_vlm_adapters.py +++ b/rfs/rebuild_vlm_adapters.py @@ -7,6 +7,33 @@ from .vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available +def vlm_design_adapter(reference_path: str | Path, explicit_model: str | None = None) -> dict: + model = resolve_vlm_model("RFS_REBUILD_DESIGN_MODEL", "RFS_REBUILD_LAYOUT_MODEL", explicit_model=explicit_model) + prompt = """ +Analyze the complete reference diagram before local object detection. Return JSON only. + +Describe the narrative, reading order, editable layers, asset-source policy, and semantic flow graph. +Do not write PowerPoint code. Do not bake text or connectors into raster assets. +Use normalized bbox_percent coordinates in [0,1]. + +Allowed layer kinds: background, panel, card, visual_slot, text, connector, legend, ignore. +Allowed asset policies: reference_crop, api_generate, placeholder, ppt_shape, editable_text, ppt_connector, ignore. + +Return: +{ + "summary": "...", + "narrative": {"figure_type": "...", "main_story": "...", "key_message": "..."}, + "reading_order": ["stable_object_id"], + "layers": [{"id":"slot_input","kind":"visual_slot","label":"Input","bbox_percent":{"x":0,"y":0,"w":0.1,"h":0.1},"asset_source_policy":"api_generate","asset_source_reason":"...","prompt_subject":"...","asset_type":"generic","semantic_role":"input","confidence":0.0}], + "asset_policies": [{"slot_id":"slot_input","policy":"api_generate","reason":"...","confidence":0.0}], + "flow_graph": {"nodes":[{"id":"slot_input","label":"Input","kind":"visual_slot","confidence":0.0}],"edges":[{"id":"flow_1","source_id":"slot_input","target_id":"slot_output","relation":"flows_to","expected_connector":"solid_arrow","confidence":0.0}]} +} +""".strip() + result = call_vlm_json(prompt, [reference_path], model=model) + result["_vlm_model"] = model + return result + + def _brief_slots(slots: list[dict]) -> list[dict]: return [{ "id": item.get("id"), @@ -164,8 +191,9 @@ def vlm_semantic_adapter(reference_path: str | Path, slots: list[dict], panels: def build_rebuild_vlm_adapters(out_dir: str | Path) -> dict[str, Callable | None]: if not vlm_credentials_available(): - return {"layout": None, "control": None, "semantic": None} + return {"design": None, "layout": None, "control": None, "semantic": None} return { + "design": vlm_design_adapter, "layout": vlm_layout_adapter, "control": vlm_control_adapter_factory(out_dir), "semantic": vlm_semantic_adapter, diff --git a/tests/test_rebuild_editable.py b/tests/test_rebuild_editable.py index f7a142d..c601f14 100644 --- a/tests/test_rebuild_editable.py +++ b/tests/test_rebuild_editable.py @@ -37,6 +37,32 @@ def _fixture(path: Path) -> Path: class RebuildEditableTests(unittest.TestCase): + def test_global_design_plan_artifacts_are_written(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "rebuild" + + result = rebuild_editable( + reference, + out, + asset_mode="placeholder", + text_mode="off", + layout_mode="heuristic", + control_mode="heuristic", + design_plan_mode="heuristic", + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["design_plan_effective_mode"], "heuristic") + for name in [ + "reference_logic_plan.json", + "reference_layer_plan.json", + "reference_generation_plan.json", + "reference_flow_graph.json", + ]: + self.assertTrue((out / name).exists(), name) + def test_cli_placeholder_run_writes_required_outputs(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 444fabc7bf70022b0fbe590852ebed1930359e7a Mon Sep 17 00:00:00 2001 From: illion Date: Sun, 19 Jul 2026 10:48:31 +0800 Subject: [PATCH 03/74] Add raw OCR grouping for rebuild text --- rfs/text_grouping.py | 213 ++++++++++++++++++++++++++++++++++++ tests/test_text_grouping.py | 73 ++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 rfs/text_grouping.py create mode 100644 tests/test_text_grouping.py diff --git a/rfs/text_grouping.py b/rfs/text_grouping.py new file mode 100644 index 0000000..c5dd3f8 --- /dev/null +++ b/rfs/text_grouping.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from statistics import median +from typing import Callable + +from .utils import write_json + + +def _clamp_bbox(bbox: dict) -> dict[str, float]: + x = max(0.0, min(0.995, float(bbox["x"]))) + y = max(0.0, min(0.995, float(bbox["y"]))) + w = max(0.001, min(float(bbox["w"]), 1.0 - x)) + h = max(0.001, min(float(bbox["h"]), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _center(bbox: dict) -> dict[str, float]: + return { + "x": round(float(bbox["x"]) + float(bbox["w"]) / 2, 4), + "y": round(float(bbox["y"]) + float(bbox["h"]) / 2, 4), + } + + +def _union_bbox(regions: list[dict], expand: float = 0.0) -> dict[str, float]: + xs0 = [float(item["bbox_percent"]["x"]) for item in regions] + ys0 = [float(item["bbox_percent"]["y"]) for item in regions] + xs1 = [float(item["bbox_percent"]["x"]) + float(item["bbox_percent"]["w"]) for item in regions] + ys1 = [float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) for item in regions] + x0 = min(xs0) - expand + y0 = min(ys0) - expand + x1 = max(xs1) + expand + y1 = max(ys1) + expand + return _clamp_bbox({"x": x0, "y": y0, "w": x1 - x0, "h": y1 - y0}) + + +def _raw_ratio(region: dict) -> float: + try: + return float(region.get("raw_estimated_font_ratio") or region.get("estimated_font_ratio") or 0.0) + except Exception: + return 0.0 + + +def _font_pt(region: dict) -> float: + try: + return float(region.get("raw_font_size_pt") or region.get("font_size_pt") or 0.0) + except Exception: + return 0.0 + + +def _line_sort_key(region: dict) -> tuple[float, float]: + bbox = region["bbox_percent"] + return float(bbox["y"]), float(bbox["x"]) + + +def _eligible_for_heuristic_grouping(region: dict) -> bool: + if not str(region.get("source") or "").startswith("reference_ocr"): + return False + if str(region.get("role") or "") == "panel_title": + return False + text = str(region.get("text") or "").strip() + return bool(text) + + +def _same_paragraph(previous: dict, current: dict) -> bool: + if str(previous.get("target_id") or "") != str(current.get("target_id") or ""): + return False + if str(previous.get("role") or "") != str(current.get("role") or ""): + return False + if str(previous.get("font_family_guess") or "") != str(current.get("font_family_guess") or ""): + return False + prev = previous["bbox_percent"] + cur = current["bbox_percent"] + prev_h = float(prev["h"]) + cur_h = float(cur["h"]) + median_h = max(0.001, median([prev_h, cur_h])) + vertical_gap = float(cur["y"]) - (float(prev["y"]) + prev_h) + if vertical_gap < -median_h * 0.25 or vertical_gap > max(0.012, median_h * 0.95): + return False + height_delta = abs(prev_h - cur_h) / median_h + if height_delta > 0.35: + return False + left_delta = abs(float(prev["x"]) - float(cur["x"])) + if left_delta > max(0.018, median_h * 1.25): + return False + width_ratio = min(float(prev["w"]), float(cur["w"])) / max(float(prev["w"]), float(cur["w"]), 0.001) + if width_ratio < 0.28: + return False + ratio_values = [value for value in [_raw_ratio(previous), _raw_ratio(current)] if value > 0] + if len(ratio_values) == 2 and abs(ratio_values[0] - ratio_values[1]) / max(median(ratio_values), 0.0001) > 0.35: + return False + return True + + +def _make_group(group_id: str, members: list[dict]) -> dict: + ordered = sorted(members, key=_line_sort_key) + bbox = _union_bbox(ordered, expand=0.0015) + raw_ratios = [_raw_ratio(item) for item in ordered if _raw_ratio(item) > 0] + raw_pts = [_font_pt(item) for item in ordered if _font_pt(item) > 0] + confidences = [float(item.get("confidence") or 0.0) for item in ordered if item.get("confidence") is not None] + estimated_ratio = round(median(raw_ratios), 5) if raw_ratios else round(float(bbox["h"]) * 0.62, 5) + font_pt = round(median(raw_pts), 2) if raw_pts else None + region = dict(ordered[0]) + region.update({ + "id": group_id, + "text": "\n".join(str(item.get("text") or "").strip() for item in ordered if str(item.get("text") or "").strip()), + "raw_text": "\n".join(str(item.get("raw_text") or item.get("text") or "").strip() for item in ordered if str(item.get("raw_text") or item.get("text") or "").strip()), + "bbox_percent": bbox, + "line_bbox_percent": [item["bbox_percent"] for item in ordered], + "word_bbox_percent": [box for item in ordered for box in (item.get("word_bbox_percent") or [item["bbox_percent"]])], + "center_percent": _center(bbox), + "width_percent": round(float(bbox["w"]), 4), + "height_percent": round(float(bbox["h"]), 4), + "estimated_font_ratio": estimated_ratio, + "raw_estimated_font_ratio": estimated_ratio, + "font_size_pt": font_pt or region.get("font_size_pt"), + "raw_font_size_pt": font_pt or region.get("raw_font_size_pt"), + "confidence": round(sum(confidences) / len(confidences), 4) if confidences else region.get("confidence"), + "source": "reference_ocr_paragraph_group_heuristic", + "ocr_member_ids": [str(item.get("id") or "") for item in ordered], + "ocr_member_count": len(ordered), + "grouping_source": "heuristic", + "editable_in": "pptx", + }) + return region + + +def group_text_regions_heuristic(regions: list[dict]) -> tuple[list[dict], dict, dict]: + eligible = [item for item in regions if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and _eligible_for_heuristic_grouping(item)] + ineligible = [item for item in regions if item not in eligible] + grouped: list[list[dict]] = [] + for region in sorted(eligible, key=lambda item: (str(item.get("target_id") or ""), str(item.get("role") or ""), *_line_sort_key(item))): + if grouped and _same_paragraph(grouped[-1][-1], region): + grouped[-1].append(region) + else: + grouped.append([region]) + + result: list[dict] = [] + group_records: list[dict] = [] + for index, members in enumerate(grouped, start=1): + if len(members) == 1: + item = dict(members[0]) + item.setdefault("ocr_member_ids", [str(item.get("id") or "")]) + item.setdefault("ocr_member_count", 1) + item.setdefault("grouping_source", "heuristic_singleton") + result.append(item) + continue + group_id = f"ref_text_group_{index:03d}" + group = _make_group(group_id, members) + result.append(group) + group_records.append({ + "group_id": group_id, + "ocr_member_ids": group["ocr_member_ids"], + "target_id": group.get("target_id"), + "role": group.get("role"), + "bbox_percent": group.get("bbox_percent"), + "text": group.get("text"), + }) + for item in ineligible: + singleton = dict(item) + singleton.setdefault("ocr_member_ids", [str(singleton.get("id") or "")]) + singleton.setdefault("ocr_member_count", 1) + singleton.setdefault("grouping_source", "heuristic_ineligible_singleton") + result.append(singleton) + result.sort(key=lambda item: _line_sort_key(item) if isinstance(item.get("bbox_percent"), dict) else (1.0, 1.0)) + + plan = { + "summary": "Heuristic OCR grouping plan.", + "mode": "heuristic", + "groups": group_records, + "ignored_ocr_ids": [], + } + report = { + "summary": "Text grouping report.", + "mode": "heuristic", + "status": "pass", + "raw_region_count": len(regions), + "grouped_region_count": len(result), + "paragraph_group_count": len(group_records), + "single_region_count": len([item for item in result if int(item.get("ocr_member_count") or 1) == 1]), + "warnings": [], + } + return result, plan, report + + +def group_text_regions( + regions: list[dict], + mode: str = "heuristic", + adapter: Callable | None = None, + **_kwargs, +) -> tuple[list[dict], dict, dict]: + effective_mode = str(mode or "heuristic").lower() + if effective_mode == "off" or not regions: + plan = {"summary": "Text grouping skipped.", "mode": effective_mode, "groups": [], "ignored_ocr_ids": []} + report = { + "summary": "Text grouping report.", + "mode": effective_mode, + "status": "skipped" if effective_mode == "off" else "pass", + "raw_region_count": len(regions), + "grouped_region_count": len(regions), + "paragraph_group_count": 0, + "single_region_count": len(regions), + "warnings": [], + } + return regions, plan, report + if effective_mode != "heuristic": + raise ValueError(f"Unsupported text grouping mode for this build: {mode}") + return group_text_regions_heuristic(regions) + + +def write_text_grouping_artifacts(out_dir, raw_geometry: dict, plan: dict, report: dict) -> None: + write_json(out_dir / "reference_text_geometry_raw.json", raw_geometry) + write_json(out_dir / "text_grouping_plan.json", plan) + write_json(out_dir / "text_grouping_report.json", report) diff --git a/tests/test_text_grouping.py b/tests/test_text_grouping.py new file mode 100644 index 0000000..a78ef06 --- /dev/null +++ b/tests/test_text_grouping.py @@ -0,0 +1,73 @@ +import unittest + +from rfs.text_grouping import group_text_regions_heuristic + + +def _region(text: str, text_id: str, x: float, y: float, w: float, h: float, target_id: str = "card_a", role: str = "body_label") -> dict: + return { + "id": text_id, + "text": text, + "raw_text": text, + "role": role, + "ocr_role_guess": role, + "target_id": target_id, + "bbox_percent": {"x": x, "y": y, "w": w, "h": h}, + "center_percent": {"x": x + w / 2, "y": y + h / 2}, + "width_percent": w, + "height_percent": h, + "estimated_font_ratio": h * 0.9, + "raw_estimated_font_ratio": h * 0.9, + "font_size_pt": 8.0, + "raw_font_size_pt": 8.0, + "font_family_guess": "Arial", + "confidence": 0.95, + "color_hex": "#222222", + "source": "reference_ocr_text_region", + "editable_in": "pptx", + } + + +class TextGroupingTests(unittest.TestCase): + def test_multiline_body_text_is_merged_to_one_paragraph_region(self): + raw = [ + _region("first line of body", "ocr_1", 0.10, 0.20, 0.24, 0.030), + _region("second line continues", "ocr_2", 0.101, 0.234, 0.25, 0.031), + _region("third line continues", "ocr_3", 0.100, 0.269, 0.22, 0.030), + ] + + grouped, plan, report = group_text_regions_heuristic(raw) + + self.assertEqual(report["paragraph_group_count"], 1) + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0]["ocr_member_ids"], ["ocr_1", "ocr_2", "ocr_3"]) + self.assertEqual(grouped[0]["text"], "first line of body\nsecond line continues\nthird line continues") + self.assertLess(grouped[0]["raw_estimated_font_ratio"], grouped[0]["height_percent"]) + self.assertEqual(plan["groups"][0]["group_id"], grouped[0]["id"]) + + def test_panel_titles_are_not_heuristically_paragraph_merged(self): + raw = [ + _region("Graph-based Textual", "ocr_title_1", 0.20, 0.04, 0.20, 0.030, target_id="panel_a", role="panel_title"), + _region("Knowledge Grounding", "ocr_title_2", 0.20, 0.075, 0.22, 0.030, target_id="panel_a", role="panel_title"), + ] + + grouped, _plan, report = group_text_regions_heuristic(raw) + + self.assertEqual(report["paragraph_group_count"], 0) + self.assertEqual(len(grouped), 2) + self.assertTrue(all(item["ocr_member_count"] == 1 for item in grouped)) + + def test_text_from_different_targets_does_not_merge(self): + raw = [ + _region("card A line one", "ocr_a1", 0.10, 0.20, 0.20, 0.030, target_id="card_a"), + _region("card B line one", "ocr_b1", 0.10, 0.234, 0.20, 0.030, target_id="card_b"), + ] + + grouped, _plan, report = group_text_regions_heuristic(raw) + + self.assertEqual(report["paragraph_group_count"], 0) + self.assertEqual(len(grouped), 2) + self.assertEqual({item["target_id"] for item in grouped}, {"card_a", "card_b"}) + + +if __name__ == "__main__": + unittest.main() From 54e8da22d23d58f0d21419609e493a8196072ee5 Mon Sep 17 00:00:00 2001 From: illion Date: Sun, 19 Jul 2026 10:53:13 +0800 Subject: [PATCH 04/74] Add OCR text grouping --- rfs/cli.py | 4 + rfs/editable_rebuild.py | 9 ++ rfs/text_grouping.py | 229 +++++++++++++++++++++++++++++++++++- rfs/text_layer.py | 29 ++++- tests/test_text_grouping.py | 54 ++++++++- 5 files changed, 319 insertions(+), 6 deletions(-) diff --git a/rfs/cli.py b/rfs/cli.py index 4c5b1f2..fd3e172 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -159,6 +159,8 @@ def build_parser() -> argparse.ArgumentParser: rebuild.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") rebuild.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") rebuild.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") + rebuild.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") + rebuild.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") rebuild.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") rebuild.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") rebuild.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") @@ -349,6 +351,8 @@ def main(argv: list[str] | None = None) -> int: design_plan_mode=args.design_plan_mode, design_plan_model=args.design_plan_model, design_adapter=rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, + text_grouping_mode=args.text_grouping_mode, + text_grouping_model=args.text_grouping_model, ) elif args.command == "rebuild-editable-eval": result = evaluate_rebuild_vlm( diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 79757eb..7713f08 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -926,6 +926,9 @@ def rebuild_editable( design_plan_mode: str = "vlm", design_plan_model: str | None = None, design_adapter: Callable | None = None, + text_grouping_mode: str = "heuristic", + text_grouping_model: str | None = None, + text_grouping_adapter: Callable | None = None, ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -947,6 +950,8 @@ def rebuild_editable( "asset_policy": asset_policy, "design_plan_mode": design_plan_mode, "design_plan_model": design_plan_model, + "text_grouping_mode": text_grouping_mode, + "text_grouping_model": text_grouping_model, "skip_analysis": skip_analysis, "compile_only": compile_only, }) @@ -1000,6 +1005,9 @@ def rebuild_editable( ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, + text_grouping_mode=text_grouping_mode, + text_grouping_model=text_grouping_model, + text_grouping_adapter=text_grouping_adapter, ) else: write_json(out_path / "reference_text_geometry.json", {"summary": "Text extraction skipped.", "detection_mode": "off", "regions": []}) @@ -1072,6 +1080,7 @@ def rebuild_editable( "design_plan_mode": design_plan_mode, "design_plan_effective_mode": design_bundle.get("logic", {}).get("effective_mode"), "design_plan_model": design_bundle.get("logic", {}).get("model"), + "text_grouping_mode": text_grouping_mode, "api_requests_attempted": asset_summary.get("api_requests_attempted", 0), "asset_count": len(asset_reports), "slot_count": len(program.get("slots", [])), diff --git a/rfs/text_grouping.py b/rfs/text_grouping.py index c5dd3f8..c823d3e 100644 --- a/rfs/text_grouping.py +++ b/rfs/text_grouping.py @@ -1,8 +1,12 @@ from __future__ import annotations +import json +from pathlib import Path from statistics import median from typing import Callable +from .vlm_client import call_vlm_json, resolve_vlm_model + from .utils import write_json @@ -124,6 +128,176 @@ def _make_group(group_id: str, members: list[dict]) -> dict: return region +def _object_brief(program: dict) -> dict: + def brief(items: list, keys: tuple[str, ...]) -> list[dict]: + result = [] + for item in items or []: + if not isinstance(item, dict): + continue + result.append({key: item.get(key) for key in keys if key in item}) + return result + + return { + "panels": brief(program.get("panels", []), ("id", "title", "bbox_percent")), + "cards": brief(program.get("cards", []), ("id", "title", "bbox_percent")), + "slots": brief(program.get("slots", []), ("id", "paper_concept", "bbox_percent")), + "arrows": brief(program.get("arrows", []), ("id", "source_id", "target_id", "source", "target", "path_percent")), + } + + +def _raw_ocr_brief(regions: list[dict]) -> list[dict]: + return [{ + "id": item.get("id"), + "text": item.get("text"), + "role_guess": item.get("role"), + "target_id": item.get("target_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "confidence": item.get("confidence"), + } for item in regions if isinstance(item, dict)] + + +def _heuristic_group_brief(regions: list[dict]) -> list[dict]: + return [{ + "id": item.get("id"), + "text": item.get("text"), + "ocr_member_ids": item.get("ocr_member_ids"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "bbox_percent": item.get("bbox_percent"), + } for item in regions if isinstance(item, dict)] + + +def _call_vlm_grouping( + reference_path: str | Path, + raw_regions: list[dict], + heuristic_regions: list[dict], + program: dict, + model: str | None = None, +) -> dict: + model_name = resolve_vlm_model("RFS_TEXT_GROUPING_MODEL", "RFS_TEXT_ROLE_MODEL", explicit_model=model) + prompt = f""" +You are grouping OCR text lines for an editable PowerPoint reconstruction. +Only output JSON. Do not output markdown, prose, Python, SVG, or PPT code. + +Task: +- Decide which raw OCR line ids belong to the same visible text paragraph/label. +- Decide whether a raw OCR line should be ignored because it is decorative, unreadable noise, or belongs only inside a raster asset. +- You may assign role and align, but you may NOT invent coordinates. +- Bboxes will be computed by code as the union of raw OCR boxes. +- Do not change text content. + +Return schema: +{{ + "summary": "...", + "groups": [ + {{"group_id":"group_1","ocr_member_ids":["ref_text_ocr_001"],"role":"panel_title|section_title|body_label|slot_caption|free_text|annotation|arrow_label|legend_label|method_label|modality_label|trait_label","align":"left|center|right","ignore":false,"confidence":0.0,"reason":"..."}} + ], + "ignored_ocr_ids": ["ref_text_ocr_999"], + "warnings": [] +}} + +Raw OCR regions: +{json.dumps(_raw_ocr_brief(raw_regions), ensure_ascii=False)} + +Heuristic groups: +{json.dumps(_heuristic_group_brief(heuristic_regions), ensure_ascii=False)} + +Objects: +{json.dumps(_object_brief(program), ensure_ascii=False)} +""".strip() + result = call_vlm_json(prompt, [reference_path], model=model_name) + result.setdefault("_vlm_model", model_name) + return result + + +def _coerce_align(value: object) -> str | None: + text = str(value or "").strip().lower() + if text in {"left", "center", "right"}: + return text + return None + + +def _apply_grouping_plan(raw_regions: list[dict], raw_plan: dict) -> tuple[list[dict], dict, list[str]]: + raw_by_id = {str(item.get("id") or ""): item for item in raw_regions} + used: set[str] = set() + ignored = {str(item) for item in raw_plan.get("ignored_ocr_ids", []) if str(item).strip()} + warnings: list[str] = [] + planned_regions: list[dict] = [] + normalized_groups = [] + for index, item in enumerate(raw_plan.get("groups", []) if isinstance(raw_plan.get("groups"), list) else [], start=1): + if not isinstance(item, dict): + continue + member_ids = [str(value) for value in item.get("ocr_member_ids", []) if str(value) in raw_by_id] + member_ids = [value for value in member_ids if value not in ignored and value not in used] + if not member_ids: + continue + if bool(item.get("ignore")): + ignored.update(member_ids) + used.update(member_ids) + continue + members = [raw_by_id[value] for value in member_ids] + if len(members) == 1: + region = dict(members[0]) + region.setdefault("ocr_member_ids", member_ids) + region.setdefault("ocr_member_count", 1) + region["grouping_source"] = "vlm_singleton" + else: + region = _make_group(str(item.get("group_id") or f"ref_text_vlm_group_{index:03d}"), members) + region["source"] = "reference_ocr_paragraph_group_vlm" + region["grouping_source"] = "vlm" + if str(item.get("role") or "").strip(): + region["role"] = str(item.get("role")) + region["ocr_role_guess"] = region.get("ocr_role_guess") or members[0].get("role") + align = _coerce_align(item.get("align")) + if align: + region["align"] = align + region["grouping_confidence"] = item.get("confidence") + region["grouping_reason"] = item.get("reason") + planned_regions.append(region) + used.update(member_ids) + normalized_groups.append({ + "group_id": region["id"], + "ocr_member_ids": member_ids, + "role": region.get("role"), + "align": region.get("align"), + "bbox_percent": region.get("bbox_percent"), + "confidence": item.get("confidence"), + "reason": item.get("reason"), + }) + + for raw in raw_regions: + rid = str(raw.get("id") or "") + if rid in used or rid in ignored: + continue + singleton = dict(raw) + singleton.setdefault("ocr_member_ids", [rid]) + singleton.setdefault("ocr_member_count", 1) + singleton.setdefault("grouping_source", "vlm_unmentioned_singleton") + planned_regions.append(singleton) + normalized_groups.append({ + "group_id": singleton["id"], + "ocr_member_ids": [rid], + "role": singleton.get("role"), + "align": singleton.get("align"), + "bbox_percent": singleton.get("bbox_percent"), + "confidence": None, + "reason": "raw OCR id was not mentioned by VLM plan", + }) + if not planned_regions and raw_regions: + warnings.append("vlm_grouping_plan_produced_no_regions") + planned_regions.sort(key=lambda item: _line_sort_key(item) if isinstance(item.get("bbox_percent"), dict) else (1.0, 1.0)) + plan = { + "summary": str(raw_plan.get("summary") or "VLM text grouping plan."), + "mode": "vlm", + "vlm_model": raw_plan.get("_vlm_model"), + "groups": normalized_groups, + "ignored_ocr_ids": sorted(ignored), + "warnings": list(raw_plan.get("warnings", [])) + warnings if isinstance(raw_plan.get("warnings", []), list) else warnings, + } + return planned_regions, plan, warnings + + def group_text_regions_heuristic(regions: list[dict]) -> tuple[list[dict], dict, dict]: eligible = [item for item in regions if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and _eligible_for_heuristic_grouping(item)] ineligible = [item for item in regions if item not in eligible] @@ -186,7 +360,9 @@ def group_text_regions( regions: list[dict], mode: str = "heuristic", adapter: Callable | None = None, - **_kwargs, + reference_path: str | Path | None = None, + program: dict | None = None, + model: str | None = None, ) -> tuple[list[dict], dict, dict]: effective_mode = str(mode or "heuristic").lower() if effective_mode == "off" or not regions: @@ -202,9 +378,54 @@ def group_text_regions( "warnings": [], } return regions, plan, report - if effective_mode != "heuristic": - raise ValueError(f"Unsupported text grouping mode for this build: {mode}") - return group_text_regions_heuristic(regions) + heuristic_regions, heuristic_plan, heuristic_report = group_text_regions_heuristic(regions) + if effective_mode == "heuristic": + return heuristic_regions, heuristic_plan, heuristic_report + if effective_mode not in {"vlm", "hybrid"}: + raise ValueError(f"Unsupported text grouping mode: {mode}") + + try: + if adapter: + raw_plan = adapter(reference_path, regions, heuristic_regions, program or {}, model) + else: + if reference_path is None: + raise RuntimeError("reference_path is required for VLM text grouping") + raw_plan = _call_vlm_grouping(reference_path, regions, heuristic_regions, program or {}, model=model) + vlm_regions, plan, warnings = _apply_grouping_plan(regions, raw_plan if isinstance(raw_plan, dict) else {}) + if not vlm_regions and regions: + raise RuntimeError("VLM text grouping produced no usable regions") + report = { + "summary": "Text grouping report.", + "mode": effective_mode, + "effective_mode": "vlm", + "status": "pass", + "raw_region_count": len(regions), + "grouped_region_count": len(vlm_regions), + "paragraph_group_count": len([item for item in vlm_regions if int(item.get("ocr_member_count") or 1) > 1]), + "single_region_count": len([item for item in vlm_regions if int(item.get("ocr_member_count") or 1) == 1]), + "warnings": warnings, + "heuristic_group_count": heuristic_report.get("paragraph_group_count", 0), + "vlm_model": plan.get("vlm_model"), + } + plan["mode"] = effective_mode + return vlm_regions, plan, report + except Exception as exc: + if effective_mode == "vlm": + fallback_status = "fallback_to_heuristic" + else: + fallback_status = "fallback_to_heuristic" + heuristic_plan = dict(heuristic_plan) + heuristic_plan["mode"] = effective_mode + heuristic_plan["effective_mode"] = "heuristic" + heuristic_plan.setdefault("warnings", []) + heuristic_plan["warnings"].append(f"vlm_grouping_failed:{exc}") + heuristic_report = dict(heuristic_report) + heuristic_report["mode"] = effective_mode + heuristic_report["effective_mode"] = "heuristic" + heuristic_report["status"] = fallback_status + heuristic_report.setdefault("warnings", []) + heuristic_report["warnings"].append(f"vlm_grouping_failed:{exc}") + return heuristic_regions, heuristic_plan, heuristic_report def write_text_grouping_artifacts(out_dir, raw_geometry: dict, plan: dict, report: dict) -> None: diff --git a/rfs/text_layer.py b/rfs/text_layer.py index d642238..0ace81b 100644 --- a/rfs/text_layer.py +++ b/rfs/text_layer.py @@ -7,6 +7,7 @@ from PIL import Image from .reference_text_extractor import extract_reference_text +from .text_grouping import group_text_regions, write_text_grouping_artifacts from .utils import write_json @@ -336,6 +337,9 @@ def build_text_layer( ocr_engine: str = "paddle", ocr_lang: str = "en_ch", ocr_adapter: Callable[[str | Path, str], list[dict]] | None = None, + text_grouping_mode: str = "heuristic", + text_grouping_model: str | None = None, + text_grouping_adapter: Callable | None = None, ) -> dict: """Build reference-first editable text artifacts and attach them to the program.""" @@ -352,7 +356,25 @@ def build_text_layer( lang=ocr_lang, ocr_adapter=ocr_adapter, ) - regions = _merge_ocr_with_fallback(ocr_regions, fallback_regions) + raw_geometry = { + "summary": "Raw line-level OCR geometry before paragraph grouping.", + "reference_path": str(reference_path), + "detection_mode": "ocr" if ocr_regions else "ocr_empty_or_unavailable", + "ocr_engine": ocr_engine, + "ocr_lang": ocr_lang, + "text_region_count": len(ocr_regions), + "text_regions": ocr_regions, + } + grouped_ocr_regions, grouping_plan, grouping_report = group_text_regions( + ocr_regions, + mode=text_grouping_mode if ocr_regions else "off", + adapter=text_grouping_adapter, + reference_path=reference_path, + program=program, + model=text_grouping_model, + ) + write_text_grouping_artifacts(out, raw_geometry, grouping_plan, grouping_report) + regions = _merge_ocr_with_fallback(grouped_ocr_regions, fallback_regions) if not ocr_regions: ocr_report["text_region_count"] = len(regions) ocr_report.setdefault("warnings", []) @@ -395,6 +417,11 @@ def build_text_layer( "detection_mode": "ocr" if ocr_regions else "reference_geometry_and_local_color_sampling", "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, + "reference_text_geometry_raw_path": "reference_text_geometry_raw.json", + "text_grouping_plan_path": "text_grouping_plan.json", + "text_grouping_report_path": "text_grouping_report.json", + "text_grouping_mode": text_grouping_mode, + "text_grouping_status": grouping_report.get("status"), "text_regions": regions, } text_program = { diff --git a/tests/test_text_grouping.py b/tests/test_text_grouping.py index a78ef06..e3536f5 100644 --- a/tests/test_text_grouping.py +++ b/tests/test_text_grouping.py @@ -1,6 +1,6 @@ import unittest -from rfs.text_grouping import group_text_regions_heuristic +from rfs.text_grouping import group_text_regions, group_text_regions_heuristic def _region(text: str, text_id: str, x: float, y: float, w: float, h: float, target_id: str = "card_a", role: str = "body_label") -> dict: @@ -68,6 +68,58 @@ def test_text_from_different_targets_does_not_merge(self): self.assertEqual(len(grouped), 2) self.assertEqual({item["target_id"] for item in grouped}, {"card_a", "card_b"}) + def test_vlm_grouping_uses_raw_ocr_union_not_vlm_bbox(self): + raw = [ + _region("line one", "ocr_1", 0.10, 0.20, 0.20, 0.030), + _region("line two", "ocr_2", 0.101, 0.234, 0.22, 0.030), + _region("noise", "ocr_noise", 0.70, 0.80, 0.04, 0.012, role="free_text"), + ] + + def fake_adapter(_reference, _raw_regions, _heuristic_regions, _program, _model): + return { + "summary": "fake grouping", + "groups": [{ + "group_id": "body_para", + "ocr_member_ids": ["ocr_1", "ocr_2"], + "role": "annotation", + "align": "left", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + "confidence": 0.93, + "reason": "two visible body lines", + }], + "ignored_ocr_ids": ["ocr_noise"], + } + + grouped, plan, report = group_text_regions(raw, mode="vlm", adapter=fake_adapter, reference_path="reference.png", program={}) + + self.assertEqual(report["status"], "pass") + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0]["id"], "body_para") + self.assertEqual(grouped[0]["role"], "annotation") + self.assertEqual(grouped[0]["align"], "left") + self.assertLess(grouped[0]["bbox_percent"]["x"], 0.105) + self.assertGreater(grouped[0]["bbox_percent"]["x"], 0.09) + self.assertLess(grouped[0]["bbox_percent"]["w"], 0.23) + self.assertEqual(plan["ignored_ocr_ids"], ["ocr_noise"]) + + def test_hybrid_grouping_falls_back_to_heuristic_when_vlm_fails(self): + raw = [ + _region("first line", "ocr_1", 0.10, 0.20, 0.20, 0.030), + _region("second line", "ocr_2", 0.10, 0.234, 0.20, 0.030), + ] + + def failing_adapter(*_args): + raise RuntimeError("grouping unavailable") + + grouped, plan, report = group_text_regions(raw, mode="hybrid", adapter=failing_adapter, reference_path="reference.png", program={}) + + self.assertEqual(report["status"], "fallback_to_heuristic") + self.assertEqual(report["effective_mode"], "heuristic") + self.assertEqual(len(grouped), 1) + self.assertEqual(grouped[0]["ocr_member_ids"], ["ocr_1", "ocr_2"]) + self.assertIn("grouping unavailable", report["warnings"][0]) + self.assertEqual(plan["effective_mode"], "heuristic") + if __name__ == "__main__": unittest.main() From 322b0b2231f218dd2e2fd8638b445562ebad6f6e Mon Sep 17 00:00:00 2001 From: illion Date: Sun, 19 Jul 2026 10:58:13 +0800 Subject: [PATCH 05/74] Add text layer ownership --- rfs/text_layer.py | 8 ++ rfs/text_layer_ownership.py | 149 +++++++++++++++++++++++++++++++++++ tests/test_text_layer_ocr.py | 17 ++++ 3 files changed, 174 insertions(+) create mode 100644 rfs/text_layer_ownership.py diff --git a/rfs/text_layer.py b/rfs/text_layer.py index 0ace81b..8c99d70 100644 --- a/rfs/text_layer.py +++ b/rfs/text_layer.py @@ -8,6 +8,7 @@ from .reference_text_extractor import extract_reference_text from .text_grouping import group_text_regions, write_text_grouping_artifacts +from .text_layer_ownership import apply_text_layer_ownership, write_text_layer_ownership_artifacts from .utils import write_json @@ -375,6 +376,8 @@ def build_text_layer( ) write_text_grouping_artifacts(out, raw_geometry, grouping_plan, grouping_report) regions = _merge_ocr_with_fallback(grouped_ocr_regions, fallback_regions) + regions, ownership_plan, ownership_report = apply_text_layer_ownership(regions, program, mode="heuristic") + write_text_layer_ownership_artifacts(out, ownership_plan, ownership_report) if not ocr_regions: ocr_report["text_region_count"] = len(regions) ocr_report.setdefault("warnings", []) @@ -382,6 +385,8 @@ def build_text_layer( items: list[dict] = [] for region in regions: + if region.get("layer_ownership") != "editable_text_layer": + continue font_size = float(region.get("font_size_pt") or _font_pt_from_reference_region(region, canvas_height_in)) token_id = _nearest_token_id(style, region["color_hex"]) item = { @@ -407,6 +412,7 @@ def build_text_layer( "ocr_confidence": region.get("confidence"), "editable_in": "pptx", "visible": True, + "layer_ownership": region.get("layer_ownership"), } items.append(item) @@ -422,6 +428,8 @@ def build_text_layer( "text_grouping_report_path": "text_grouping_report.json", "text_grouping_mode": text_grouping_mode, "text_grouping_status": grouping_report.get("status"), + "text_layer_ownership_plan_path": "text_layer_ownership_plan.json", + "text_layer_ownership_report_path": "text_layer_ownership_report.json", "text_regions": regions, } text_program = { diff --git a/rfs/text_layer_ownership.py b/rfs/text_layer_ownership.py new file mode 100644 index 0000000..c213ac0 --- /dev/null +++ b/rfs/text_layer_ownership.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from typing import Callable + +from .utils import write_json + + +OWNERSHIP_VALUES = {"editable_text_layer", "raster_asset_layer", "decorative_asset_text", "ignore"} +CRITICAL_EDITABLE_ROLES = { + "panel_title", + "section_title", + "arrow_label", + "legend_label", + "method_label", + "modality_label", + "trait_label", + "slot_caption", + "title", + "subtitle", +} + + +def _contains_center(container: dict, item: dict) -> bool: + box = item.get("bbox_percent") + cbox = container.get("bbox_percent") + if not isinstance(box, dict) or not isinstance(cbox, dict): + return False + cx = float(box["x"]) + float(box["w"]) / 2 + cy = float(box["y"]) + float(box["h"]) / 2 + return ( + float(cbox["x"]) <= cx <= float(cbox["x"]) + float(cbox["w"]) + and float(cbox["y"]) <= cy <= float(cbox["y"]) + float(cbox["h"]) + ) + + +def _slot_ids_containing_text(region: dict, program: dict) -> list[str]: + return [ + str(slot.get("id") or "") + for slot in program.get("slots", []) or [] + if isinstance(slot, dict) and str(slot.get("id") or "").strip() and _contains_center(slot, region) + ] + + +def _font_size(region: dict) -> float: + try: + return float(region.get("font_size_pt") or region.get("raw_font_size_pt") or 0.0) + except Exception: + return 0.0 + + +def _decide_heuristic(region: dict, program: dict) -> tuple[str, str]: + text = str(region.get("text") or "").strip() + role = str(region.get("role") or "") + if not text: + return "ignore", "empty text" + if _font_size(region) < 4.0 or len(text) <= 1: + return "decorative_asset_text", "tiny or one-character OCR text is treated as decorative asset text" + if role in CRITICAL_EDITABLE_ROLES: + return "editable_text_layer", f"{role} is a critical editable text role" + containing_slots = _slot_ids_containing_text(region, program) + target_id = str(region.get("target_id") or "") + if target_id in containing_slots or (containing_slots and role in {"free_text", "body_label", "annotation"}): + return "raster_asset_layer", "OCR text is visually inside a slot asset and is not a critical label" + if role in {"body_label", "annotation", "free_text"}: + return "editable_text_layer", f"{role} remains editable because it is not owned by a slot asset" + return "editable_text_layer", "default editable text ownership" + + +def apply_text_layer_ownership( + regions: list[dict], + program: dict, + mode: str = "heuristic", + adapter: Callable | None = None, +) -> tuple[list[dict], dict, dict]: + effective_mode = str(mode or "heuristic").lower() + raw_overrides = {} + warnings = [] + if effective_mode in {"vlm", "hybrid"} and adapter: + try: + raw = adapter(regions, program) + records = raw.get("items", []) if isinstance(raw, dict) else raw + for item in records or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or item.get("id") or "") + ownership = str(item.get("layer_ownership") or item.get("ownership") or "") + if text_id and ownership in OWNERSHIP_VALUES: + raw_overrides[text_id] = item + except Exception as exc: + warnings.append(f"ownership_adapter_failed:{exc}") + effective_mode = "heuristic" + elif effective_mode in {"vlm", "hybrid"}: + warnings.append("ownership_vlm_unavailable_fallback_to_heuristic") + effective_mode = "heuristic" + + planned_regions = [] + items = [] + for region in regions: + region = dict(region) + ownership, reason = _decide_heuristic(region, program) + override = raw_overrides.get(str(region.get("id") or "")) + if override: + ownership = str(override.get("layer_ownership") or override.get("ownership") or ownership) + reason = str(override.get("reason") or reason) + if ownership not in OWNERSHIP_VALUES: + warnings.append(f"invalid_ownership:{region.get('id')}:{ownership}") + ownership = "editable_text_layer" + reason = "invalid ownership value fell back to editable text" + region["layer_ownership"] = ownership + region["layer_ownership_source"] = "vlm" if override else "heuristic" + region["layer_ownership_reason"] = reason + planned_regions.append(region) + items.append({ + "text_id": region.get("id"), + "text": region.get("text"), + "role": region.get("role"), + "target_id": region.get("target_id"), + "layer_ownership": ownership, + "source": region["layer_ownership_source"], + "reason": reason, + "included_in_text_program": ownership == "editable_text_layer", + }) + + plan = { + "summary": "Text layer ownership plan.", + "mode": mode, + "effective_mode": effective_mode, + "items": items, + "warnings": warnings, + } + report = { + "summary": "Text layer ownership report.", + "mode": mode, + "effective_mode": effective_mode, + "status": "pass" if not warnings else "warning", + "text_region_count": len(planned_regions), + "editable_text_count": sum(1 for item in items if item["layer_ownership"] == "editable_text_layer"), + "raster_asset_text_count": sum(1 for item in items if item["layer_ownership"] == "raster_asset_layer"), + "decorative_asset_text_count": sum(1 for item in items if item["layer_ownership"] == "decorative_asset_text"), + "ignored_text_count": sum(1 for item in items if item["layer_ownership"] == "ignore"), + "warnings": warnings, + "items": items, + } + return planned_regions, plan, report + + +def write_text_layer_ownership_artifacts(out_dir, plan: dict, report: dict) -> None: + write_json(out_dir / "text_layer_ownership_plan.json", plan) + write_json(out_dir / "text_layer_ownership_report.json", report) diff --git a/tests/test_text_layer_ocr.py b/tests/test_text_layer_ocr.py index 1d74238..7ab2a01 100644 --- a/tests/test_text_layer_ocr.py +++ b/tests/test_text_layer_ocr.py @@ -8,6 +8,7 @@ from rfs.ppt_compiler import compile_ppt from rfs.text_layer import build_text_layer +from rfs.text_layer_ownership import apply_text_layer_ownership def _base_program() -> dict: @@ -40,6 +41,22 @@ def _style() -> dict: class TextLayerOcrTests(unittest.TestCase): + def test_text_ownership_keeps_critical_labels_editable(self): + program = { + "slots": [{"id": "slot_a", "bbox_percent": {"x": 0.1, "y": 0.1, "w": 0.5, "h": 0.5}}] + } + regions = [ + {"id": "title", "text": "Method", "role": "panel_title", "target_id": "slot_a", "font_size_pt": 12, "bbox_percent": {"x": 0.2, "y": 0.2, "w": 0.2, "h": 0.1}}, + {"id": "decorative", "text": "x", "role": "free_text", "target_id": "slot_a", "font_size_pt": 3, "bbox_percent": {"x": 0.3, "y": 0.3, "w": 0.05, "h": 0.05}}, + ] + + planned, _plan, report = apply_text_layer_ownership(regions, program) + + by_id = {item["id"]: item for item in planned} + self.assertEqual(by_id["title"]["layer_ownership"], "editable_text_layer") + self.assertEqual(by_id["decorative"]["layer_ownership"], "decorative_asset_text") + self.assertEqual(report["editable_text_count"], 1) + def test_fake_ocr_creates_reference_text_geometry_without_duplicate_panel_title(self): with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) From 7657a2c932558e551873a57d83931e1e0477cf2b Mon Sep 17 00:00:00 2001 From: illion Date: Sun, 19 Jul 2026 11:01:09 +0800 Subject: [PATCH 06/74] Add deterministic rebuild visual quality report --- rfs/editable_rebuild.py | 10 ++ rfs/rebuild_visual_critic.py | 186 ++++++++++++++++++++++++++++ tests/test_rebuild_visual_critic.py | 80 ++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 rfs/rebuild_visual_critic.py create mode 100644 tests/test_rebuild_visual_critic.py diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 7713f08..042c1f2 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -18,6 +18,7 @@ from .ppt_compiler import compile_ppt from .rebuild_design_planner import plan_rebuild_design from .rebuild_vlm_validation import build_rebuild_vlm_validation_report +from .rebuild_visual_critic import run_rebuild_visual_quality_check from .text_layer import build_text_layer from .utils import ensure_dir, write_json, write_text @@ -1046,6 +1047,13 @@ def rebuild_editable( regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration, asset_policy=asset_policy) vlm_validation_report = build_rebuild_vlm_validation_report(out_path, reference_geometry, reference_controls, semantic_report, asset_summary) + visual_quality_report = run_rebuild_visual_quality_check( + out_path, + program, + reference_geometry=reference_geometry, + reference_controls=reference_controls, + ownership_report=_load_json_or_empty(out_path / "text_layer_ownership_report.json"), + ) write_json(out_path / "figure_program.json", program) pptx_path = compile_ppt(program, out_path) @@ -1089,6 +1097,7 @@ def rebuild_editable( "text_mode": text_mode, "control_mode": control_mode, "layout_mode": layout_mode, + "rebuild_visual_quality_status": visual_quality_report.get("status"), "reports": { "input_manifest": str(out_path / "input_manifest.json"), "reference_logic_plan": str(out_path / "reference_logic_plan.json"), @@ -1101,6 +1110,7 @@ def rebuild_editable( "slot_inventory": str(out_path / "slot_inventory.json"), "slot_semantic_report": str(out_path / "slot_semantic_report.json"), "rebuild_vlm_validation_report": str(out_path / "rebuild_vlm_validation_report.json"), + "rebuild_visual_quality_report": str(out_path / "rebuild_visual_quality_report.json"), "asset_generation_specs": str(out_path / "asset_generation_specs.json"), "asset_generation_report": str(out_path / "asset_generation_report.json"), "asset_economy_report": str(out_path / "asset_economy_report.json"), diff --git a/rfs/rebuild_visual_critic.py b/rfs/rebuild_visual_critic.py new file mode 100644 index 0000000..d8a402d --- /dev/null +++ b/rfs/rebuild_visual_critic.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .utils import write_json + + +def _bbox(item: dict) -> dict | None: + box = item.get("bbox_percent") + return box if isinstance(box, dict) else None + + +def _valid_bbox(box: dict | None) -> bool: + if not isinstance(box, dict): + return False + try: + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + except Exception: + return False + return 0 <= x <= 1 and 0 <= y <= 1 and w > 0 and h > 0 and x + w <= 1.0001 and y + h <= 1.0001 + + +def _overlap_area(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + return ix * iy + + +def _overlap_ratio(a: dict, b: dict) -> float: + area = _overlap_area(a, b) + if area <= 0: + return 0.0 + a_area = float(a["w"]) * float(a["h"]) + b_area = float(b["w"]) * float(b["h"]) + return area / max(min(a_area, b_area), 0.000001) + + +def _visible_text_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + return [item for item in text_program.get("items", []) if isinstance(item, dict) and item.get("visible", True)] + + +def _detect_text_issues(program: dict) -> list[dict]: + issues = [] + items = _visible_text_items(program) + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": "text_bbox_out_of_bounds", "text_id": item.get("id"), "reason": "text bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > 0.18: + issues.append({ + "type": "text_overlap", + "text_id": left.get("id"), + "other_text_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": "visible text boxes overlap substantially", + }) + grouped: dict[tuple[str, str], list[dict]] = {} + for item in items: + key = (str(item.get("target_id") or ""), str(item.get("role") or "")) + grouped.setdefault(key, []).append(item) + for (target_id, role), group in grouped.items(): + if len(group) < 3: + continue + centers = [ + float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2 + for item in group + if _valid_bbox(_bbox(item)) + ] + if len(centers) >= 3 and max(centers) - min(centers) > 0.035: + issues.append({ + "type": "text_group_misaligned", + "target_id": target_id, + "role": role, + "text_ids": [item.get("id") for item in group], + "center_y_span": round(max(centers) - min(centers), 4), + "reason": "same target/role text group is visually uneven", + }) + return issues + + +def _detect_object_issues(program: dict) -> list[dict]: + issues = [] + for collection_name in ("panels", "slots"): + items = [item for item in program.get(collection_name, []) if isinstance(item, dict)] + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": f"{collection_name[:-1]}_bbox_out_of_bounds", "id": item.get("id"), "reason": "bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > (0.35 if collection_name == "slots" else 0.18): + issues.append({ + "type": f"{collection_name[:-1]}_overlap", + "id": left.get("id"), + "other_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": f"{collection_name[:-1]} boxes overlap substantially", + }) + return issues + + +def _detect_arrow_issues(program: dict) -> list[dict]: + issues = [] + for arrow in program.get("arrows", []) or []: + if not isinstance(arrow, dict): + continue + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + issues.append({"type": "arrow_missing_path", "arrow_id": arrow.get("id"), "reason": "arrow has no usable path_percent"}) + return issues + + +def _detect_ownership_issues(program: dict, ownership_report: dict | None) -> list[dict]: + issues = [] + text_ids = {str(item.get("source_reference_text_id") or item.get("id") or "") for item in _visible_text_items(program)} + report = ownership_report if isinstance(ownership_report, dict) else {} + for item in report.get("items", []) or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or "") + ownership = str(item.get("layer_ownership") or "") + included = bool(item.get("included_in_text_program")) + if ownership != "editable_text_layer" and text_id in text_ids: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "non-editable-owned text is present in text_program"}) + if ownership == "editable_text_layer" and not included: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "editable-owned text is missing from text_program"}) + return issues + + +def run_rebuild_visual_quality_check( + out_dir: str | Path, + program: dict, + reference_geometry: dict | None = None, + reference_controls: dict | None = None, + ownership_report: dict | None = None, + mode: str = "heuristic", +) -> dict[str, Any]: + text_issues = _detect_text_issues(program) + object_issues = _detect_object_issues(program) + arrow_issues = _detect_arrow_issues(program) + ownership_issues = _detect_ownership_issues(program, ownership_report) + issues = text_issues + object_issues + arrow_issues + ownership_issues + blocking_types = {"text_overlap", "text_bbox_out_of_bounds", "arrow_missing_path", "text_layer_ownership_conflict"} + blocking = [item for item in issues if item.get("type") in blocking_types] + report = { + "summary": "Deterministic rebuild visual quality report.", + "mode": mode, + "status": "blocked" if blocking else ("warning" if issues else "pass"), + "issue_count": len(issues), + "blocking_issue_count": len(blocking), + "text_issue_count": len(text_issues), + "object_issue_count": len(object_issues), + "arrow_issue_count": len(arrow_issues), + "ownership_issue_count": len(ownership_issues), + "issues": issues, + "reference_geometry_status": (reference_geometry or {}).get("status"), + "reference_controls_status": (reference_controls or {}).get("status"), + "policy": "deterministic check only; no program mutation", + } + write_json(Path(out_dir) / "rebuild_visual_quality_report.json", report) + return report diff --git a/tests/test_rebuild_visual_critic.py b/tests/test_rebuild_visual_critic.py new file mode 100644 index 0000000..b931729 --- /dev/null +++ b/tests/test_rebuild_visual_critic.py @@ -0,0 +1,80 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.rebuild_visual_critic import run_rebuild_visual_quality_check + + +class RebuildVisualCriticTests(unittest.TestCase): + def test_deterministic_report_flags_text_overlap_bounds_and_missing_arrow_path(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + program = { + "text_program": { + "items": [ + { + "id": "text_a", + "source_reference_text_id": "ref_a", + "text": "A", + "bbox_percent": {"x": 0.10, "y": 0.10, "w": 0.20, "h": 0.05}, + }, + { + "id": "text_b", + "source_reference_text_id": "ref_b", + "text": "B", + "bbox_percent": {"x": 0.12, "y": 0.11, "w": 0.20, "h": 0.05}, + }, + { + "id": "text_bad", + "source_reference_text_id": "ref_bad", + "text": "bad", + "bbox_percent": {"x": 0.98, "y": 0.10, "w": 0.10, "h": 0.05}, + }, + ] + }, + "panels": [], + "slots": [], + "arrows": [{"id": "arrow_missing", "source_id": "a", "target_id": "b"}], + } + + report = run_rebuild_visual_quality_check(out, program) + + self.assertEqual(report["status"], "blocked") + issue_types = {item["type"] for item in report["issues"]} + self.assertIn("text_overlap", issue_types) + self.assertIn("text_bbox_out_of_bounds", issue_types) + self.assertIn("arrow_missing_path", issue_types) + self.assertTrue((out / "rebuild_visual_quality_report.json").exists()) + + def test_ownership_conflict_is_reported(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + program = { + "text_program": { + "items": [{ + "id": "text_a", + "source_reference_text_id": "ref_a", + "text": "A", + "bbox_percent": {"x": 0.10, "y": 0.10, "w": 0.20, "h": 0.05}, + }] + }, + "panels": [], + "slots": [], + "arrows": [], + } + ownership = { + "items": [{ + "text_id": "ref_a", + "layer_ownership": "raster_asset_layer", + "included_in_text_program": False, + }] + } + + report = run_rebuild_visual_quality_check(out, program, ownership_report=ownership) + + self.assertEqual(report["ownership_issue_count"], 1) + self.assertEqual(report["issues"][0]["type"], "text_layer_ownership_conflict") + + +if __name__ == "__main__": + unittest.main() From a7bfac6a389c86a8b640d6c24ed09d9d9f98dbc4 Mon Sep 17 00:00:00 2001 From: illion Date: Sun, 19 Jul 2026 11:34:11 +0800 Subject: [PATCH 07/74] Add fallback rebuild preview renderer --- rfs/editable_rebuild.py | 9 +- rfs/rebuild_preview_renderer.py | 156 ++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 rfs/rebuild_preview_renderer.py diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 042c1f2..4a05dc5 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -17,10 +17,11 @@ from .layout_semantic_planner import plan_slot_semantics from .ppt_compiler import compile_ppt from .rebuild_design_planner import plan_rebuild_design +from .rebuild_preview_renderer import render_rebuild_preview from .rebuild_vlm_validation import build_rebuild_vlm_validation_report from .rebuild_visual_critic import run_rebuild_visual_quality_check from .text_layer import build_text_layer -from .utils import ensure_dir, write_json, write_text +from .utils import ensure_dir, read_json, write_json, write_text ASSET_THRESHOLDS = { @@ -781,6 +782,9 @@ def _export_preview(pptx_path: Path, out: Path) -> Path | None: preview = out / "rebuild_preview.png" if platform.system() != "Windows": write_text(out / "preview_export_error.txt", "PowerPoint preview export is only available on Windows with Microsoft PowerPoint installed.") + program_path = out / "figure_program.json" + if program_path.exists(): + return render_rebuild_preview(read_json(program_path), out, preview) return None try: import win32com.client @@ -793,6 +797,9 @@ def _export_preview(pptx_path: Path, out: Path) -> Path | None: return preview except Exception as exc: write_text(out / "preview_export_error.txt", str(exc)) + program_path = out / "figure_program.json" + if program_path.exists(): + return render_rebuild_preview(read_json(program_path), out, preview) return None diff --git a/rfs/rebuild_preview_renderer.py b/rfs/rebuild_preview_renderer.py new file mode 100644 index 0000000..d166397 --- /dev/null +++ b/rfs/rebuild_preview_renderer.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image, ImageColor, ImageDraw, ImageFont + + +def _hex_color(value: object, fallback: str = "#000000") -> str: + if not isinstance(value, str): + return fallback + text = value.strip() + if not text: + return fallback + if not text.startswith("#"): + text = "#" + text + try: + ImageColor.getrgb(text) + return text + except ValueError: + return fallback + + +def _bbox_px(bbox: dict, width: int, height: int) -> tuple[int, int, int, int]: + x = int(round(float(bbox.get("x", 0)) * width)) + y = int(round(float(bbox.get("y", 0)) * height)) + w = int(round(float(bbox.get("w", 0)) * width)) + h = int(round(float(bbox.get("h", 0)) * height)) + return x, y, max(1, w), max(1, h) + + +def _font(size_px: int, bold: bool = False) -> ImageFont.ImageFont: + candidates = [ + "arialbd.ttf" if bold else "arial.ttf", + "calibrib.ttf" if bold else "calibri.ttf", + "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", + ] + for name in candidates: + try: + return ImageFont.truetype(name, max(4, int(size_px))) + except OSError: + continue + return ImageFont.load_default() + + +def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: int) -> None: + if not item.get("visible", True): + return + if str(item.get("layer_ownership") or "editable_text_layer") != "editable_text_layer": + return + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + text = str(item.get("text") or "") + if not text: + return + font_size_pt = float(item.get("font_size_pt") or 8) + size_px = max(4, int(round(font_size_pt * canvas_h / 387.0))) + font = _font(size_px, bold=bool(item.get("bold"))) + color = _hex_color(item.get("color_hex"), "#263747") + align = str(item.get("align") or "center").lower() + text_bbox = draw.textbbox((0, 0), text, font=font) + text_w = max(1, text_bbox[2] - text_bbox[0]) + text_h = max(1, text_bbox[3] - text_bbox[1]) + if align == "left": + tx = x + elif align == "right": + tx = x + w - text_w + else: + tx = x + (w - text_w) / 2 + ty = y + (h - text_h) / 2 + draw.text((tx, ty), text, fill=color, font=font) + + +def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: int) -> None: + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + return + points: list[tuple[int, int]] = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((int(round(float(point[0]) * canvas_w)), int(round(float(point[1]) * canvas_h)))) + if len(points) < 2: + return + color = _hex_color(arrow.get("stroke_color") or arrow.get("outline_color"), "#3F5063") + line_width = max(1, int(round(float(arrow.get("stroke_width_pt") or 1.5) * canvas_h / 277.0))) + render_style = str(arrow.get("render_style") or "").lower() + if render_style == "filled_block_arrow": + fill = _hex_color(arrow.get("fill_color"), "#AFC6DE") + start, end = points[0], points[-1] + thickness = max(8, int(round(float(arrow.get("block_arrow_thickness_percent") or 0.052) * canvas_h))) + if abs(end[0] - start[0]) >= abs(end[1] - start[1]): + y = int(round((start[1] + end[1]) / 2)) + left, right = sorted([start[0], end[0]]) + draw.rounded_rectangle((left, y - thickness // 2, right, y + thickness // 2), radius=thickness // 3, fill=fill, outline=color, width=line_width) + else: + x = int(round((start[0] + end[0]) / 2)) + top, bottom = sorted([start[1], end[1]]) + draw.rounded_rectangle((x - thickness // 2, top, x + thickness // 2, bottom), radius=thickness // 3, fill=fill, outline=color, width=line_width) + return + draw.line(points, fill=color, width=line_width, joint="curve") + + +def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str | Path | None = None) -> Path: + out = Path(out_dir) + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + width = int(canvas.get("width_px") or 1600) + height = int(canvas.get("height_px") or 900) + background = _hex_color(canvas.get("background"), "#FFFFFF") + image = Image.new("RGB", (width, height), background) + draw = ImageDraw.Draw(image) + + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + palette = style.get("palette") or ["#F7F8F7", "#D2DCDE", "#A09596"] + + for idx, panel in enumerate(program.get("panels", [])): + bbox = panel.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + local_style = panel_styles.get(panel.get("id"), {}) if isinstance(panel_styles.get(panel.get("id")), dict) else {} + fill = _hex_color(local_style.get("fill_color"), palette[idx % len(palette)]) + stroke = _hex_color(local_style.get("stroke_color"), "#FFFFFF") + draw.rounded_rectangle((x, y, x + w, y + h), radius=max(2, min(12, h // 18)), fill=fill, outline=stroke, width=2) + + asset_by_id = {str(asset.get("id")): asset for asset in program.get("assets", []) if isinstance(asset, dict)} + for slot in sorted(program.get("slots", []), key=lambda item: int(item.get("z_index") or 20)): + bbox = slot.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + asset = asset_by_id.get(str(slot.get("asset_id") or slot.get("id"))) + asset_path = out / str(asset.get("path")) if asset else out / "assets" / f"{slot.get('asset_id')}.png" + if asset_path.exists(): + with Image.open(asset_path) as asset_img: + asset_img = asset_img.convert("RGB") + asset_img.thumbnail((w, h), Image.Resampling.LANCZOS) + left = x + (w - asset_img.width) // 2 + top = y + (h - asset_img.height) // 2 + image.paste(asset_img, (left, top)) + + for arrow in program.get("arrows", []): + if isinstance(arrow, dict): + _draw_arrow(draw, arrow, width, height) + + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} + text_items = text_program.get("items", []) if isinstance(text_program, dict) else [] + visible_items = [item for item in text_items if isinstance(item, dict)] + for item in sorted(visible_items, key=lambda item: (float(item.get("z_index") or 80), str(item.get("id") or ""))): + _draw_text(draw, item, width, height) + + target = Path(preview_path) if preview_path else out / "rebuild_preview.png" + target.parent.mkdir(parents=True, exist_ok=True) + image.save(target) + return target From 9157b25515713c798790bc85f51f2796cb27b59f Mon Sep 17 00:00:00 2001 From: illion Date: Sun, 19 Jul 2026 11:50:43 +0800 Subject: [PATCH 08/74] Tighten visual critic alignment grouping --- rfs/rebuild_visual_critic.py | 21 +++++++++++++++------ tests/test_rebuild_visual_critic.py | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/rfs/rebuild_visual_critic.py b/rfs/rebuild_visual_critic.py index d8a402d..5dfa2f2 100644 --- a/rfs/rebuild_visual_critic.py +++ b/rfs/rebuild_visual_critic.py @@ -50,6 +50,14 @@ def _visible_text_items(program: dict) -> list[dict]: return [item for item in text_program.get("items", []) if isinstance(item, dict) and item.get("visible", True)] +def _text_alignment_group_key(item: dict) -> tuple[str, str] | None: + for key in ("paragraph_id", "text_group_id", "group_id", "source_group_id"): + value = str(item.get(key) or "").strip() + if value: + return (key, value) + return None + + def _detect_text_issues(program: dict) -> list[dict]: issues = [] items = _visible_text_items(program) @@ -75,9 +83,10 @@ def _detect_text_issues(program: dict) -> list[dict]: }) grouped: dict[tuple[str, str], list[dict]] = {} for item in items: - key = (str(item.get("target_id") or ""), str(item.get("role") or "")) - grouped.setdefault(key, []).append(item) - for (target_id, role), group in grouped.items(): + key = _text_alignment_group_key(item) + if key: + grouped.setdefault(key, []).append(item) + for (group_key, group_id), group in grouped.items(): if len(group) < 3: continue centers = [ @@ -88,11 +97,11 @@ def _detect_text_issues(program: dict) -> list[dict]: if len(centers) >= 3 and max(centers) - min(centers) > 0.035: issues.append({ "type": "text_group_misaligned", - "target_id": target_id, - "role": role, + "group_key": group_key, + "group_id": group_id, "text_ids": [item.get("id") for item in group], "center_y_span": round(max(centers) - min(centers), 4), - "reason": "same target/role text group is visually uneven", + "reason": "explicit text group is visually uneven", }) return issues diff --git a/tests/test_rebuild_visual_critic.py b/tests/test_rebuild_visual_critic.py index b931729..0dc6da7 100644 --- a/tests/test_rebuild_visual_critic.py +++ b/tests/test_rebuild_visual_critic.py @@ -75,6 +75,23 @@ def test_ownership_conflict_is_reported(self): self.assertEqual(report["ownership_issue_count"], 1) self.assertEqual(report["issues"][0]["type"], "text_layer_ownership_conflict") + def test_same_target_role_without_explicit_group_is_not_forced_aligned(self): + with tempfile.TemporaryDirectory() as tmp: + program = { + "text_program": {"items": [ + {"id": "a", "target_id": "panel", "role": "body_label", "bbox_percent": {"x": 0.1, "y": 0.1, "w": 0.1, "h": 0.03}}, + {"id": "b", "target_id": "panel", "role": "body_label", "bbox_percent": {"x": 0.3, "y": 0.4, "w": 0.1, "h": 0.03}}, + {"id": "c", "target_id": "panel", "role": "body_label", "bbox_percent": {"x": 0.5, "y": 0.7, "w": 0.1, "h": 0.03}}, + ]}, + "panels": [], + "slots": [], + "arrows": [], + } + + report = run_rebuild_visual_quality_check(Path(tmp), program) + + self.assertNotIn("text_group_misaligned", {item["type"] for item in report["issues"]}) + if __name__ == "__main__": unittest.main() From e3bcbf903d54716ecf0fa889077c92070d71c01f Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 17:28:54 +0800 Subject: [PATCH 09/74] Add paper-grounded editable figure workflow --- rfs/cli.py | 63 +++++++++ rfs/editable_rebuild.py | 9 ++ rfs/paper_to_editable.py | 124 +++++++++++++++++ rfs/paper_to_image/planner.py | 11 +- rfs/paper_to_image/review.py | 8 +- rfs/semantic_contract.py | 231 ++++++++++++++++++++++++++++++++ tests/test_semantic_contract.py | 44 ++++++ 7 files changed, 488 insertions(+), 2 deletions(-) create mode 100644 rfs/paper_to_editable.py create mode 100644 rfs/semantic_contract.py create mode 100644 tests/test_semantic_contract.py diff --git a/rfs/cli.py b/rfs/cli.py index fd3e172..27e9567 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -10,6 +10,7 @@ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable from .paper_to_image import run_paper_to_image +from .paper_to_editable import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro from .professional_repair import vlm_professional_repair_adapter from .presentations_qa import run_presentations_qa @@ -147,6 +148,37 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") paper_image.add_argument("--json", action="store_true", help="Emit JSON.") + paper_editable = sub.add_parser("paper-to-editable", help="Generate a paper-grounded visual reference and rebuild it as an editable PowerPoint figure.") + paper_editable.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + paper_editable.add_argument("--out", required=True, help="Output directory for paper analysis, image candidates, semantic contracts, and editable PPTX.") + paper_editable.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") + paper_editable.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") + paper_editable.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") + paper_editable.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + paper_editable.add_argument("--planner-model") + paper_editable.add_argument("--image-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") + paper_editable.add_argument("--image-candidates", type=int, default=3) + paper_editable.add_argument("--aspect-ratio", default="auto") + paper_editable.add_argument("--language", default="English") + paper_editable.add_argument("--image-model") + paper_editable.add_argument("--image-retries", type=int, default=2) + paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") + paper_editable.add_argument("--review-model") + paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + paper_editable.add_argument("--template", choices=["auto", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + paper_editable.add_argument("--repair-rounds", type=int, default=1) + paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") + paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") + paper_editable.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") + paper_editable.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") + paper_editable.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") + paper_editable.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") + paper_editable.add_argument("--allow-engineering-preview", action="store_true", help="Allow offline/non-production image previews to exercise the editable pipeline. Never enabled by default.") + paper_editable.add_argument("--no-export-preview", action="store_true") + paper_editable.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "vlm", "off"], default="auto") + paper_editable.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + paper_editable.add_argument("--json", action="store_true", help="Emit JSON.") + rebuild = sub.add_parser("rebuild-editable", help="Rebuild a reference image into a reusable editable PowerPoint composition.") rebuild.add_argument("--reference", required=True, help="Reference image path.") rebuild.add_argument("--out", required=True, help="Output directory.") @@ -325,6 +357,37 @@ def main(argv: list[str] | None = None) -> int: ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, ) + elif args.command == "paper-to-editable": + result = run_paper_to_editable( + paper=args.paper, + out=args.out, + preferences_path=args.preferences, + positive_references=args.positive_reference, + negative_references=args.negative_reference, + planner_mode=args.planner_mode, + planner_model=args.planner_model, + image_asset_mode=args.image_asset_mode, + image_candidates=args.image_candidates, + aspect_ratio=args.aspect_ratio, + language=args.language, + image_model=args.image_model, + image_retries=args.image_retries, + review_mode=args.review_mode, + review_model=args.review_model, + domain_profile=args.domain_profile, + template=args.template, + repair_rounds=args.repair_rounds, + rebuild_asset_mode=args.rebuild_asset_mode, + rebuild_asset_policy=args.rebuild_asset_policy, + layout_mode=args.layout_mode, + control_mode=args.control_mode, + text_mode=args.text_mode, + design_plan_mode=args.design_plan_mode, + export_preview=not args.no_export_preview, + allow_engineering_preview=args.allow_engineering_preview, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + ) elif args.command == "rebuild-editable": rebuild_adapters = build_rebuild_vlm_adapters(args.out) result = rebuild_editable( diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 4a05dc5..04f523d 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -20,6 +20,7 @@ from .rebuild_preview_renderer import render_rebuild_preview from .rebuild_vlm_validation import build_rebuild_vlm_validation_report from .rebuild_visual_critic import run_rebuild_visual_quality_check +from .semantic_contract import apply_paper_semantic_contract from .text_layer import build_text_layer from .utils import ensure_dir, read_json, write_json, write_text @@ -937,6 +938,7 @@ def rebuild_editable( text_grouping_mode: str = "heuristic", text_grouping_model: str | None = None, text_grouping_adapter: Callable | None = None, + semantic_contract: dict | str | Path | None = None, ) -> dict: reference_path = Path(reference) if not reference_path.exists(): @@ -960,6 +962,7 @@ def rebuild_editable( "design_plan_model": design_plan_model, "text_grouping_mode": text_grouping_mode, "text_grouping_model": text_grouping_model, + "semantic_contract": str(semantic_contract) if isinstance(semantic_contract, (str, Path)) else ("inline" if semantic_contract else None), "skip_analysis": skip_analysis, "compile_only": compile_only, }) @@ -1039,6 +1042,10 @@ def rebuild_editable( elif not semantic_report: semantic_report = {"summary": "Semantic planning skipped by --skip-analysis.", "semantic_vlm_status": "skipped", "slots": []} write_json(out_path / "slot_semantic_report.json", semantic_report) + semantic_binding_report = {} + if semantic_contract: + contract = read_json(semantic_contract) if isinstance(semantic_contract, (str, Path)) else semantic_contract + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_path) write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) specs = _make_asset_specs(program, archived_reference, out_path, asset_policy=asset_policy) @@ -1105,6 +1112,7 @@ def rebuild_editable( "control_mode": control_mode, "layout_mode": layout_mode, "rebuild_visual_quality_status": visual_quality_report.get("status"), + "semantic_binding_status": semantic_binding_report.get("status") if semantic_binding_report else None, "reports": { "input_manifest": str(out_path / "input_manifest.json"), "reference_logic_plan": str(out_path / "reference_logic_plan.json"), @@ -1118,6 +1126,7 @@ def rebuild_editable( "slot_semantic_report": str(out_path / "slot_semantic_report.json"), "rebuild_vlm_validation_report": str(out_path / "rebuild_vlm_validation_report.json"), "rebuild_visual_quality_report": str(out_path / "rebuild_visual_quality_report.json"), + "semantic_binding_report": str(out_path / "semantic_binding_report.json") if semantic_binding_report else None, "asset_generation_specs": str(out_path / "asset_generation_specs.json"), "asset_generation_report": str(out_path / "asset_generation_report.json"), "asset_economy_report": str(out_path / "asset_economy_report.json"), diff --git a/rfs/paper_to_editable.py b/rfs/paper_to_editable.py new file mode 100644 index 0000000..2d34caa --- /dev/null +++ b/rfs/paper_to_editable.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .editable_rebuild import rebuild_editable +from .paper_to_image import run_paper_to_image +from .rebuild_vlm_adapters import build_rebuild_vlm_adapters +from .semantic_contract import load_paper_semantic_contract +from .utils import ensure_dir, write_json + + +def run_paper_to_editable( + paper: str | Path, + out: str | Path, + preferences_path: str | Path | None = None, + positive_references: list[str] | None = None, + negative_references: list[str] | None = None, + planner_mode: str = "vlm", + planner_model: str | None = None, + image_asset_mode: str = "image2", + image_candidates: int = 3, + aspect_ratio: str | None = None, + language: str | None = None, + image_model: str | None = None, + image_retries: int = 2, + review_mode: str = "vlm", + review_model: str | None = None, + domain_profile: str = "auto", + template: str = "auto", + repair_rounds: int = 1, + rebuild_asset_mode: str = "api", + rebuild_asset_policy: str = "smart-api", + layout_mode: str = "hybrid", + control_mode: str = "hybrid", + text_mode: str = "ocr", + design_plan_mode: str = "vlm", + export_preview: bool = True, + allow_engineering_preview: bool = False, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + critic_adapter=None, +) -> dict[str, Any]: + root = ensure_dir(out).resolve() + image_dir = root / "paper_to_image" + editable_dir = root / "editable" + image_result = run_paper_to_image( + paper=paper, + out=image_dir, + preferences_path=preferences_path, + positive_references=positive_references, + negative_references=negative_references, + planner_mode=planner_mode, + planner_model=planner_model, + asset_mode=image_asset_mode, + candidates=image_candidates, + aspect_ratio=aspect_ratio, + language=language, + image_model=image_model, + image_retries=image_retries, + review_mode=review_mode, + review_model=review_model, + domain_profile=domain_profile, + template=template, + repair_rounds=repair_rounds, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + critic_adapter=critic_adapter, + ) + reference = image_result.get("selected_image") + reference_status = "production_selected_image" + if not reference and allow_engineering_preview: + reference = image_result.get("engineering_preview") + reference_status = "engineering_preview_explicitly_allowed" + if not reference: + blocked = { + "summary": "Paper-to-editable delivery blocked before rebuild because no production-approved image exists.", + "ok": False, + "delivery_blocked": True, + "out_dir": str(root), + "paper_to_image": image_result, + "policy": "failed image candidates are never promoted to production editable output", + } + write_json(root / "paper_to_editable_result.json", blocked) + return blocked + + contract = load_paper_semantic_contract(image_dir) + adapters = build_rebuild_vlm_adapters(editable_dir) + rebuild_ocr_engine = ocr_engine if ocr_engine in {"paddle", "easyocr", "off"} else "paddle" + editable_result = rebuild_editable( + reference=reference, + out=editable_dir, + asset_mode=rebuild_asset_mode, + asset_policy=rebuild_asset_policy, + text_mode=text_mode, + layout_mode=layout_mode, + control_mode=control_mode, + export_preview=export_preview, + ocr_engine=rebuild_ocr_engine, + ocr_lang=ocr_lang, + vlm_layout_adapter=adapters["layout"] if layout_mode in {"vlm", "hybrid"} else None, + control_adapter=adapters["control"] if control_mode in {"vlm", "hybrid"} else None, + semantic_adapter=adapters["semantic"] if layout_mode in {"vlm", "hybrid"} or control_mode in {"vlm", "hybrid"} else None, + design_plan_mode=design_plan_mode, + design_adapter=adapters["design"] if design_plan_mode == "vlm" else None, + semantic_contract=contract, + ) + result = { + "summary": "Paper-to-editable workflow completed with paper-grounded semantic bindings.", + "ok": bool(editable_result.get("ok")), + "delivery_blocked": False, + "out_dir": str(root), + "reference_status": reference_status, + "reference_image": str(reference), + "paper_to_image_dir": str(image_dir), + "editable_dir": str(editable_dir), + "pptx": editable_result.get("pptx"), + "preview": editable_result.get("preview"), + "semantic_binding_status": editable_result.get("semantic_binding_status"), + "paper_to_image": image_result, + "editable_rebuild": editable_result, + } + write_json(root / "paper_to_editable_result.json", result) + return result diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index a2f269f..000c887 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -359,6 +359,15 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: if invalid: errors.append(f"module {module_id or index + 1} references unknown evidence ids: {invalid}") + endpoint_ids = set(module_ids) + for field in ("inputs", "outputs", "innovations"): + for index, item in enumerate(spec.get(field) if isinstance(spec.get(field), list) else []): + if not isinstance(item, dict): + continue + endpoint_id = str(item.get("id") or item.get("name") or item.get("visible_label") or item.get("text") or "").strip() + if endpoint_id: + endpoint_ids.add(endpoint_id) + relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] for index, relation in enumerate(relations): if not isinstance(relation, dict): @@ -366,7 +375,7 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: continue source = str(relation.get("source") or "").strip() target = str(relation.get("target") or "").strip() - if source not in module_ids or target not in module_ids: + if source not in endpoint_ids or target not in endpoint_ids: errors.append(f"relation {index + 1} has unknown endpoint: {source} -> {target}") evidence_ids = relation.get("evidence_ids") if isinstance(relation.get("evidence_ids"), list) else [] if not evidence_ids: diff --git a/rfs/paper_to_image/review.py b/rfs/paper_to_image/review.py index 1c17128..57d26f3 100644 --- a/rfs/paper_to_image/review.py +++ b/rfs/paper_to_image/review.py @@ -272,7 +272,13 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s grounded += 1 elif item.get("status") in {"required", "optional"}: errors.append(f"{field}:{item.get('id')} lacks evidence") - module_ids = {item.get("id") for item in review.get("modules", [])} | {item.get("id") for item in review.get("research_objects", [])} + endpoint_fields = ("inputs", "outputs", "research_objects", "concepts", "modules", "innovations") + module_ids = { + item.get("id") + for field in endpoint_fields + for item in review.get(field, []) + if isinstance(item, dict) and item.get("id") + } for relation in review.get("relations", []): source = relation.get("source_id") or relation.get("source") target = relation.get("target_id") or relation.get("target") diff --git a/rfs/semantic_contract.py b/rfs/semantic_contract.py new file mode 100644 index 0000000..de807ad --- /dev/null +++ b/rfs/semantic_contract.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +from .utils import read_json, write_json + + +def load_paper_semantic_contract(run_dir: str | Path) -> dict[str, Any]: + root = Path(run_dir) + return { + "summary": "Paper-grounded semantic contract for editable figure reconstruction.", + "source_run_dir": str(root.resolve()), + "paper_review": read_json(root / "paper_review.json"), + "figure_specification": read_json(root / "figure_specification.json"), + "paper_summary": read_json(root / "paper_summary.json"), + "design_plan": read_json(root / "design_plan.json"), + "layout_intent": read_json(root / "layout_intent.json"), + "style_plan": read_json(root / "style_plan.json"), + } + + +def _text(item: dict, fallback: str = "") -> str: + for key in ("visible_label", "name", "text", "statement", "label", "id"): + value = str(item.get(key) or "").strip() + if value: + return value + return fallback + + +def _entities(contract: dict) -> list[dict]: + spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + entities: list[dict] = [] + for field, role in (("inputs", "input"), ("modules", "module"), ("outputs", "output"), ("innovations", "innovation")): + values = spec.get(field) if isinstance(spec.get(field), list) else [] + for index, raw in enumerate(values, start=1): + if not isinstance(raw, dict): + continue + label = _text(raw) + if not label: + continue + entity_id = str(raw.get("id") or raw.get("name") or f"{role}_{index:02d}").strip() + entities.append({ + "id": entity_id, + "label": label, + "role": role, + "evidence_ids": list(raw.get("evidence_ids") or []), + "source": raw, + }) + return entities + + +def _tokens(value: object) -> set[str]: + return {token for token in re.findall(r"[a-z0-9]+", str(value or "").lower()) if len(token) > 1} + + +def _object_text(obj: dict) -> str: + return " ".join(str(obj.get(key) or "") for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt")) + + +def _center(obj: dict) -> tuple[float, float]: + box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + return float(box.get("x") or 0) + float(box.get("w") or 0) / 2, float(box.get("y") or 0) + float(box.get("h") or 0) / 2 + + +def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dict], list[dict]]: + objects = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + if len(objects) < len(entities): + seen = {str(item.get("id")) for item in objects} + objects.extend( + item for item in program.get("cards", []) + program.get("panels", []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and str(item.get("id")) not in seen + ) + objects.sort(key=lambda item: (_center(item)[0], _center(item)[1], str(item.get("id") or ""))) + available = list(objects) + mapping: dict[str, dict] = {} + records: list[dict] = [] + for entity in entities: + entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) + scored = [] + for obj in available: + obj_tokens = _tokens(_object_text(obj)) + score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) + scored.append((score, -_center(obj)[0], obj)) + if not scored: + records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) + continue + scored.sort(key=lambda value: (value[0], value[1]), reverse=True) + chosen = scored[0][2] if scored[0][0] > 0 else available[0] + available.remove(chosen) + mapping[entity["id"]] = chosen + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "object_id": chosen.get("id"), + "match_score": round(max(0.0, scored[0][0]), 4), + "status": "mapped", + }) + return mapping, records + + +def _inside(inner: dict, outer: dict) -> bool: + cx = float(inner.get("x") or 0) + float(inner.get("w") or 0) / 2 + cy = float(inner.get("y") or 0) + float(inner.get("h") or 0) / 2 + return float(outer.get("x") or 0) <= cx <= float(outer.get("x") or 0) + float(outer.get("w") or 0) and float(outer.get("y") or 0) <= cy <= float(outer.get("y") or 0) + float(outer.get("h") or 0) + + +def _label_bbox(obj: dict) -> dict[str, float]: + box = obj["bbox_percent"] + x, y, w, h = (float(box[key]) for key in ("x", "y", "w", "h")) + label_h = min(0.055, max(0.026, h * 0.18)) + return {"x": round(x, 4), "y": round(max(0.0, y), 4), "w": round(w, 4), "h": round(label_h, 4)} + + +def _anchor(source: dict, target: dict) -> list[list[float]]: + sx, sy = _center(source) + tx, ty = _center(target) + sb, tb = source["bbox_percent"], target["bbox_percent"] + if abs(tx - sx) >= abs(ty - sy): + start = [float(sb["x"]) + (float(sb["w"]) if tx >= sx else 0.0), sy] + end = [float(tb["x"]) + (0.0 if tx >= sx else float(tb["w"])), ty] + mid = (start[0] + end[0]) / 2 + points = [start, [mid, start[1]], [mid, end[1]], end] + else: + start = [sx, float(sb["y"]) + (float(sb["h"]) if ty >= sy else 0.0)] + end = [tx, float(tb["y"]) + (0.0 if ty >= sy else float(tb["h"]))] + mid = (start[1] + end[1]) / 2 + points = [start, [start[0], mid], [end[0], mid], end] + clean: list[list[float]] = [] + for point in points: + rounded = [round(max(0.0, min(1.0, point[0])), 4), round(max(0.0, min(1.0, point[1])), 4)] + if not clean or clean[-1] != rounded: + clean.append(rounded) + return clean + + +def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: + entities = _entities(contract) + mapping, mapping_records = _assign_entities(entities, program) + mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} + existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] + preserved = [item for item in existing if not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes)] + semantic_text = [] + for entity in entities: + obj = mapping.get(entity["id"]) + if not obj: + continue + box = _label_bbox(obj) + semantic_text.append({ + "id": f"semantic_text_{entity['id']}", + "text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_evidence_ids": entity["evidence_ids"], + "bbox_percent": box, + "center_percent": {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)}, + "width_percent": box["w"], + "height_percent": box["h"], + "font_size_pt": 10.0, + "color_hex": "#263747", + "bold": entity["role"] in {"module", "innovation"}, + "align": "center", + "font_family_guess": "Arial", + "fit_strategy": "paper_semantic_contract_exact_label", + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + text_program = { + **text_program, + "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", + "semantic_authority": "paper_ground_truth", + "items": preserved + semantic_text, + } + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + + spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + semantic_arrows = [] + skipped_relations = [] + for index, relation in enumerate(spec.get("relations") if isinstance(spec.get("relations"), list) else [], start=1): + if not isinstance(relation, dict): + continue + source_id = str(relation.get("source") or relation.get("source_id") or "") + target_id = str(relation.get("target") or relation.get("target_id") or "") + source, target = mapping.get(source_id), mapping.get(target_id) + if not source or not target: + skipped_relations.append({"source": source_id, "target": target_id, "reason": "unmapped_endpoint"}) + continue + semantic_arrows.append({ + "id": str(relation.get("id") or f"semantic_relation_{index:02d}"), + "source_id": str(source.get("id")), + "target_id": str(target.get("id")), + "semantic_source_id": source_id, + "semantic_target_id": target_id, + "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), + "label": str(relation.get("label") or relation.get("statement") or ""), + "evidence_ids": list(relation.get("evidence_ids") or []), + "path_percent": _anchor(source, target), + "render_style": "elbow_connector", + "route_intent": "paper_semantic_relation", + "stroke_color": "#3F5063", + "stroke_width_pt": 1.8, + "arrowhead": "triangle", + "editable_in": "pptx", + }) + if semantic_arrows: + program["arrows"] = semantic_arrows + + report = { + "summary": "Paper semantic contract application report.", + "status": "pass" if len(mapping) == len(entities) and not skipped_relations else "warning", + "semantic_authority": "paper_ground_truth", + "visual_authority": "generated_reference_image", + "entity_count": len(entities), + "mapped_entity_count": len(mapping), + "relation_count": len(spec.get("relations") or []), + "mapped_relation_count": len(semantic_arrows), + "mappings": mapping_records, + "skipped_relations": skipped_relations, + "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", + } + out = Path(out_dir) + write_json(out / "paper_semantic_contract.json", contract) + write_json(out / "semantic_binding_report.json", report) + write_json(out / "text_program.json", text_program) + return program, report diff --git a/tests/test_semantic_contract.py b/tests/test_semantic_contract.py new file mode 100644 index 0000000..6e1610d --- /dev/null +++ b/tests/test_semantic_contract.py @@ -0,0 +1,44 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.semantic_contract import apply_paper_semantic_contract + + +class SemanticContractTests(unittest.TestCase): + def test_exact_paper_labels_and_relations_override_image_interpretation(self): + program = { + "slots": [ + {"id": "left", "bbox_percent": {"x": 0.05, "y": 0.25, "w": 0.2, "h": 0.3}}, + {"id": "middle", "bbox_percent": {"x": 0.4, "y": 0.25, "w": 0.2, "h": 0.3}}, + {"id": "right", "bbox_percent": {"x": 0.75, "y": 0.25, "w": 0.2, "h": 0.3}}, + ], + "cards": [], + "panels": [], + "arrows": [{"id": "image_guess", "path_percent": [[0.1, 0.1], [0.9, 0.9]]}], + "text_program": {"items": [{"id": "ocr_wrong", "text": "lnput", "bbox_percent": {"x": 0.08, "y": 0.3, "w": 0.1, "h": 0.04}}]}, + } + contract = {"figure_specification": { + "inputs": [{"id": "input", "name": "Exact Input", "evidence_ids": ["E1"]}], + "modules": [{"id": "method", "name": "Paper Method", "evidence_ids": ["E2"]}], + "outputs": [{"id": "output", "name": "Exact Output", "evidence_ids": ["E3"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "method", "type": "data_flow", "evidence_ids": ["E4"]}, + {"source": "method", "target": "output", "type": "data_flow", "evidence_ids": ["E5"]}, + ], + }} + with tempfile.TemporaryDirectory() as tmp: + updated, report = apply_paper_semantic_contract(program, contract, Path(tmp)) + + labels = {item["text"] for item in updated["text_program"]["items"]} + self.assertEqual(labels, {"Exact Input", "Paper Method", "Exact Output"}) + self.assertEqual(report["mapped_relation_count"], 2) + for arrow in updated["arrows"]: + for start, end in zip(arrow["path_percent"], arrow["path_percent"][1:]): + self.assertTrue(start[0] == end[0] or start[1] == end[1], arrow["path_percent"]) + self.assertTrue((Path(tmp) / "semantic_binding_report.json").exists()) + + +if __name__ == "__main__": + unittest.main() From 91fd7ea1ffd9969c4610a852cfeba8c7d77b5e1e Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 17:43:51 +0800 Subject: [PATCH 10/74] Restructure repository as a Codex plugin --- .codex-plugin/plugin.json | 29 ++ .github/workflows/ci.yml | 8 +- README.md | 30 +- benchmarks/README.md | 16 + codex-skills/research-figure-making/SKILL.md | 456 ------------------ .../research-figure-making/agents/openai.yaml | 7 - docs/architecture.md | 121 ++--- docs/gpt-reproduction-workflow.md | 2 +- docs/help-wanted.md | 4 +- docs/workflow.md | 8 +- .../legacy_reference_rebuilds/README.md | 5 + ...build_autofigure_architecture_ai_assets.py | 0 .../rebuild_c60_editable.py | 0 ...ild_virtual_interview_pipeline_editable.py | 0 skills/research-figure-studio/SKILL.md | 80 +++ .../research-figure-studio/agents/openai.yaml | 7 + .../references/critic_stage.md | 0 .../references/data_figures.md | 0 .../references/enhancement_pipeline.md | 0 .../references/figure_program.md | 0 .../references/image_block_prompting.md | 0 .../references/journal_quality_checklist.md | 0 .../references/paper_to_figure.md | 0 .../references/reference_image_alignment.md | 0 .../references/stylist_stage.md | 0 .../references/visual_framework_figures.md | 0 .../scripts/estimate_asset_fill.py | 0 .../scripts/validate_framework_outputs.py | 0 28 files changed, 234 insertions(+), 539 deletions(-) create mode 100644 .codex-plugin/plugin.json create mode 100644 benchmarks/README.md delete mode 100644 codex-skills/research-figure-making/SKILL.md delete mode 100644 codex-skills/research-figure-making/agents/openai.yaml create mode 100644 experiments/legacy_reference_rebuilds/README.md rename {scripts => experiments/legacy_reference_rebuilds}/rebuild_autofigure_architecture_ai_assets.py (100%) rename {scripts => experiments/legacy_reference_rebuilds}/rebuild_c60_editable.py (100%) rename {scripts => experiments/legacy_reference_rebuilds}/rebuild_virtual_interview_pipeline_editable.py (100%) create mode 100644 skills/research-figure-studio/SKILL.md create mode 100644 skills/research-figure-studio/agents/openai.yaml rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/critic_stage.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/data_figures.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/enhancement_pipeline.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/figure_program.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/image_block_prompting.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/journal_quality_checklist.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/paper_to_figure.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/reference_image_alignment.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/stylist_stage.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/references/visual_framework_figures.md (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/scripts/estimate_asset_fill.py (100%) rename {codex-skills/research-figure-making => skills/research-figure-studio}/scripts/validate_framework_outputs.py (100%) diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..14e8321 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,29 @@ +{ + "name": "research-figure-studio", + "version": "0.2.0", + "description": "Generate paper-grounded scientific figures as editable PowerPoint compositions.", + "author": { + "name": "ResearchFigureStudio contributors", + "url": "https://github.com/yiweiqin/ResearchFigureStudio" + }, + "homepage": "https://github.com/yiweiqin/ResearchFigureStudio", + "repository": "https://github.com/yiweiqin/ResearchFigureStudio", + "license": "MIT", + "keywords": ["research", "scientific-figures", "powerpoint", "pptx", "codex"], + "skills": "./skills/", + "interface": { + "displayName": "Research Figure Studio", + "shortDescription": "Paper to editable PowerPoint figures", + "longDescription": "Build scientifically grounded framework figures whose exact labels and relations remain editable in PowerPoint.", + "developerName": "ResearchFigureStudio contributors", + "category": "Productivity", + "capabilities": ["Read", "Write"], + "websiteURL": "https://github.com/yiweiqin/ResearchFigureStudio", + "defaultPrompt": [ + "Turn this paper into an editable PPT framework figure.", + "Rebuild this scientific figure as editable PowerPoint.", + "Check whether this figure matches the paper semantics." + ], + "brandColor": "#2F6F8F" + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8c5ccd..9e78731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,16 +20,16 @@ jobs: - name: Compile package and bundled scripts run: | python -m compileall -q rfs - python -m py_compile codex-skills/research-figure-making/scripts/validate_framework_outputs.py - python -m py_compile codex-skills/research-figure-making/scripts/estimate_asset_fill.py + python -m py_compile skills/research-figure-studio/scripts/validate_framework_outputs.py + python -m py_compile skills/research-figure-studio/scripts/estimate_asset_fill.py - name: Run unit tests run: python -m unittest discover -s tests -q - name: Check bundled skill metadata shell: python run: | from pathlib import Path - skill = Path('codex-skills/research-figure-making/SKILL.md') + skill = Path('skills/research-figure-studio/SKILL.md') text = skill.read_text(encoding='utf-8') assert text.startswith('---'), 'SKILL.md must start with YAML frontmatter' - assert 'name: research-figure-making' in text + assert 'name: research-figure-studio' in text assert 'description:' in text diff --git a/README.md b/README.md index fb27c69..4d79dc2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Summary -ResearchFigureStudio is a PPTX-first research figure generation pipeline for paper-grounded, reference-guided scientific framework figures. It turns a paper plus a user-provided visual reference image into an editable PowerPoint composition assembled from many slot-level image assets, not one flattened full-diagram bitmap. +ResearchFigureStudio is a paper-to-editable-PPT scientific figure engine and Codex plugin. Paper contracts provide exact labels and scientific relations; generated or user-provided images provide layout and visual style; deterministic code compiles the result into editable PowerPoint objects. The current workflow is optimized for AI/ML/NLP system figures: @@ -77,6 +77,22 @@ python -m pip install -e ".[ocr]" Windows with PowerPoint installed gives the best PPTX/PDF/PNG export path. The Python package itself can still generate and validate most intermediate artifacts without external image APIs when using `--asset-mode placeholder`. +## Paper To Editable PPTX + +The main product route is: + +```powershell +rfs paper-to-editable ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_editable" ` + --positive-reference "C:\path\optional_style_reference.png" ` + --json +``` + +This writes paper-review and image-generation artifacts under `paper_to_image/`, then writes the semantic contract, editable figure program, PPTX, preview, and QA reports under `editable/`. If no image candidate passes the production gates, the workflow stops before creating an official editable deliverable. + +For offline engineering validation only, add `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview`. This does not create a production-approved result. + ## Paper To Image Without PPTX Use `paper-to-image` when the required endpoint is a generated raster framework @@ -376,7 +392,7 @@ Optional arrow-beautification pass: rfs make-framework ` --paper "C:\path\paper.pdf" ` --reference "C:\path\reference.png" ` - --out D:\ResearchFigureStudio\output\aesthetic_experiment ` + --out .\output\aesthetic_experiment ` --slot-count 40 ` --slot-source reference-primary ` --control-localizer-mode hybrid ` @@ -466,7 +482,7 @@ Run validation: ```powershell rfs validate --out "output\paper_reference_image2" --json -python codex-skills\research-figure-making\scripts\validate_framework_outputs.py "output\paper_reference_image2" +python skills\research-figure-studio\scripts\validate_framework_outputs.py "output\paper_reference_image2" ``` ## Codex Skill @@ -474,15 +490,15 @@ python codex-skills\research-figure-making\scripts\validate_framework_outputs.py This repository includes the Codex skill under: ```text -codex-skills/research-figure-making +skills/research-figure-studio ``` To install it locally into Codex: ```powershell -$dst = Join-Path $env:USERPROFILE ".codex\skills\research-figure-making" +$dst = Join-Path $env:USERPROFILE ".codex\skills\research-figure-studio" if (Test-Path $dst) { Remove-Item -Recurse -Force $dst } -Copy-Item -Recurse "codex-skills\research-figure-making" $dst +Copy-Item -Recurse "skills\research-figure-studio" $dst ``` The skill documents the full research-figure workflow and includes the standalone framework-output validator. @@ -492,7 +508,7 @@ The skill documents the full research-figure workflow and includes the standalon ```powershell python -m compileall -q rfs python -m unittest discover -s tests -q -python -m py_compile codex-skills\research-figure-making\scripts\validate_framework_outputs.py +python -m py_compile skills\research-figure-studio\scripts\validate_framework_outputs.py ``` ## Repository Hygiene diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..a52a020 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,16 @@ +# Benchmarks + +Benchmarks are the product-level acceptance layer for paper-to-editable figure generation. + +Each case should provide a paper, optional positive/negative visual references, a semantic expectation file, and expected production gates. Do not commit copyrighted papers or secrets; fixtures may point to locally supplied inputs. + +Recommended benchmark dimensions: + +- exact paper-label preservation +- entity and relation coverage +- connector endpoint correctness +- editability of text, panels, cards, and arrows +- visual similarity to the selected reference image +- PPTX package validity and preview rendering +- cross-machine execution with no repository-specific absolute paths +- API cost and elapsed time diff --git a/codex-skills/research-figure-making/SKILL.md b/codex-skills/research-figure-making/SKILL.md deleted file mode 100644 index fe1efa6..0000000 --- a/codex-skills/research-figure-making/SKILL.md +++ /dev/null @@ -1,456 +0,0 @@ ---- -name: research-figure-making -description: >- - Use this skill when the user provides a research paper, manuscript, method - section, experiment section, LaTeX/Word/PDF/Markdown draft, result table, CSV, - figure sketch, or old paper figure and asks to create, redesign, or improve - publication-quality scientific figures, including paper-driven framework - figures, system overview diagrams, model architecture figures, pipeline - diagrams, data plots, main-result figures, multi-panel figures, or PPT/Lark - editable figures. Prioritize AI/ML/NLP research papers, paper-grounded visual - extraction, AI-generated visual blocks for framework figures, reproducible - Python plots for data figures, and PPTX-first editable outputs with optional - Lark/SVG exports. Also use for - Chinese-language requests that mean drawing figures from a paper, making paper - framework diagrams, system diagrams, data figures, main paper figures, PPT - figures, Lark whiteboard figures, method figures, result figures, or - top-tier-publication figures. ---- - -# Research Figure Making - -## Summary - -Create publication-oriented research figures from the user's actual paper content. -Do not use generic architecture templates until the paper has been read and the -paper-specific concepts, variables, modules, and claims have been extracted. -For image-rich framework figures, default to many slot-level image2 blocks, -PPTX-first editable composition, front Summary sections, no semantic cropping, -and hard asset-fill validation. - -## Trigger - -Use this skill when the user asks for figures based on: - -- a paper draft, method section, experiment section, abstract, related work, or notes -- LaTeX, Word, PDF, Markdown, plain text, tables, CSV/Excel, logs, or existing figures -- requests such as "draw Figure 1 from my paper", "make a system overview", - "create a framework diagram", "turn this paper into PPT figures", "improve my - data plot", or "redesign this figure for a top-tier paper" -- Chinese-language requests that mean drawing framework figures from a paper, - making system diagrams from method sections, turning experimental results into - data figures, creating main paper figures, PPT figures, or top-tier paper figures - -## Mandatory Framework Protocol - -For system architecture, framework, pipeline, model overview, or Figure 1 style -requests where the user wants an image-rich figure, this protocol is mandatory: - -## Front Summary Rule - -Every text artifact must start with a `Summary` section before details. This -applies to the figure brief, slot inventory, style sheet, figure program, -prompts, asset quality report, asset visual review, layout plan, visual critic, -alignment review, critic report, and final response. For Markdown, the first -non-empty heading must be `# Summary` or `## Summary`. For JSON, use a top-level -non-empty `summary` field. - -## Local RFS Route - -When `D:\ResearchFigureStudio` and the `rfs` CLI are available, use them as the -default implementation for image-rich framework figures: - -```powershell -rfs make-framework --paper --reference --out --slot-count 40 --slot-source reference-primary --complexity-profile reference-dense --candidates-per-slot 4 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 8 --asset-mode image2 --asset-workers 6 --asset-retries 3 --asset-review-mode heuristic --critic-mode heuristic -``` - -Use `--asset-mode image2` for real image generation through the Yunwu -OpenAI-compatible Images API. The logical `image-2` target maps to Yunwu's listed -`gpt-image-2` model unless `RFS_IMAGE_MODEL` or `IMAGE_MODEL` overrides it. Keep -`--asset-mode gemini` as a fallback and use `--asset-mode placeholder` only for -engineering validation. Use `--locator-mode vlm` when a -VLM should borrow the LiveFigure-style positioning idea by returning -`layout_plan.json` coordinates. The VLM must not write arbitrary PPT code or -generate a single full diagram. For API-cost control, run a small real pass first -with `--slot-count 25 --candidates-per-slot 1 --asset-workers 3`, then a full -pass after the contract validates. Yunwu image2/Gemini slot generation may run -concurrently through RFS workers; do not fall back to manual one-by-one image2 -generation unless the API path is unavailable. -Prompt planning is not a cost-saving heuristic stage by default: use -`--prompt-plan-mode vlm` so the model inspects the reference image and each -local slot before producing `slot_prompt_plan.json`. Use heuristic prompt -planning only for offline engineering validation. Use `--prompt-plan-workers` -to parallelize per-slot VLM prompt planning; use `--asset-workers` separately to -parallelize Yunwu image2/Gemini image generation. -Use `--control-localizer-mode hybrid` by default for arrow and connector -localization. This AutoFigure-inspired stage writes -`reference_control_candidates.json`, `slot_overlay.png`, and -`reference_control_overlay.png`; VLM binding may assign source-target semantics -from those overlays, while the fallback heuristic keeps the workflow offline. -The VLM may patch arrow IDs, anchors, and normalized paths only; it must not -write PPT code, rasterize arrows, or redraw the full figure. -Use `--arrow-style-mode reference` by default. This stage may soften PPT -connector line caps, vary stroke widths, assign bundle/lane metadata, and score -crossing/bend/overlap quality, but the reference image remains the hard -constraint. It must not replace reference-derived flow logic with a generic -graph router. Orthogonal obstacle-aware routing is allowed only for missing -paths or arrows explicitly marked `route_policy=fallback_reroute_allowed`; it -must never rewrite reference-locked `path_percent` values. -Use `--arrow-style-mode aesthetic` only for an explicit experimental -beautification pass. In that mode, the router may apply curve connectors and -halo underlays by default. Bundle lane offsets require explicit route opt-in -and must stay inside `reference_tunnel_percent`; the router must record -`reference_original_path_percent`, `reference_path_delta_max`, and -`reference_tunnel_preserved`, and it must not change source-target logic. -Use `rfs presentations-qa --out ` only as an optional QA pass when -the Presentations plugin is available. Presentations may import/render the PPTX, -extract layout JSON, and report connector/font/rendering drift, but it must not -mutate the PPTX, rewrite arrows, regenerate layout, or replace RFS as the -authoritative compiler for reference-locked connector-heavy figures. - -Before execution, read these references in this order: - -1. `references/paper_to_figure.md` -2. `references/visual_framework_figures.md` -3. `references/reference_image_alignment.md` when a reference image is provided -4. `references/stylist_stage.md` before writing image2 prompts -5. `references/figure_program.md` before composition or scripting -6. `references/image_block_prompting.md` before writing image2 prompts -7. `references/critic_stage.md` before final delivery -8. `references/journal_quality_checklist.md` before final delivery - -Execution checklist: - -1. Read the paper and write a paper-grounded figure brief. -2. If the user provides a reference image, parameterize it before generation: - identify slots, canvas ratios, safe areas, and fit policies. - When the user says the reference image should guide the figure, treat its - slot positions and flow logic as layout source-of-truth. The paper calibrates - terminology and scientific labels; it must not cause a generic paper-derived - grid to replace the reference image's spatial logic. - In reference-primary mode, the reference image is the highest authority for - layout, visual object choice, style, framework colors, arrow logic, and - visual rhythm. The paper only assists terminology and scientific mapping. - Write `reference_geometry.json` with precise panel/slot geometry before any - prompt generation. Every panel and slot must include `bbox_percent`, - `center_percent`, `width_percent`, `height_percent`, - `aspect_ratio_decimal`, `aspect_ratio_w_h`, `target_pixels`, - `target_pixels_exact`, and `generation_min_pixels` with at least three - decimals where numeric precision matters. `target_pixels` must equal - `target_pixels_exact`; use `generation_min_pixels` only for the image - generator's minimum resolution safeguard. - Also write `reference_control_candidates.json`, - `slot_overlay.png`, `reference_control_overlay.png`, and - `reference_controls.json` for measured arrows, connector lines, dashed - loops, transition arrows, framework shapes, and text regions. Control - candidates are analogous to a boxlib/overlay layer: they identify possible - arrows and connectors from the reference image before semantic binding. - Bound controls must record geometry, `source_id`, `target_id`, - `source_anchor`, `target_anchor`, multi-point `path_percent`, color token, - and `render_policy: ppt_shape_not_image_asset`. - Then write `arrow_style_profile.json`, `selected_arrow_routes.json`, and - `arrow_quality_report.json`. These files must state - `reference_image_hard_constraint`, preserve locked reference paths, and only - synthesize missing or explicitly fallback-allowed paths. If fallback routing - is used, record `routing_algorithm`, `route_generation_status`, candidate - count, and obstacle/crossing metrics. - If `--arrow-style-mode aesthetic` is used, also record original reference - paths, tunnel width, path delta, halo settings, connector type, and whether - each adjusted route stayed inside the reference tunnel. -3. Create a slot inventory before using image2. Default to 25-50 image slots for - a normal paper system figure. This count means non-arrow image slots only; - arrows, connector lines, dashed loops, panel frames, and titles must not be - counted as image2 assets. -4. Write `reference_style_profile.json` and `style_sheet.md` before writing - image2 prompts. They must define reference-derived color tokens, palette, - line weight, shadows, viewpoint, icon complexity, background, visual density, - font-layer rules, and image2 text policy. -5. Write `layout_plan.json` and `figure_program.json` before composition. - `layout_plan.json` records normalized reference-guided coordinates; it may be - produced by heuristic rules or VLM positioning, but never by freeform PPT - code. `figure_program.json` must record `canvas`, - `panels`, `slots`, `assets`, `labels`, `arrows`, `groups`, and - `export_targets`; slots must bind paper concepts, reference bboxes, target - ratios, safe areas, fit policies, and asset IDs. `export_targets` must - include the main editable PPTX target, usually `editable_composition.pptx`. - In reference-primary mode, `bbox_percent` values extracted from the reference - image must be preserved into `layout_plan.json`, `figure_program.json`, and - PPT placement. A VLM locator may refine arrows or identify missing slots, but - it must not freely redraw the macro framework or overwrite reference-derived - slot positions. Do not convert exact reference ratios into coarse presets - like `1:1`, `4:3`, `3:4`, `16:9`, or `9:16`; use precise decimal ratios such - as `0.538:1.000`. - Text size, position, color, and hierarchy are reference-primary too. Do not - create a default `publication_scale.json`, do not enforce a fixed - `paper_double_column` target width, and do not fail a figure only because a - reference-matched label would be small after manuscript scaling. Instead, - write `reference_text_geometry.json`, `text_program.json`, and - `text_alignment_report.json`. These files must preserve the reference - image's text bbox, center, relative font height, color, role, and hierarchy. - The paper may adapt terminology, but the PPT text layer must stay editable - and must not override the reference layout with a generic typography rule. -6. Write `slot_visual_spec.json`, `reference_slot_prompt_brief.json`, and - `slot_prompt_plan.json` before image generation. `slot_visual_spec.json` - records each slot's reference crop objects, foreground subject, secondary - objects, micro details, background fill elements, scientific mechanism - detail, required visual complexity, and forbidden simplifications. Except - explicit legend/badge slots, normal image slots must be planned as dense mini - scientific scenes/cards with 2-5 layered objects, not simple standalone - icons. The brief records what each reference slot does and what paper concept - it carries. `slot_prompt_plan.json` must be generated by a VLM by default and - include per-slot `slot_function`, local reference style, concrete - `visual_metaphor`, `must_show`, `avoid_showing`, visual-complexity fields, - and `image_prompt_core`. Every image slot must have - `reference_slot_crops/.png`, and the prompt planner must use the - full reference image, the local slot crop, the global style profile, local - color token ids, precise geometry, and the paper concept. Arrows and - connector controls never receive image prompts. For normal 25-50 slot - figures, run VLM prompt planning in parallel unless rate limits force a lower - worker count. -7. Generate image assets per slot, preferably through RFS Gemini/Yunwu - concurrent workers when configured. Manual image2 generation is only a - fallback. Do not generate one full architecture image as the final figure. -8. Create `asset_quality_report.json`, `asset_complexity_report.json`, - `composition_quality_report.json`, `asset_visual_review.json`, selected asset - contact sheet, and candidate contact sheet. The quality report must record - `content_fill_percent`, `empty_margin_percent`, `edge_cutoff_status`, - `ratio_status`, selected candidate, and `action` for every image block. The - complexity report must record `detail_score`, `object_count_estimate`, - `simple_icon_risk`, `reference_crop_match`, `style_match`, and - `selected_reason`; unresolved `too_simple`, `generic_icon`, - `reference_crop_ignored`, `single_object_on_blank_background`, or - `style_drift` must block delivery. - `composition_quality_report.json` must prove that PPT insertion used - frameless image slots, no extra white tile, no caption inside the image slot, - and at least 95% image-slot area fill. -9. Compose the final figure in `editable_composition.pptx` by default. Use PPT - editable layers for containers, arrows, connector lines, dashed loops, labels, - formulas, legends, and panel IDs. Image slots are direct frameless assets; - arrows are PPT shapes/connectors, not generated PNGs. -10. Write an alignment review noting source grounding, slot count, rejected assets, - and reference-image mismatches. -11. Write `visual_critic_iter_0.json` and `critic_report.md` or - `critic_report.json` before final delivery. - It must check paper faithfulness, slot count, text controllability, whether - the output used a single full diagram, vector-only fallback, semantic - cropping, incomplete image blocks, asset fill quality, asset complexity, - reference-crop match, style match, and PPT editability. - Major failures must be fixed or the delivery must stop. -12. Run `scripts/validate_framework_outputs.py ` before final - delivery when a local output directory exists. -13. Optionally run `rfs presentations-qa --out ` after validation. - Treat `autoRouteConnectorPx failed` or similar plugin connector fallback - warnings as QA evidence. Do not let the Presentations plugin rebuild the - figure or override RFS connector geometry. - -Hard workflow order: - -`input archive -> paper brief -> reference slot/control analysis -> reference_geometry.json -> -reference_control_candidates.json -> slot_overlay.png/reference_control_overlay.png -> -reference_controls.json -> arrow_style_profile.json/selected_arrow_routes.json/arrow_quality_report.json -> -reference_style_profile.json/style sheet -> -layout_plan.json -> figure_program.json -> reference_text_geometry.json/text_program.json/text_alignment_report.json -> slot_visual_spec.json -> -reference_slot_prompt_brief.json -> slot_prompt_plan.json -> image2/Gemini slot prompts -> generated -assets/asset_quality_report/asset_complexity_report/composition_quality_report/asset_visual_review/contact sheets -> -editable_composition.pptx -> PDF/PNG export -> visual critic -> critic report -> -final validation/export -> optional presentations QA report` - -Image block fill rules: - -- Target useful visual content fill: 90%-97% of the canvas. -- Minimum passing content fill: 85% unless the slot is explicitly marked as a - sparse symbol with a documented reason. -- Maximum pure empty margin in any direction: 10%. -- Safe area means the key subject is not cut off; it does not mean leaving a - large blank border. Background texture, secondary details, card surfaces, and - non-critical supporting marks should extend close to the canvas edge. -- If a block has too much whitespace, regenerate or redesign the prompt. Padding - is only for ratio matching, not for fixing a tiny subject on a blank canvas. - -Forbidden shortcuts: - -- Do not satisfy an image-rich framework request with vector-only shapes unless - the user explicitly asks for a vector-only diagram. -- Do not silently replace image2 blocks with PPT shapes, SVG, or Lark shapes - because that is faster or easier. -- Do not deliver only an SVG source file for an editable research figure unless - the user explicitly asks for SVG-only or Illustrator/Inkscape-first work. -- Do not generate a single full diagram image and then screenshot or crop it as - the final result. -- Do not use a browser or canvas screenshot as the primary artifact unless it is - a rendered composition assembled from generated slot assets. -- Do not use the Presentations plugin as the primary compiler for image-rich - research figures; it is QA-only unless the user explicitly asks for a - separate presentation deck workflow. -- If image2/image generation is unavailable, say so and stop or ask for a - fallback; do not silently switch to a vector-only workflow. - -Required output files for image-rich framework figures: - -- `slot_inventory.json` or `slot_inventory.md` -- `reference_geometry.json` -- `reference_control_candidates.json` -- `slot_overlay.png` -- `reference_control_overlay.png` -- `reference_controls.json` -- `arrow_style_profile.json` -- `selected_arrow_routes.json` -- `arrow_quality_report.json` -- `reference_style_profile.json` -- `style_sheet.md` or `style_sheet.json` -- `input_manifest.json` -- `layout_plan.json` -- `figure_program.json` -- `reference_text_geometry.json` -- `text_program.json` -- `text_alignment_report.json` -- `slot_visual_spec.json` -- `reference_slot_prompt_brief.json` -- `slot_prompt_plan.json` -- `reference_slot_crops/.png` for every non-arrow image slot -- `prompts.md` -- at least 25 generated image assets for normal system figures -- `asset_candidate_contact_sheet.png` -- `asset_quality_report.json` -- `asset_complexity_report.json` -- `composition_quality_report.json` -- `asset_visual_review.json` -- `asset_contact_sheet.png` -- `editable_composition.pptx` -- final figure export -- `visual_critic_iter_0.json` -- `alignment_review.md` -- `critic_report.md` or `critic_report.json` - -Optional Presentations-plugin QA outputs: - -- `presentations_plugin_qa_report.json` -- `presentations_plugin_qa_report.md` -- `presentations_plugin_qa_workspace/` - -Validation must fail if the output uses only a single generated full-diagram -image, lacks slot assets, lacks prompts, lacks a contact sheet, or records -semantic cropping as the fitting strategy. Validation must also fail when the -style sheet, layout plan, figure program, reference text geometry, text program, -text alignment report, slot visual spec, reference slot -prompt brief, slot prompt plan, reference geometry, asset quality report, asset -complexity report, composition quality report, asset visual review, visual -critic report, critic report, or editable PPTX source is missing. -Validation must fail when any unresolved image block has `content_fill_percent` -below its minimum or `empty_margin_percent` above its maximum. -Validation must fail when a slot uses a coarse preset ratio, when PPT insertion -adds an extra white tile, or when an inserted image fills less than 95% of its -slot area. -Validation must fail when PPT text is not bound to -`reference_text_geometry.json`, when text center/bbox/font-ratio drift from the -reference exceeds the recorded alignment tolerance, or when critical text is -not editable in PPTX. Validation must not fail only because the text would be -small under a default paper-double-column scaling assumption. -Validation must fail when a normal non-legend slot lacks `secondary_objects` or -`micro_details`, or when a selected asset has unresolved `too_simple`, -`generic_icon`, `reference_crop_ignored`, `single_object_on_blank_background`, -or `style_drift`. -Validation must fail when arrow, dashed loop, transition, or connector elements -appear in `slots`/`assets`; they must appear in `reference_controls.json` and -`figure_program.json` as editable PPT controls with source/target logic and -style tokens. Validation must also fail when `reference_control_candidates.json` -or the slot/control overlay images are missing, when a bound control lacks -`source_id`, `target_id`, `source_anchor`, `target_anchor`, or at least two -`path_percent` points, or when composition reports that a control was not -rendered as an editable PPT connector. Validation must fail when -`arrow_style_profile.json`, `selected_arrow_routes.json`, or -`arrow_quality_report.json` are missing, or when an arrow style stage overrides -a locked reference path without a documented reason. Validation must also fail -when fallback obstacle routing is applied to a reference-locked path, or when -route artifacts omit `routing_algorithm` / `route_generation_status`. -For aesthetic mode, validation or critic review must fail when -`reference_tunnel_preserved` is false, when source-target binding changes, or -when a curve/bundle/halo style is baked into raster images instead of PPT -editable connector shapes. -Validation must also fail when image slots lack local reference crops, local color token ids, or -`reference_style_profile.json` grounding. - -## Core Workflow - -1. Read the source material first. Extract the research problem, method story, - inputs/outputs, named components, training/inference flow, datasets, baselines, - metrics, and main claims before planning any figure. -2. Create a candidate figure inventory. Include only figures that carry paper - information: method overview, system pipeline, model architecture, data - construction, main results, ablation, case study, error analysis, or appendix - figures. -3. Route each figure: - - **Framework / architecture / pipeline**: use the visual-block workflow in - `references/visual_framework_figures.md`. - If the user provides a full reference image first, - also use `references/reference_image_alignment.md`. - Before image2 prompting, write a style sheet using - `references/stylist_stage.md` and a structured program using - `references/figure_program.md`. - Then create `reference_slot_prompt_brief.json` and VLM-generated - `slot_prompt_plan.json`; for image2 visual blocks, use - `references/image_block_prompting.md` to - generate slot-level prompts with target aspect ratios and low-blank-space - requirements. - Before delivery, run the critic stage in `references/critic_stage.md`. - - **Data / statistical / result plots**: use `references/data_figures.md`. - - **Mixed paper figure or multi-panel layout**: combine both routes, then use - PPTX-first composition for the editable working file. -4. Keep scientific structure editable. Use generated images for visual blocks, - but add labels, formulas, arrows, panel marks, grouping boxes, legends, and - captions with editable PPT tools by default. -5. For reference-image-driven framework figures, parameterize the reference image - first: identify visual slots, estimate each slot's relative bounding box and - aspect ratio, then generate many small blocks for those slots instead of a few - large macro-panel images. -6. Run the journal-quality check before final delivery. See - `references/journal_quality_checklist.md`. - -## Required Grounding Rules - -- Every framework module name must come from the user's paper or be explicitly - introduced as an inferred simplification. -- Never insert stock modules such as encoder, retriever, memory, or decoder unless - the paper actually uses that concept or the user approves the abstraction. -- AI-generated visual blocks may illustrate scientific objects, data flow, - model components, reasoning stages, or experimental settings, but must not be - trusted for exact text, formulas, numbers, axes, or citations. -- For data figures, Python remains the source of truth for computation and plots. - PPT editing is for layout, annotation, panel assembly, and presentation polish, - not for changing plotted values. -- Keep source artifacts: raw input, extracted figure brief, generated prompts, - full reference image when provided, original image blocks, enhanced blocks, - slot inventory, layout plan, style sheet, figure program, asset quality - report, asset visual review, visual critic report, critic report, alignment - review notes, editable composition, and final exports. - -## Output Policy - -Default outputs: - -- editable working file: `editable_composition.pptx` by default -- publication export: PDF plus 600 DPI PNG/TIFF by default -- optional SVG/PDF vector export only when the venue, user, or downstream - Illustrator/Inkscape workflow explicitly needs it -- reproducibility files: Python plotting script and cleaned data for data plots -- prompt record: visual-block prompts and selected/rejected image notes -- for image-rich framework figures: slot inventory, image2 prompts, generated - assets, asset quality report, asset visual review, contact sheets, style - sheet, layout plan, figure program, editable composition, visual critic, - critic report, final export, and alignment review that pass - `rfs validate` or `scripts/validate_framework_outputs.py` when outputs are - local - -Use `references/paper_to_figure.md` for paper analysis and figure planning. Use -`references/enhancement_pipeline.md` when generated images need upscaling, -sharpening, background removal, or artifact cleanup. Use -`references/reference_image_alignment.md` only when the user supplies a complete -reference framework image as the visual blueprint for the final editable -composition. Do not create a full reference image yourself unless the user -explicitly asks for one. Use `references/stylist_stage.md` to lock style before -prompting. Use `references/figure_program.md` to define the structured -intermediate layout before scripting or composition. Use -`references/image_block_prompting.md` whenever image2 prompts are needed for -framework sub-blocks. Use `references/critic_stage.md` to block final delivery -when the workflow violates the required image-rich, editable, no-crop rules. - diff --git a/codex-skills/research-figure-making/agents/openai.yaml b/codex-skills/research-figure-making/agents/openai.yaml deleted file mode 100644 index e4b5c2e..0000000 --- a/codex-skills/research-figure-making/agents/openai.yaml +++ /dev/null @@ -1,7 +0,0 @@ -interface: - display_name: "Research Figure Making" - short_description: "Create paper-grounded scientific figures and editable framework diagrams." - default_prompt: "Use $research-figure-making to turn my paper draft into publication-quality framework and data figures." - -policy: - allow_implicit_invocation: true diff --git a/docs/architecture.md b/docs/architecture.md index bf376f3..5fb3561 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,69 +1,74 @@ -# Summary +# Summary -ResearchFigureStudio is organized as a small deterministic pipeline, not a patched copy of LiveFigure. The borrowed ideas are reference-image-based positioning and AutoFigure-style structured control localization: the system writes overlay/candidate JSON for visual elements, then renders editable PPT objects from deterministic programs. +ResearchFigureStudio has one product goal: turn a paper into a scientifically faithful, editable PowerPoint figure. Paper-derived contracts own semantics; generated or supplied images own layout and visual style; deterministic code owns PPTX composition and validation. -## Architecture +## Product architecture + +```mermaid +flowchart LR + P["Paper"] --> G["PaperGroundTruth"] + R["Optional visual references"] --> I["Reference image generation"] + G --> I + I --> V["Approved visual reference"] + V --> A["Layout, cards, slots, style analysis"] + G --> B["Semantic binding"] + A --> B + B --> C["Editable figure program"] + C --> PPT["Editable PPTX"] + PPT --> Q["Deterministic and optional VLM QA"] +``` + +The semantic path never passes through OCR alone. OCR may help recover geometry from an image, but exact labels, entity IDs, relations, and evidence links are restored from `paper_review.json` and `figure_specification.json` before compilation. + +## Repository map ```text -Input Loader - -> Paper Analyzer - -> Reference Analyzer - -> Control Localizer - -> Arrow Stylist/Router - -> Stylist - -> Layout Locator - -> Figure Program Builder - -> Prompt Planner - -> Asset Generator - -> Asset Reviewer - -> PPT Compiler - -> Exporter - -> Visual Critic - -> Validator +.codex-plugin/ Codex plugin manifest +skills/research-figure-studio Thin agent instructions; no machine paths +rfs/ Installable, agent-independent Python engine + paper_to_image/ Paper review, grounding, planning, image candidates + coevolution/ Experimental creator/judge research + paper_to_editable.py Product workflow orchestrator + semantic_contract.py Paper truth → editable object bindings + editable_rebuild.py Reference image → editable figure program/PPTX + ppt_compiler.py Deterministic PowerPoint renderer + rebuild_visual_critic.py Deterministic rebuild QA +tests/ Automated behavior and contract tests +benchmarks/ Real product acceptance cases and metrics +experiments/ Research prototypes and historical scripts +docs/ Architecture, workflows, decisions, roadmaps +scripts/ Reusable installation/maintenance scripts only ``` -## Modules +The current package remains under `rfs/` to avoid a disruptive all-at-once move. A future `src/rfs/` migration should be a packaging-only change after the paper-to-editable contracts and benchmarks stabilize. + +## Authority model + +| Concern | Source of truth | +|---|---| +| Scientific entities and exact labels | `paper_review.json`, `figure_specification.json` | +| Scientific relations and evidence | Paper semantic contract | +| Layout, visual rhythm, palette, object style | Approved generated/user reference image | +| Editable object geometry | `figure_program.json` | +| PowerPoint rendering | `ppt_compiler.py` | +| Production eligibility | Candidate, semantic, visual, and PPTX validation reports | + +## Stable workflow boundaries -- `rfs/input_archive.py`: copies paper and reference into `out/inputs/` and writes `input_manifest.json`. -- `rfs/input_loader.py`: extracts text from source documents. -- `rfs/paper_analyzer.py`: creates paper-grounded figure brief. -- `rfs/reference_analyzer.py`: creates 25-50 non-arrow slot inventory from the reference image geometry and paper figure goal; it also detects arrow/control candidates, writes overlays, and emits `reference_control_candidates.json` / `reference_controls.json`. -- `rfs/arrow_router.py`: assigns reference-preserving arrow aesthetics, bundle metadata, line softness, and route QA reports without changing reference-image flow logic. It also provides reference-constrained orthogonal fallback routing for missing or explicitly fallback-allowed connectors only. In aesthetic mode it records original paths and applies reference-tunnel bundle offsets only for explicitly opted-in routes. -- `rfs/stylist.py`: writes the style sheet before image prompting. -- `rfs/layout_locator.py`: creates `layout_plan.json`; heuristic mode is local, VLM mode returns JSON coordinates only. -- `rfs/program_builder.py`: creates `figure_program.json`, the single source for PPT compilation, preserving control source/target anchors and normalized routes from `reference_controls.json`. -- `rfs/prompt_planner.py`: creates `reference_slot_prompt_brief.json` and `slot_prompt_plan.json`; default VLM mode inspects the full reference image plus local slot crops and writes `image_prompt_core` for each slot. Per-slot calls can run in parallel via `--prompt-plan-workers`. -- `rfs/asset_generator.py`: generates 1-5 candidates per slot, selects the best block, and writes contact sheets and QA metrics. -- `rfs/asset_reviewer.py`: performs heuristic or VLM visual review of selected assets. -- `rfs/ppt_compiler.py`: renders editable PPTX from `figure_program.json` with contain-fit images and editable labels/arrows/groups; multi-segment arrows and dashed loops are rendered as PPT connector shapes, not image assets. -- `rfs/exporter.py`: exports PDF/PNG where local tooling supports it. -- `rfs/visual_critic.py`: compares reference and final render; VLM mode can propose JSON coordinate corrections and arrow-only patches without rewriting the whole figure. -- `rfs/validator.py`: blocks delivery when required artifacts or hard constraints are missing. +- `paper-to-image`: produces reviewed visual candidates and paper-grounding artifacts; it never creates PPTX. +- `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. +- `paper-to-editable`: runs both stages and requires a production-approved image unless engineering preview use is explicitly enabled. +- Plugin skill: selects and invokes workflows; it does not contain the engine or hardcoded repository paths. -## Non-Goals +## Production rules -- Do not import LiveFigure as a dependency. -- Do not let a VLM write arbitrary PowerPoint code. -- Do not generate a single full architecture image and crop it into pieces. -- Do not make SVG the main editing source. -- Do not bake critical scientific text, formulas, arrows, or labels into image blocks. -- Do not count arrows, connector lines, dashed loops, or transition symbols as generated image slots. +- Failed candidates never become `selected_image.png`. +- Engineering previews are opt-in and never production-approved. +- Critical text and relations remain editable PPT objects. +- Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. +- VLM correction is optional and bounded; deterministic reports remain available offline. +- Historical sample-specific scripts stay under `experiments/legacy_reference_rebuilds/` and are not imported by production code. -## Data Contracts +## Planned package refinement -- `slot_inventory.json`: paper concept plus candidate slot metadata. -- `reference_control_candidates.json`: AutoFigure-inspired boxlib-like candidate list for arrows, connectors, loops, and branch routes. -- `slot_overlay.png`: visual overlay of detected slot IDs for human/VLM binding. -- `reference_control_overlay.png`: visual overlay of control candidate IDs such as `AR01`. -- `reference_controls.json`: bound editable PPT controls with `source_id`, `target_id`, anchors, path, style token, and render policy. -- `arrow_style_profile.json`: reference-first arrow style rules, fallback routing policy, and routing algorithm metadata for main flow, branch, convergence, feedback loops, and module flow. -- `selected_arrow_routes.json`: selected route/style assignments, bundle IDs, lane indices, reference-lock status, route generation status, and routing algorithm. -- `arrow_quality_report.json`: crossing, bend, obstacle-overlap, and aesthetic-score diagnostics for PPT connectors. -- `composition_quality_report.json`: records PPT arrow connector type, halo usage, route style, and editability alongside slot composition checks. -- `layout_plan.json`: normalized positions, panels, arrows, and z order. -- `figure_program.json`: final composition program consumed by the PPT compiler. -- `reference_slot_prompt_brief.json`: per-slot paper concept, local reference role, geometry, and function briefing sent before prompt planning. -- `slot_prompt_plan.json`: VLM-generated per-slot prompt plan, including `image_prompt_core`, `must_show`, and `avoid_showing`. -- `asset_quality_report.json`: fill/margin/ratio/cutoff/candidate-selection metrics. -- `asset_visual_review.json`: semantic and visual review of selected image blocks. -- `visual_critic_iter_*.json`: reference alignment and optional coordinate correction proposals. +As interfaces stabilize, split `rfs/` internally into `contracts`, `providers`, `analysis`, `planning`, `generation`, `composition`, `evaluation`, and `workflows`. Do this incrementally with compatibility imports, not as a mass rename that obscures behavior changes. diff --git a/docs/gpt-reproduction-workflow.md b/docs/gpt-reproduction-workflow.md index e36e4e9..c3811db 100644 --- a/docs/gpt-reproduction-workflow.md +++ b/docs/gpt-reproduction-workflow.md @@ -57,7 +57,7 @@ Existing one-off scripts in `scripts/` are historical examples. Do not treat the Install locally: ```powershell -cd D:\ResearchFigureStudio +cd python -m pip install --upgrade pip python -m pip install -e . ``` diff --git a/docs/help-wanted.md b/docs/help-wanted.md index 2000b47..63e80e0 100644 --- a/docs/help-wanted.md +++ b/docs/help-wanted.md @@ -67,8 +67,8 @@ We welcome help on: - `rfs/ppt_compiler.py` - `rfs/visual_critic.py` - `rfs/validator.py` -- `codex-skills/research-figure-making/references/reference_image_alignment.md` -- `codex-skills/research-figure-making/references/figure_program.md` +- `skills/research-figure-studio/references/reference_image_alignment.md` +- `skills/research-figure-studio/references/figure_program.md` ## Desired End State diff --git a/docs/workflow.md b/docs/workflow.md index 929790f..515ca6a 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -29,20 +29,20 @@ The workflow is designed to keep scientific content, layout, image generation, a Offline engineering check: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\offline_check --asset-mode placeholder --locator-mode heuristic --control-localizer-mode heuristic --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json -rfs validate --out D:\ResearchFigureStudio\output\offline_check --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\offline_check --asset-mode placeholder --locator-mode heuristic --control-localizer-mode heuristic --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json +rfs validate --out .\output\offline_check --json ``` Small real image check: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\real_small --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\real_small --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json ``` Full quality run: ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out D:\ResearchFigureStudio\output\full_quality --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json +rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.png" --out .\output\full_quality --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --control-localizer-mode hybrid --arrow-style-mode reference --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json ``` ## Parallel Asset Generation diff --git a/experiments/legacy_reference_rebuilds/README.md b/experiments/legacy_reference_rebuilds/README.md new file mode 100644 index 0000000..0eeed27 --- /dev/null +++ b/experiments/legacy_reference_rebuilds/README.md @@ -0,0 +1,5 @@ +# Legacy reference-specific rebuilds + +These scripts document the repository's early image-to-PPT experiments. They intentionally remain outside the reusable `rfs` package because they contain sample-specific assumptions and historical local paths. + +Use them only as implementation references for visual details. Do not import them from production workflows, plugin skills, tests, or packaging code. New reusable behavior belongs in `rfs/` with tests and path-independent inputs. diff --git a/scripts/rebuild_autofigure_architecture_ai_assets.py b/experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py similarity index 100% rename from scripts/rebuild_autofigure_architecture_ai_assets.py rename to experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py diff --git a/scripts/rebuild_c60_editable.py b/experiments/legacy_reference_rebuilds/rebuild_c60_editable.py similarity index 100% rename from scripts/rebuild_c60_editable.py rename to experiments/legacy_reference_rebuilds/rebuild_c60_editable.py diff --git a/scripts/rebuild_virtual_interview_pipeline_editable.py b/experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py similarity index 100% rename from scripts/rebuild_virtual_interview_pipeline_editable.py rename to experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py diff --git a/skills/research-figure-studio/SKILL.md b/skills/research-figure-studio/SKILL.md new file mode 100644 index 0000000..f5bc4af --- /dev/null +++ b/skills/research-figure-studio/SKILL.md @@ -0,0 +1,80 @@ +--- +name: research-figure-studio +description: Create paper-grounded, editable PowerPoint scientific framework figures from papers and optional visual references. Use for paper-to-PPT figures, editable architecture diagrams, method overviews, system pipelines, or image-to-editable-PPT reconstruction. +--- + +# Research Figure Studio + +## Summary + +Create scientifically faithful figures whose labels, entities, and relations come from the paper, while generated images or user references provide layout and visual style. The deliverable is an editable PPTX, not a flattened diagram image. + +## Preferred route + +Use the installed `rfs` command from any working directory. Do not assume a repository location or embed machine-specific paths. + +For a complete paper-to-editable workflow: + +```powershell +rfs paper-to-editable --paper --out --positive-reference --json +``` + +For image-only reconstruction: + +```powershell +rfs rebuild-editable --reference --out --asset-policy smart-api --json +``` + +Use `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview` only for explicit offline engineering validation. Never present that result as production-approved. + +## Required semantic policy + +- Read and structure the paper before generating the visual reference. +- Treat `paper_review.json` and `figure_specification.json` as scientific ground truth. +- Treat the generated/reference image as layout and style evidence only. +- Keep exact paper labels, relations, formulas, numbers, and citations out of raster assets whenever possible. +- Bind paper entities and relations into editable PPT text and connector objects through `paper_semantic_contract.json` and `semantic_binding_report.json`. +- Never reconstruct scientific truth by OCR alone when paper contracts are available. +- Never promote a failed candidate or engineering preview to production output automatically. + +## Workflow + +1. Archive the paper and references. +2. Build the evidence map and structured paper review. +3. Validate evidence grounding, including relations whose endpoints may be inputs, outputs, concepts, research objects, modules, or innovations. +4. Plan the figure and generate candidate reference images. +5. Stop if no production candidate passes all gates. +6. Analyze the selected image globally before local extraction. +7. Rebuild panels, cards, assets, editable labels, and editable connectors. +8. Apply the paper semantic contract so exact labels and scientific relations override image/OCR guesses. +9. Generate deterministic visual QA and a fallback preview. +10. Validate the PPTX and report warnings or blockers. + +## References + +Read only what the task needs: + +- Paper analysis: `references/paper_to_figure.md` +- Framework figures: `references/visual_framework_figures.md` +- User-provided image alignment: `references/reference_image_alignment.md` +- Structured figure program: `references/figure_program.md` +- Image block prompting: `references/image_block_prompting.md` +- Critic and delivery gates: `references/critic_stage.md` and `references/journal_quality_checklist.md` +- Data plots: `references/data_figures.md` + +## Delivery checklist + +The normal paper-to-editable output should contain: + +- `paper_to_image/paper_review.json` +- `paper_to_image/figure_specification.json` +- `paper_to_image/planning_validation_report.json` +- `editable/paper_semantic_contract.json` +- `editable/semantic_binding_report.json` +- `editable/figure_program.json` +- `editable/text_program.json` +- `editable/rebuild_visual_quality_report.json` +- `editable/editable_composition.pptx` +- `editable/rebuild_preview.png` when preview export is requested + +If production gates fail, return the engineering artifacts and the blocker; do not claim that the figure is ready for paper submission. diff --git a/skills/research-figure-studio/agents/openai.yaml b/skills/research-figure-studio/agents/openai.yaml new file mode 100644 index 0000000..891e199 --- /dev/null +++ b/skills/research-figure-studio/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Research Figure Studio" + short_description: "Create paper-grounded scientific figures as editable PowerPoint diagrams." + default_prompt: "Use $research-figure-studio to turn my paper into an editable PowerPoint framework figure." + +policy: + allow_implicit_invocation: true diff --git a/codex-skills/research-figure-making/references/critic_stage.md b/skills/research-figure-studio/references/critic_stage.md similarity index 100% rename from codex-skills/research-figure-making/references/critic_stage.md rename to skills/research-figure-studio/references/critic_stage.md diff --git a/codex-skills/research-figure-making/references/data_figures.md b/skills/research-figure-studio/references/data_figures.md similarity index 100% rename from codex-skills/research-figure-making/references/data_figures.md rename to skills/research-figure-studio/references/data_figures.md diff --git a/codex-skills/research-figure-making/references/enhancement_pipeline.md b/skills/research-figure-studio/references/enhancement_pipeline.md similarity index 100% rename from codex-skills/research-figure-making/references/enhancement_pipeline.md rename to skills/research-figure-studio/references/enhancement_pipeline.md diff --git a/codex-skills/research-figure-making/references/figure_program.md b/skills/research-figure-studio/references/figure_program.md similarity index 100% rename from codex-skills/research-figure-making/references/figure_program.md rename to skills/research-figure-studio/references/figure_program.md diff --git a/codex-skills/research-figure-making/references/image_block_prompting.md b/skills/research-figure-studio/references/image_block_prompting.md similarity index 100% rename from codex-skills/research-figure-making/references/image_block_prompting.md rename to skills/research-figure-studio/references/image_block_prompting.md diff --git a/codex-skills/research-figure-making/references/journal_quality_checklist.md b/skills/research-figure-studio/references/journal_quality_checklist.md similarity index 100% rename from codex-skills/research-figure-making/references/journal_quality_checklist.md rename to skills/research-figure-studio/references/journal_quality_checklist.md diff --git a/codex-skills/research-figure-making/references/paper_to_figure.md b/skills/research-figure-studio/references/paper_to_figure.md similarity index 100% rename from codex-skills/research-figure-making/references/paper_to_figure.md rename to skills/research-figure-studio/references/paper_to_figure.md diff --git a/codex-skills/research-figure-making/references/reference_image_alignment.md b/skills/research-figure-studio/references/reference_image_alignment.md similarity index 100% rename from codex-skills/research-figure-making/references/reference_image_alignment.md rename to skills/research-figure-studio/references/reference_image_alignment.md diff --git a/codex-skills/research-figure-making/references/stylist_stage.md b/skills/research-figure-studio/references/stylist_stage.md similarity index 100% rename from codex-skills/research-figure-making/references/stylist_stage.md rename to skills/research-figure-studio/references/stylist_stage.md diff --git a/codex-skills/research-figure-making/references/visual_framework_figures.md b/skills/research-figure-studio/references/visual_framework_figures.md similarity index 100% rename from codex-skills/research-figure-making/references/visual_framework_figures.md rename to skills/research-figure-studio/references/visual_framework_figures.md diff --git a/codex-skills/research-figure-making/scripts/estimate_asset_fill.py b/skills/research-figure-studio/scripts/estimate_asset_fill.py similarity index 100% rename from codex-skills/research-figure-making/scripts/estimate_asset_fill.py rename to skills/research-figure-studio/scripts/estimate_asset_fill.py diff --git a/codex-skills/research-figure-making/scripts/validate_framework_outputs.py b/skills/research-figure-studio/scripts/validate_framework_outputs.py similarity index 100% rename from codex-skills/research-figure-making/scripts/validate_framework_outputs.py rename to skills/research-figure-studio/scripts/validate_framework_outputs.py From 879b9eb2239690ca35d2113ec04d02ff602b109b Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 17:47:00 +0800 Subject: [PATCH 11/74] Render editable card frames in rebuild outputs --- rfs/ppt_compiler.py | 44 +++++++++++++++++++++++++++------ rfs/rebuild_preview_renderer.py | 20 +++++++++++++++ tests/test_ppt_compiler.py | 13 ++++++++++ 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/rfs/ppt_compiler.py b/rfs/ppt_compiler.py index 5e4dbee..5436329 100644 --- a/rfs/ppt_compiler.py +++ b/rfs/ppt_compiler.py @@ -64,6 +64,26 @@ def _add_round_rect(slide, x: float, y: float, w: float, h: float, fill: str, st return shape +def _add_card_frame(slide, card: dict, width_in: float, height_in: float): + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + shape_type = MSO_SHAPE.RECTANGLE if str(card.get("shape_kind") or "rounded_rect").lower() == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + if float(card.get("fill_transparency", 1.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(card.get("fill_color") or "#FFFFFF")) + shape.line.color.rgb = _rgb(str(card.get("stroke_color") or "#59AFCB")) + shape.line.width = Pt(float(card.get("stroke_width_pt") or 1.5)) + dash_style = str(card.get("dash_style") or "solid").lower() + if dash_style in {"dash", "dashed"}: + shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH + elif dash_style in {"dot", "dotted"}: + shape.line.dash_style = getattr(MSO_LINE_DASH_STYLE, "ROUND_DOT", MSO_LINE_DASH_STYLE.DASH) + shape.shadow.inherit = False + return shape + + def _add_label(slide, text: str, x: float, y: float, w: float, h: float, font_size: float = 9, bold: bool = False, align=PP_ALIGN.CENTER, color: str = "#163B4D", font_family: str | None = None): box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family) @@ -379,16 +399,25 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, ) - for idx, card in enumerate(program.get("cards", [])): - bbox = card.get("bbox_percent") - if not isinstance(bbox, dict): + rendered_cards = [] + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index", 12))): + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): continue - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - fill = card.get("fill_color") or "#FFFFFF" - stroke = card.get("stroke_color") or (palette[(idx + 1) % len(palette)] if palette else "#B8C0CC") - _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=float(card.get("stroke_width_pt") or 1.0)) + _add_card_frame(slide, card, width_in, height_in) + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) if card.get("title") and not has_text_program: _add_label(slide, str(card.get("title")), x + 0.03, y + 0.03, max(0.05, w - 0.06), min(0.22, h * 0.32), font_size=7, bold=True) + rendered_cards.append({ + "card_id": card.get("id"), + "semantic_role": card.get("semantic_role"), + "shape_kind": card.get("shape_kind", "rounded_rect"), + "bbox_percent": card.get("bbox_percent"), + "dash_style": card.get("dash_style", "solid"), + "fill_transparency": float(card.get("fill_transparency", 1.0)), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok", + }) rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) @@ -554,6 +583,7 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: "caption_inside_image_slot_allowed": False, }, "slots": composition_items, + "cards": rendered_cards, "arrows": rendered_arrows, "text": rendered_text_items, }) diff --git a/rfs/rebuild_preview_renderer.py b/rfs/rebuild_preview_renderer.py index d166397..c2ecef9 100644 --- a/rfs/rebuild_preview_renderer.py +++ b/rfs/rebuild_preview_renderer.py @@ -101,6 +101,22 @@ def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: draw.line(points, fill=color, width=line_width, joint="curve") +def _draw_card(draw: ImageDraw.ImageDraw, card: dict, canvas_w: int, canvas_h: int) -> None: + bbox = card.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + stroke = _hex_color(card.get("stroke_color"), "#59AFCB") + fill = None if float(card.get("fill_transparency", 1.0)) >= 1.0 else _hex_color(card.get("fill_color"), "#FFFFFF") + line_width = max(1, int(round(float(card.get("stroke_width_pt") or 1.5) * canvas_h / 360.0))) + radius = max(0, int(round(float(card.get("corner_radius") or 0.08) * min(w, h)))) + rect = (x, y, x + w, y + h) + if str(card.get("shape_kind") or "rounded_rect").lower() == "rect": + draw.rectangle(rect, fill=fill, outline=stroke, width=line_width) + else: + draw.rounded_rectangle(rect, radius=radius, fill=fill, outline=stroke, width=line_width) + + def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str | Path | None = None) -> Path: out = Path(out_dir) canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} @@ -124,6 +140,10 @@ def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str stroke = _hex_color(local_style.get("stroke_color"), "#FFFFFF") draw.rounded_rectangle((x, y, x + w, y + h), radius=max(2, min(12, h // 18)), fill=fill, outline=stroke, width=2) + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index") or 12)): + if isinstance(card, dict): + _draw_card(draw, card, width, height) + asset_by_id = {str(asset.get("id")): asset for asset in program.get("assets", []) if isinstance(asset, dict)} for slot in sorted(program.get("slots", []), key=lambda item: int(item.get("z_index") or 20)): bbox = slot.get("bbox_percent") diff --git a/tests/test_ppt_compiler.py b/tests/test_ppt_compiler.py index e86f6a2..459abe9 100644 --- a/tests/test_ppt_compiler.py +++ b/tests/test_ppt_compiler.py @@ -29,6 +29,17 @@ def test_multisegment_and_dashed_arrows_are_rendered_as_ppt_connectors(self): "bbox_percent": {"x": 0.05, "y": 0.05, "w": 0.90, "h": 0.80}, "editable_in": "pptx", }], + "cards": [{ + "id": "card_a", + "semantic_role": "outer_group_boundary", + "bbox_percent": {"x": 0.08, "y": 0.16, "w": 0.76, "h": 0.54}, + "shape_kind": "rect", + "stroke_color": "#59AFCB", + "stroke_width_pt": 1.5, + "dash_style": "dash", + "fill_transparency": 1.0, + "z_index": 12, + }], "slots": [ { "id": "slot_a", @@ -87,6 +98,8 @@ def test_multisegment_and_dashed_arrows_are_rendered_as_ppt_connectors(self): by_id = {item["arrow_id"]: item for item in report["arrows"]} self.assertEqual(by_id["multi_a"]["segment_count"], 3) self.assertEqual(by_id["loop_a"]["segment_count"], 4) + self.assertEqual(report["cards"][0]["card_id"], "card_a") + self.assertEqual(report["cards"][0]["render_policy"], "ppt_shape_not_image_asset") with zipfile.ZipFile(pptx) as archive: slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") From 0968cc66e26deb994d62353527debc69f2c6b508 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 17:49:31 +0800 Subject: [PATCH 12/74] Ignore local paper-to-editable smoke outputs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 413d876..2eab502 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ env/ tmp/translation_env/ tmp/codex-presentations/ tmp/pdfs/*/ +tmp/paper-to-editable-smoke*/ *.key *.pem From d09d12e3157cd178b26a3091fe5ae886ebd5263a Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 18:50:04 +0800 Subject: [PATCH 13/74] Introduce stable engine package boundaries --- .gitignore | 1 + docs/architecture.md | 10 +- experiments/literature_review/README.md | 8 + .../raw_text}/diagrammergpt.txt | 0 .../literature_review/raw_text}/scidoc.txt | 0 .../literature_review/raw_text}/scirex.txt | 0 .../raw_text}/structured_ie.txt | 0 .../scripts}/extract_layout_text.py | 0 .../scripts}/translate_papers.py | 0 rfs/analysis/__init__.py | 10 + rfs/cli.py | 2 +- rfs/composition/__init__.py | 6 + rfs/composition/pptx.py | 590 +++++++++++++++++ rfs/composition/preview.py | 176 ++++++ rfs/contracts/__init__.py | 5 + rfs/contracts/semantic.py | 231 +++++++ rfs/editable_rebuild.py | 7 +- rfs/evaluation/__init__.py | 5 + rfs/evaluation/rebuild_visual.py | 195 ++++++ rfs/generation/__init__.py | 6 + rfs/paper_to_editable.py | 126 +--- rfs/planning/__init__.py | 6 + rfs/ppt_compiler.py | 592 +----------------- rfs/professional_rebuild.py | 2 +- rfs/providers/__init__.py | 14 + rfs/rebuild_preview_renderer.py | 177 +----- rfs/rebuild_visual_critic.py | 197 +----- rfs/semantic_contract.py | 233 +------ rfs/workflow.py | 2 +- rfs/workflows/__init__.py | 5 + rfs/workflows/paper_to_editable.py | 124 ++++ tests/README.md | 11 + tests/__init__.py | 1 + tests/e2e/__init__.py | 1 + tests/integration/__init__.py | 1 + tests/{ => integration}/test_coevolution.py | 0 .../{ => integration}/test_paper_to_image.py | 0 .../test_presentations_qa.py | 0 .../test_rebuild_editable.py | 0 tests/{ => integration}/test_validator.py | 0 tests/unit/__init__.py | 1 + tests/{ => unit}/test_arrow_router.py | 0 tests/unit/test_package_boundaries.py | 43 ++ tests/{ => unit}/test_ppt_compiler.py | 0 .../{ => unit}/test_rebuild_visual_critic.py | 0 tests/{ => unit}/test_semantic_contract.py | 0 tests/{ => unit}/test_text_grouping.py | 0 tests/{ => unit}/test_text_layer_ocr.py | 0 48 files changed, 1475 insertions(+), 1313 deletions(-) create mode 100644 experiments/literature_review/README.md rename {tmp/pdfs => experiments/literature_review/raw_text}/diagrammergpt.txt (100%) rename {tmp/pdfs => experiments/literature_review/raw_text}/scidoc.txt (100%) rename {tmp/pdfs => experiments/literature_review/raw_text}/scirex.txt (100%) rename {tmp/pdfs => experiments/literature_review/raw_text}/structured_ie.txt (100%) rename {tmp/pdfs => experiments/literature_review/scripts}/extract_layout_text.py (100%) rename {tmp/pdfs => experiments/literature_review/scripts}/translate_papers.py (100%) create mode 100644 rfs/analysis/__init__.py create mode 100644 rfs/composition/__init__.py create mode 100644 rfs/composition/pptx.py create mode 100644 rfs/composition/preview.py create mode 100644 rfs/contracts/__init__.py create mode 100644 rfs/contracts/semantic.py create mode 100644 rfs/evaluation/__init__.py create mode 100644 rfs/evaluation/rebuild_visual.py create mode 100644 rfs/generation/__init__.py create mode 100644 rfs/planning/__init__.py create mode 100644 rfs/providers/__init__.py create mode 100644 rfs/workflows/__init__.py create mode 100644 rfs/workflows/paper_to_editable.py create mode 100644 tests/README.md create mode 100644 tests/__init__.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/integration/__init__.py rename tests/{ => integration}/test_coevolution.py (100%) rename tests/{ => integration}/test_paper_to_image.py (100%) rename tests/{ => integration}/test_presentations_qa.py (100%) rename tests/{ => integration}/test_rebuild_editable.py (100%) rename tests/{ => integration}/test_validator.py (100%) create mode 100644 tests/unit/__init__.py rename tests/{ => unit}/test_arrow_router.py (100%) create mode 100644 tests/unit/test_package_boundaries.py rename tests/{ => unit}/test_ppt_compiler.py (100%) rename tests/{ => unit}/test_rebuild_visual_critic.py (100%) rename tests/{ => unit}/test_semantic_contract.py (100%) rename tests/{ => unit}/test_text_grouping.py (100%) rename tests/{ => unit}/test_text_layer_ocr.py (100%) diff --git a/.gitignore b/.gitignore index 2eab502..913afc4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ env/ tmp/translation_env/ tmp/codex-presentations/ tmp/pdfs/*/ +tmp/pdfs/ tmp/paper-to-editable-smoke*/ *.key *.pem diff --git a/docs/architecture.md b/docs/architecture.md index 5fb3561..54beeea 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,6 +26,14 @@ The semantic path never passes through OCR alone. OCR may help recover geometry .codex-plugin/ Codex plugin manifest skills/research-figure-studio Thin agent instructions; no machine paths rfs/ Installable, agent-independent Python engine + contracts/ Paper semantic contracts and stable schemas + providers/ VLM and external model integrations + analysis/ Paper, layout, and text analysis entry points + planning/ Paper and rebuild planning entry points + generation/ Reference and slot-asset generation entry points + workflows/ Product-level workflow entry points + composition/ PPTX compilation and fallback preview rendering + evaluation/ Deterministic rebuild quality checks paper_to_image/ Paper review, grounding, planning, image candidates coevolution/ Experimental creator/judge research paper_to_editable.py Product workflow orchestrator @@ -33,7 +41,7 @@ rfs/ Installable, agent-independent Python engine editable_rebuild.py Reference image → editable figure program/PPTX ppt_compiler.py Deterministic PowerPoint renderer rebuild_visual_critic.py Deterministic rebuild QA -tests/ Automated behavior and contract tests +tests/ Unit, integration, and future end-to-end tests benchmarks/ Real product acceptance cases and metrics experiments/ Research prototypes and historical scripts docs/ Architecture, workflows, decisions, roadmaps diff --git a/experiments/literature_review/README.md b/experiments/literature_review/README.md new file mode 100644 index 0000000..a0f8eb0 --- /dev/null +++ b/experiments/literature_review/README.md @@ -0,0 +1,8 @@ +# Literature review archive + +This directory contains early research notes and extracted paper text used to understand related diagram-generation systems. + +- `raw_text/` contains archived text extracts for internal literature analysis. +- `scripts/` contains historical extraction and translation helpers. Some scripts preserve machine-specific paths and are not production utilities. + +Nothing under this directory may be imported by `rfs/`, the Codex plugin skill, or automated production workflows. diff --git a/tmp/pdfs/diagrammergpt.txt b/experiments/literature_review/raw_text/diagrammergpt.txt similarity index 100% rename from tmp/pdfs/diagrammergpt.txt rename to experiments/literature_review/raw_text/diagrammergpt.txt diff --git a/tmp/pdfs/scidoc.txt b/experiments/literature_review/raw_text/scidoc.txt similarity index 100% rename from tmp/pdfs/scidoc.txt rename to experiments/literature_review/raw_text/scidoc.txt diff --git a/tmp/pdfs/scirex.txt b/experiments/literature_review/raw_text/scirex.txt similarity index 100% rename from tmp/pdfs/scirex.txt rename to experiments/literature_review/raw_text/scirex.txt diff --git a/tmp/pdfs/structured_ie.txt b/experiments/literature_review/raw_text/structured_ie.txt similarity index 100% rename from tmp/pdfs/structured_ie.txt rename to experiments/literature_review/raw_text/structured_ie.txt diff --git a/tmp/pdfs/extract_layout_text.py b/experiments/literature_review/scripts/extract_layout_text.py similarity index 100% rename from tmp/pdfs/extract_layout_text.py rename to experiments/literature_review/scripts/extract_layout_text.py diff --git a/tmp/pdfs/translate_papers.py b/experiments/literature_review/scripts/translate_papers.py similarity index 100% rename from tmp/pdfs/translate_papers.py rename to experiments/literature_review/scripts/translate_papers.py diff --git a/rfs/analysis/__init__.py b/rfs/analysis/__init__.py new file mode 100644 index 0000000..c0522fe --- /dev/null +++ b/rfs/analysis/__init__.py @@ -0,0 +1,10 @@ +"""Input and visual analysis entry points. + +Implementations remain in their existing modules while interfaces stabilize. +""" + +from ..layout_planner import plan_reference_layout +from ..paper_to_image.analyzer import parse_paper +from ..reference_text_extractor import extract_reference_text + +__all__ = ["extract_reference_text", "parse_paper", "plan_reference_layout"] diff --git a/rfs/cli.py b/rfs/cli.py index 27e9567..70883dc 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -10,7 +10,7 @@ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable from .paper_to_image import run_paper_to_image -from .paper_to_editable import run_paper_to_editable +from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro from .professional_repair import vlm_professional_repair_adapter from .presentations_qa import run_presentations_qa diff --git a/rfs/composition/__init__.py b/rfs/composition/__init__.py new file mode 100644 index 0000000..0d26f49 --- /dev/null +++ b/rfs/composition/__init__.py @@ -0,0 +1,6 @@ +"""Deterministic editable-figure composition and preview rendering.""" + +from .pptx import compile_ppt +from .preview import render_rebuild_preview + +__all__ = ["compile_ppt", "render_rebuild_preview"] diff --git a/rfs/composition/pptx.py b/rfs/composition/pptx.py new file mode 100644 index 0000000..cc3eabb --- /dev/null +++ b/rfs/composition/pptx.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.dml import MSO_LINE_DASH_STYLE +from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE +from pptx.enum.text import PP_ALIGN, MSO_ANCHOR +from pptx.oxml import parse_xml +from pptx.util import Inches, Pt + +from ..utils import pct_to_inches, write_json + + +def _rgb(hex_color: str) -> RGBColor: + value = hex_color.strip().lstrip("#") or "000000" + return RGBColor(int(value[:2], 16), int(value[2:4], 16), int(value[4:6], 16)) + + +def _apply_arrow(connector, size: str = "sm") -> None: + ln = connector.line._get_or_add_ln() + ln.append(parse_xml(f'')) + + +def _apply_line_cap(connector, cap: str = "round") -> None: + value = {"round": "rnd", "square": "sq"}.get(str(cap).lower()) + if not value: + return + ln = connector.line._get_or_add_ln() + ln.set("cap", value) + + +def _connector_type_for_route(route_style: str): + if str(route_style).lower() in {"soft_curve", "dashed_loop", "dashed_spline_like"}: + return MSO_CONNECTOR.CURVE + return MSO_CONNECTOR.STRAIGHT + + +def _set_text(shape, text: str, font_size: float = 10, bold: bool = False, color: str = "#163B4D", align=PP_ALIGN.CENTER, font_family: str | None = None) -> None: + tf = shape.text_frame + tf.clear() + tf.word_wrap = True + tf.vertical_anchor = MSO_ANCHOR.MIDDLE + p = tf.paragraphs[0] + p.alignment = align + run = p.add_run() + run.text = text + run.font.size = Pt(float(font_size)) + run.font.bold = bold + if font_family: + run.font.name = str(font_family) + run.font.color.rgb = _rgb(color) + + +def _add_round_rect(slide, x: float, y: float, w: float, h: float, fill: str, stroke: str, width_pt: float = 1.2): + shape = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x), Inches(y), Inches(w), Inches(h)) + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(fill) + shape.line.color.rgb = _rgb(stroke) + shape.line.width = Pt(width_pt) + shape.shadow.inherit = False + return shape + + +def _add_card_frame(slide, card: dict, width_in: float, height_in: float): + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + shape_type = MSO_SHAPE.RECTANGLE if str(card.get("shape_kind") or "rounded_rect").lower() == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + if float(card.get("fill_transparency", 1.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(card.get("fill_color") or "#FFFFFF")) + shape.line.color.rgb = _rgb(str(card.get("stroke_color") or "#59AFCB")) + shape.line.width = Pt(float(card.get("stroke_width_pt") or 1.5)) + dash_style = str(card.get("dash_style") or "solid").lower() + if dash_style in {"dash", "dashed"}: + shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH + elif dash_style in {"dot", "dotted"}: + shape.line.dash_style = getattr(MSO_LINE_DASH_STYLE, "ROUND_DOT", MSO_LINE_DASH_STYLE.DASH) + shape.shadow.inherit = False + return shape + + +def _add_label(slide, text: str, x: float, y: float, w: float, h: float, font_size: float = 9, bold: bool = False, align=PP_ALIGN.CENTER, color: str = "#163B4D", font_family: str | None = None): + box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) + _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family) + return box + + +def _add_title_block(slide, program: dict, width_in: float, height_in: float) -> None: + title_block = program.get("title_block") + if not isinstance(title_block, dict): + return + title = str(title_block.get("title", "")).strip() + subtitle = str(title_block.get("subtitle", "")).strip() + bbox = title_block.get("bbox_percent") + if not title or not isinstance(bbox, dict): + return + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + title_h = min(h * 0.58, 0.42) + subtitle_h = max(0.18, min(h - title_h, 0.26)) + _add_label( + slide, + title, + x, + y, + w, + title_h, + font_size=int(title_block.get("title_font_size", 24)), + bold=True, + align=PP_ALIGN.LEFT, + ) + if subtitle: + subtitle_box = _add_label( + slide, + subtitle, + x, + y + title_h * 0.94, + w, + subtitle_h, + font_size=int(title_block.get("subtitle_font_size", 11)), + bold=False, + align=PP_ALIGN.LEFT, + ) + for paragraph in subtitle_box.text_frame.paragraphs: + for run in paragraph.runs: + run.font.color.rgb = _rgb(str(title_block.get("subtitle_color", "#333333"))) + + +def _short_caption(text: str) -> str: + replacements = { + "participant seated before 49-inch screen": "Participant setup", + "3D virtual interviewer on screen": "Virtual interviewer", + "wide-angle camera capturing full body": "Full-body camera", + "virtual interview room setup": "Interview room", + "interview question prompt on screen": "Question prompt", + "raw interview video recording": "Raw video", + "287 participant video collection": "287 participants", + "36 interview questions": "36 questions", + "full-body video thumbnails": "Full-body clips", + "participant clip grid": "Clip grid", + "FFmpeg extracts audio from video": "FFmpeg audio", + "audio waveform stream": "Audio stream", + "FunASR speech recognition": "FunASR ASR", + "spoken text with timestamps": "Text + timestamps", + "MTCNN face detection": "MTCNN face", + "face clips": "Face clips", + "AlphaPose full-body skeleton extraction": "AlphaPose pose", + "full-body pose skeleton stream": "Pose stream", + "frame sampling from video": "Frame sampling", + "sampled video frames": "Video frames", + "timestamp alignment clock": "Alignment clock", + "aligned audio and text streams": "Audio + text", + "aligned face and pose streams": "Face + pose", + "aligned frame stream": "Frame stream", + "face modality": "Face", + "frame modality": "Frame", + "pose modality": "Pose", + "audio modality": "Audio", + "text modality": "Text", + "NEO-FFI-3 questionnaire": "NEO-FFI-3", + "OCEAN score vector": "OCEAN scores", + "Openness score": "Openness", + "Conscientiousness score": "Conscientiousness", + "Extraversion score": "Extraversion", + "Agreeableness score": "Agreeableness", + "Neuroticism score": "Neuroticism", + } + if text in replacements: + return replacements[text] + if len(text) <= 22: + return text + words = text.replace("/", " ").split() + return " ".join(words[:3]) if len(words) > 3 else text[:22] + + +def _picture_contain_box(image_path: Path, x: float, y: float, w: float, h: float) -> tuple[float, float, float, float, float]: + with Image.open(image_path) as img: + iw, ih = img.size + image_ratio = iw / max(ih, 1) + slot_ratio = w / max(h, 0.001) + if image_ratio > slot_ratio: + fit_w = w + fit_h = w / image_ratio + else: + fit_h = h + fit_w = h * image_ratio + left = x + (w - fit_w) / 2 + top = y + (h - fit_h) / 2 + fill_percent = fit_w * fit_h / max(w * h, 0.001) * 100 + return left, top, fit_w, fit_h, fill_percent + + +def _add_picture_contain(slide, image_path: Path, x: float, y: float, w: float, h: float): + left, top, fit_w, fit_h, _fill_percent = _picture_contain_box(image_path, x, y, w, h) + return slide.shapes.add_picture(str(image_path), Inches(left), Inches(top), width=Inches(fit_w), height=Inches(fit_h)) + + +def _panel_map(program: dict) -> dict[str, dict]: + return {panel["id"]: panel for panel in program.get("panels", [])} + + +def _text_program_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + items = text_program.get("items", []) + return [item for item in items if isinstance(item, dict) and item.get("visible", True)] + + +def _text_item_for_target(program: dict, target_id: str, role: str) -> dict | None: + for item in _text_program_items(program): + if str(item.get("target_id")) == target_id and str(item.get("role")) == role: + return item + return None + + +def _align_from_text_item(item: dict): + value = str(item.get("align") or "").lower() + if value == "left": + return PP_ALIGN.LEFT + if value == "right": + return PP_ALIGN.RIGHT + return PP_ALIGN.CENTER + + +def _object_map(program: dict) -> dict[str, dict]: + objects = {panel["id"]: panel for panel in program.get("panels", [])} + objects.update({slot["id"]: slot for slot in program.get("slots", [])}) + return objects + + +def _bbox_center(bbox: dict, canvas_w: float, canvas_h: float) -> tuple[float, float]: + x, y, w, h = pct_to_inches(bbox, canvas_w, canvas_h) + return x + w / 2, y + h / 2 + + +def _arrow_endpoints(source: dict, target: dict, canvas_w: float, canvas_h: float) -> tuple[float, float, float, float]: + sbox = source["bbox_percent"] + tbox = target["bbox_percent"] + sx, sy = _bbox_center(sbox, canvas_w, canvas_h) + tx, ty = _bbox_center(tbox, canvas_w, canvas_h) + s_left, s_top, s_w, s_h = pct_to_inches(sbox, canvas_w, canvas_h) + t_left, t_top, t_w, t_h = pct_to_inches(tbox, canvas_w, canvas_h) + + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + if dx >= 0: + return s_left + s_w + 0.03, sy, t_left - 0.03, ty + return s_left - 0.03, sy, t_left + t_w + 0.03, ty + if dy >= 0: + return sx, s_top + s_h + 0.03, tx, t_top - 0.03 + return sx, s_top - 0.03, tx, t_top + t_h + 0.03 + + +def _arrow_points_from_path(path: list, width_in: float, height_in: float) -> list[tuple[float, float]]: + points = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((float(point[0]) * width_in, float(point[1]) * height_in)) + return points + + +def _draw_program_arrows(slide, program: dict, width_in: float, height_in: float) -> list[dict]: + objects_by_id = _object_map(program) + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + token_map = {str(item.get("token_id")): item for item in style.get("color_tokens", []) if isinstance(item, dict)} + rendered: list[dict] = [] + for arrow in program.get("arrows", []): + if arrow.get("type") == "custom_bus": + continue + source = objects_by_id.get(arrow.get("source") or arrow.get("source_id")) + target = objects_by_id.get(arrow.get("target") or arrow.get("target_id")) + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if len(path) >= 2 and all(isinstance(point, list) and len(point) >= 2 for point in path): + points = _arrow_points_from_path(path, width_in, height_in) + elif source and target: + x1, y1, x2, y2 = _arrow_endpoints(source, target, width_in, height_in) + points = [(x1, y1), (x2, y2)] + else: + continue + token = token_map.get(str(arrow.get("style_token_id"))) + arrow_color = str(token.get("hex")) if token else str(arrow.get("stroke_color") or "#1F6F8B") + control_kind = str(arrow.get("control_kind") or arrow.get("type", "")).lower() + dashed = control_kind in {"dashed_loop", "dashed", "loop"} + if str(arrow.get("line_pattern", "")).lower() in {"dash", "dashed"}: + dashed = True + line_width = float(arrow.get("stroke_width_pt") or style.get("arrow_weight_pt") or 1.7) + route_style = str(arrow.get("route_style") or "") + connector_type = _connector_type_for_route(route_style) + halo_width = float(arrow.get("halo_width_pt") or 0.0) + halo_color = str(arrow.get("halo_color") or "#FFFFFF") + arrowhead_size = str(arrow.get("arrowhead_size") or "sm").lower() + if arrowhead_size not in {"sm", "med", "lg"}: + arrowhead_size = "sm" + segment_count = 0 + for idx, ((x1, y1), (x2, y2)) in enumerate(zip(points[:-1], points[1:])): + if halo_width > 0: + halo = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + halo.line.color.rgb = _rgb(halo_color) + halo.line.width = Pt(max(line_width + 1.2, halo_width)) + _apply_line_cap(halo, str(arrow.get("line_cap") or "round")) + if dashed: + halo.line.dash_style = MSO_LINE_DASH_STYLE.DASH + connector = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + connector.line.color.rgb = _rgb(arrow_color) + connector.line.width = Pt(line_width) + _apply_line_cap(connector, str(arrow.get("line_cap") or "round")) + if dashed: + connector.line.dash_style = MSO_LINE_DASH_STYLE.DASH + if idx == len(points) - 2: + _apply_arrow(connector, arrowhead_size) + segment_count += 1 + rendered.append({ + "arrow_id": arrow.get("id"), + "control_kind": control_kind or "straight_arrow", + "semantic_role": arrow.get("semantic_role"), + "route_style": route_style, + "bundle_id": arrow.get("bundle_id"), + "lane_index": arrow.get("lane_index"), + "line_cap": arrow.get("line_cap", "round"), + "line_pattern": "dash" if dashed else "solid", + "connector_type": "curve" if connector_type == MSO_CONNECTOR.CURVE else "straight", + "halo_width_pt": halo_width, + "halo_color": halo_color if halo_width > 0 else None, + "stroke_width_pt": line_width, + "arrowhead_size": arrowhead_size, + "routing_algorithm": arrow.get("routing_algorithm"), + "route_generation_status": arrow.get("route_generation_status"), + "segment_count": segment_count, + "point_count": len(points), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok" if segment_count else "not_rendered", + }) + return rendered + + +def _ocean_label(text: str) -> str: + low = text.lower() + if "openness" in low: + return "Openness" + if "conscientiousness" in low: + return "Conscientiousness" + if "extraversion" in low: + return "Extraversion" + if "agreeableness" in low: + return "Agreeableness" + if "neuroticism" in low: + return "Neuroticism" + return _short_caption(text) + + +def compile_ppt(program: dict, out_dir: str | Path) -> Path: + out = Path(out_dir) + canvas = program["canvas"] + width_in = float(canvas["width_in"]) + height_in = float(canvas["height_in"]) + style = program.get("style", {}) + palette = style.get("palette") or style.get("reference_palette") or ["#2D6FB7", "#E17721", "#6B57C8", "#1B9A94", "#4B9B52", "#D44E5D"] + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + + prs = Presentation() + prs.slide_width = Inches(width_in) + prs.slide_height = Inches(height_in) + slide = prs.slides.add_slide(prs.slide_layouts[6]) + + _add_title_block(slide, program, width_in, height_in) + + panels_by_id = _panel_map(program) + panel_shapes = {} + has_text_program = bool(_text_program_items(program)) + ocr_panel_title_targets = { + str(item.get("target_id")) + for item in _text_program_items(program) + if str(item.get("role")) == "panel_title" and "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() + } + for idx, panel in enumerate(program["panels"]): + x, y, w, h = pct_to_inches(panel["bbox_percent"], width_in, height_in) + local_style = panel_styles.get(panel["id"], {}) if isinstance(panel_styles.get(panel["id"], {}), dict) else {} + fill = local_style.get("fill_color") or palette[idx % len(palette)] + stroke = local_style.get("stroke_color") or palette[(idx + 1) % len(palette)] + header_color = local_style.get("header_color") or stroke + shape = _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=1.4) + panel_shapes[panel["id"]] = shape + header_h = min(0.34, h * 0.17) + header = _add_round_rect(slide, x, y, w, header_h, header_color, header_color, width_pt=0.8) + title_text_item = _text_item_for_target(program, panel["id"], "panel_title") + _set_text( + header, + "" if str(panel["id"]) in ocr_panel_title_targets else panel["title"], + font_size=float(title_text_item.get("font_size_pt")) if title_text_item else (9 if w < 2.0 else 10), + bold=True, + color=str(title_text_item.get("color_hex")) if title_text_item else "#FFFFFF", + font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, + ) + + rendered_cards = [] + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index", 12))): + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + _add_card_frame(slide, card, width_in, height_in) + x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) + if card.get("title") and not has_text_program: + _add_label(slide, str(card.get("title")), x + 0.03, y + 0.03, max(0.05, w - 0.06), min(0.22, h * 0.32), font_size=7, bold=True) + rendered_cards.append({ + "card_id": card.get("id"), + "semantic_role": card.get("semantic_role"), + "shape_kind": card.get("shape_kind", "rounded_rect"), + "bbox_percent": card.get("bbox_percent"), + "dash_style": card.get("dash_style", "solid"), + "fill_transparency": float(card.get("fill_transparency", 1.0)), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + "status": "ok", + }) + + rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) + + # Slot image layer and editable captions. + composition_items = [] + caption_queue = [] + ordered_slots = sorted(program["slots"], key=lambda item: int(item.get("z_index", 20))) + for slot in ordered_slots: + x, y, w, h = pct_to_inches(slot["bbox_percent"], width_in, height_in) + asset_path = out / "assets" / f"{slot['asset_id']}.png" + tile_added = False + if slot.get("composition_type") == "symbol_cutout": + if asset_path.exists(): + _add_picture_contain(slide, asset_path, x, y, w, h) + _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) + else: + fill_percent = 0.0 + if bool(slot.get("show_slot_caption", False)) and not has_text_program: + parent = panels_by_id.get(slot.get("panel_id")) + if parent: + px, py, pw, ph = pct_to_inches(parent["bbox_percent"], width_in, height_in) + label_x = x + w + 0.035 + label_w = max(0.28, min(0.86, px + pw - label_x - 0.03)) + caption_queue.append(( + _ocean_label(slot.get("display_label") or slot["paper_concept"]), + label_x, + y + h * 0.10, + label_w, + h * 0.80, + 5 if label_w < 0.55 else 6, + PP_ALIGN.LEFT, + )) + composition_items.append({ + "slot_id": slot["id"], + "asset_id": slot["asset_id"], + "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + "tile_frame_added": tile_added, + "caption_inside_image_slot": False, + "slot_bbox_percent": slot["bbox_percent"], + "image_slot_area_fill_percent": round(fill_percent, 2), + "status": "ok" if fill_percent >= 95 else "image_area_below_95", + }) + continue + if asset_path.exists(): + _add_picture_contain(slide, asset_path, x, y, w, h) + _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) + else: + fill_percent = 0.0 + if bool(slot.get("show_slot_caption", False)) and not has_text_program: + caption_h = min(0.20, h * 0.18) + caption = slot.get("display_label") or _short_caption(slot["paper_concept"]) + caption_queue.append((caption, x + w * 0.03, y + h + 0.01, w * 0.94, caption_h, 5 if w < 0.8 else 6, PP_ALIGN.CENTER)) + composition_items.append({ + "slot_id": slot["id"], + "asset_id": slot["asset_id"], + "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + "tile_frame_added": tile_added, + "caption_inside_image_slot": False, + "slot_bbox_percent": slot["bbox_percent"], + "image_slot_area_fill_percent": round(fill_percent, 2), + "status": "ok" if fill_percent >= 95 else "image_area_below_95", + }) + + for caption, x, y, w, h, font_size, align in caption_queue: + _add_label(slide, caption, x, y, w, h, font_size=font_size, align=align) + + rendered_text_items = [] + for item in _text_program_items(program): + is_ocr_text = "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() + if str(item.get("role")) == "panel_title" and not is_ocr_text: + rendered_text_items.append({ + "text_id": item.get("id"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "rendered_as": "panel_header_text", + "editable_in": "pptx", + }) + continue + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + _add_label( + slide, + str(item.get("text") or ""), + x, + y, + w, + h, + font_size=float(item.get("font_size_pt") or 6), + bold=bool(item.get("bold")), + align=_align_from_text_item(item), + color=str(item.get("color_hex") or "#263747"), + font_family=str(item.get("font_family_guess") or "") or None, + ) + rendered_text_items.append({ + "text_id": item.get("id"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "color_hex": item.get("color_hex"), + "font_family_guess": item.get("font_family_guess"), + "fit_strategy": item.get("fit_strategy"), + "ocr_confidence": item.get("ocr_confidence"), + "editable_in": "pptx", + "rendered_as": "ppt_textbox", + }) + + for label in program.get("labels", []): + bbox = label.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = pct_to_inches(bbox, width_in, height_in) + _add_label( + slide, + str(label.get("text") or ""), + x, + y, + w, + h, + font_size=float(label.get("font_size_pt") or 9), + bold=bool(label.get("bold", True)), + align=PP_ALIGN.LEFT if str(label.get("align")).lower() == "left" else PP_ALIGN.CENTER, + color=str(label.get("color_hex") or "#263747"), + ) + + # Shared resource bus as editable connectors. + shared = panels_by_id.get("shared_resource_library") + if shared: + sx, sy, sw, sh = pct_to_inches(shared["bbox_percent"], width_in, height_in) + bus_y = sy - 0.30 + line = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(sx + 0.1), Inches(bus_y), Inches(sx + sw - 0.1), Inches(bus_y)) + line.line.color.rgb = _rgb("#1F6F8B") + line.line.width = Pt(1.2) + for panel in program["panels"]: + if panel["id"] == "shared_resource_library": + continue + cx, _cy = _bbox_center(panel["bbox_percent"], width_in, height_in) + if not (sx + 0.1 <= cx <= sx + sw - 0.1): + continue + down = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(cx), Inches(bus_y), Inches(cx), Inches(sy)) + down.line.color.rgb = _rgb("#1F6F8B") + down.line.width = Pt(1.0) + if not has_text_program: + _add_label(slide, "shared resources", sx + 0.2, bus_y - 0.22, 1.5, 0.18, font_size=7, align=PP_ALIGN.LEFT) + + if program.get("show_visible_title"): + title = program.get("paper_brief", {}).get("title_guess") or "Research System Figure" + _add_label(slide, title[:90], width_in - 5.2, height_in - 0.50, 4.9, 0.38, font_size=7, bold=True, align=PP_ALIGN.RIGHT) + + pptx_path = out / "editable_composition.pptx" + prs.save(pptx_path) + write_json(out / "composition_quality_report.json", { + "summary": "PPT composition quality report checking frameless slot insertion, no extra white tiles, and image area fill inside each reference slot.", + "policy": { + "slot_frame_policy": "frameless_slot", + "picture_fill_policy": "direct_full_slot_contain_no_tile", + "min_image_slot_area_fill_percent": 95, + "caption_inside_image_slot_allowed": False, + }, + "slots": composition_items, + "cards": rendered_cards, + "arrows": rendered_arrows, + "text": rendered_text_items, + }) + return pptx_path diff --git a/rfs/composition/preview.py b/rfs/composition/preview.py new file mode 100644 index 0000000..c2ecef9 --- /dev/null +++ b/rfs/composition/preview.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from pathlib import Path + +from PIL import Image, ImageColor, ImageDraw, ImageFont + + +def _hex_color(value: object, fallback: str = "#000000") -> str: + if not isinstance(value, str): + return fallback + text = value.strip() + if not text: + return fallback + if not text.startswith("#"): + text = "#" + text + try: + ImageColor.getrgb(text) + return text + except ValueError: + return fallback + + +def _bbox_px(bbox: dict, width: int, height: int) -> tuple[int, int, int, int]: + x = int(round(float(bbox.get("x", 0)) * width)) + y = int(round(float(bbox.get("y", 0)) * height)) + w = int(round(float(bbox.get("w", 0)) * width)) + h = int(round(float(bbox.get("h", 0)) * height)) + return x, y, max(1, w), max(1, h) + + +def _font(size_px: int, bold: bool = False) -> ImageFont.ImageFont: + candidates = [ + "arialbd.ttf" if bold else "arial.ttf", + "calibrib.ttf" if bold else "calibri.ttf", + "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", + ] + for name in candidates: + try: + return ImageFont.truetype(name, max(4, int(size_px))) + except OSError: + continue + return ImageFont.load_default() + + +def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: int) -> None: + if not item.get("visible", True): + return + if str(item.get("layer_ownership") or "editable_text_layer") != "editable_text_layer": + return + bbox = item.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + text = str(item.get("text") or "") + if not text: + return + font_size_pt = float(item.get("font_size_pt") or 8) + size_px = max(4, int(round(font_size_pt * canvas_h / 387.0))) + font = _font(size_px, bold=bool(item.get("bold"))) + color = _hex_color(item.get("color_hex"), "#263747") + align = str(item.get("align") or "center").lower() + text_bbox = draw.textbbox((0, 0), text, font=font) + text_w = max(1, text_bbox[2] - text_bbox[0]) + text_h = max(1, text_bbox[3] - text_bbox[1]) + if align == "left": + tx = x + elif align == "right": + tx = x + w - text_w + else: + tx = x + (w - text_w) / 2 + ty = y + (h - text_h) / 2 + draw.text((tx, ty), text, fill=color, font=font) + + +def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: int) -> None: + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + return + points: list[tuple[int, int]] = [] + for point in path: + if isinstance(point, list) and len(point) >= 2: + points.append((int(round(float(point[0]) * canvas_w)), int(round(float(point[1]) * canvas_h)))) + if len(points) < 2: + return + color = _hex_color(arrow.get("stroke_color") or arrow.get("outline_color"), "#3F5063") + line_width = max(1, int(round(float(arrow.get("stroke_width_pt") or 1.5) * canvas_h / 277.0))) + render_style = str(arrow.get("render_style") or "").lower() + if render_style == "filled_block_arrow": + fill = _hex_color(arrow.get("fill_color"), "#AFC6DE") + start, end = points[0], points[-1] + thickness = max(8, int(round(float(arrow.get("block_arrow_thickness_percent") or 0.052) * canvas_h))) + if abs(end[0] - start[0]) >= abs(end[1] - start[1]): + y = int(round((start[1] + end[1]) / 2)) + left, right = sorted([start[0], end[0]]) + draw.rounded_rectangle((left, y - thickness // 2, right, y + thickness // 2), radius=thickness // 3, fill=fill, outline=color, width=line_width) + else: + x = int(round((start[0] + end[0]) / 2)) + top, bottom = sorted([start[1], end[1]]) + draw.rounded_rectangle((x - thickness // 2, top, x + thickness // 2, bottom), radius=thickness // 3, fill=fill, outline=color, width=line_width) + return + draw.line(points, fill=color, width=line_width, joint="curve") + + +def _draw_card(draw: ImageDraw.ImageDraw, card: dict, canvas_w: int, canvas_h: int) -> None: + bbox = card.get("bbox_percent") + if not isinstance(bbox, dict): + return + x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) + stroke = _hex_color(card.get("stroke_color"), "#59AFCB") + fill = None if float(card.get("fill_transparency", 1.0)) >= 1.0 else _hex_color(card.get("fill_color"), "#FFFFFF") + line_width = max(1, int(round(float(card.get("stroke_width_pt") or 1.5) * canvas_h / 360.0))) + radius = max(0, int(round(float(card.get("corner_radius") or 0.08) * min(w, h)))) + rect = (x, y, x + w, y + h) + if str(card.get("shape_kind") or "rounded_rect").lower() == "rect": + draw.rectangle(rect, fill=fill, outline=stroke, width=line_width) + else: + draw.rounded_rectangle(rect, radius=radius, fill=fill, outline=stroke, width=line_width) + + +def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str | Path | None = None) -> Path: + out = Path(out_dir) + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + width = int(canvas.get("width_px") or 1600) + height = int(canvas.get("height_px") or 900) + background = _hex_color(canvas.get("background"), "#FFFFFF") + image = Image.new("RGB", (width, height), background) + draw = ImageDraw.Draw(image) + + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} + palette = style.get("palette") or ["#F7F8F7", "#D2DCDE", "#A09596"] + + for idx, panel in enumerate(program.get("panels", [])): + bbox = panel.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + local_style = panel_styles.get(panel.get("id"), {}) if isinstance(panel_styles.get(panel.get("id")), dict) else {} + fill = _hex_color(local_style.get("fill_color"), palette[idx % len(palette)]) + stroke = _hex_color(local_style.get("stroke_color"), "#FFFFFF") + draw.rounded_rectangle((x, y, x + w, y + h), radius=max(2, min(12, h // 18)), fill=fill, outline=stroke, width=2) + + for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index") or 12)): + if isinstance(card, dict): + _draw_card(draw, card, width, height) + + asset_by_id = {str(asset.get("id")): asset for asset in program.get("assets", []) if isinstance(asset, dict)} + for slot in sorted(program.get("slots", []), key=lambda item: int(item.get("z_index") or 20)): + bbox = slot.get("bbox_percent") + if not isinstance(bbox, dict): + continue + x, y, w, h = _bbox_px(bbox, width, height) + asset = asset_by_id.get(str(slot.get("asset_id") or slot.get("id"))) + asset_path = out / str(asset.get("path")) if asset else out / "assets" / f"{slot.get('asset_id')}.png" + if asset_path.exists(): + with Image.open(asset_path) as asset_img: + asset_img = asset_img.convert("RGB") + asset_img.thumbnail((w, h), Image.Resampling.LANCZOS) + left = x + (w - asset_img.width) // 2 + top = y + (h - asset_img.height) // 2 + image.paste(asset_img, (left, top)) + + for arrow in program.get("arrows", []): + if isinstance(arrow, dict): + _draw_arrow(draw, arrow, width, height) + + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} + text_items = text_program.get("items", []) if isinstance(text_program, dict) else [] + visible_items = [item for item in text_items if isinstance(item, dict)] + for item in sorted(visible_items, key=lambda item: (float(item.get("z_index") or 80), str(item.get("id") or ""))): + _draw_text(draw, item, width, height) + + target = Path(preview_path) if preview_path else out / "rebuild_preview.png" + target.parent.mkdir(parents=True, exist_ok=True) + image.save(target) + return target diff --git a/rfs/contracts/__init__.py b/rfs/contracts/__init__.py new file mode 100644 index 0000000..7c8325d --- /dev/null +++ b/rfs/contracts/__init__.py @@ -0,0 +1,5 @@ +"""Stable data-contract APIs shared across ResearchFigureStudio workflows.""" + +from .semantic import apply_paper_semantic_contract, load_paper_semantic_contract + +__all__ = ["apply_paper_semantic_contract", "load_paper_semantic_contract"] diff --git a/rfs/contracts/semantic.py b/rfs/contracts/semantic.py new file mode 100644 index 0000000..fbd973e --- /dev/null +++ b/rfs/contracts/semantic.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +from ..utils import read_json, write_json + + +def load_paper_semantic_contract(run_dir: str | Path) -> dict[str, Any]: + root = Path(run_dir) + return { + "summary": "Paper-grounded semantic contract for editable figure reconstruction.", + "source_run_dir": str(root.resolve()), + "paper_review": read_json(root / "paper_review.json"), + "figure_specification": read_json(root / "figure_specification.json"), + "paper_summary": read_json(root / "paper_summary.json"), + "design_plan": read_json(root / "design_plan.json"), + "layout_intent": read_json(root / "layout_intent.json"), + "style_plan": read_json(root / "style_plan.json"), + } + + +def _text(item: dict, fallback: str = "") -> str: + for key in ("visible_label", "name", "text", "statement", "label", "id"): + value = str(item.get(key) or "").strip() + if value: + return value + return fallback + + +def _entities(contract: dict) -> list[dict]: + spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + entities: list[dict] = [] + for field, role in (("inputs", "input"), ("modules", "module"), ("outputs", "output"), ("innovations", "innovation")): + values = spec.get(field) if isinstance(spec.get(field), list) else [] + for index, raw in enumerate(values, start=1): + if not isinstance(raw, dict): + continue + label = _text(raw) + if not label: + continue + entity_id = str(raw.get("id") or raw.get("name") or f"{role}_{index:02d}").strip() + entities.append({ + "id": entity_id, + "label": label, + "role": role, + "evidence_ids": list(raw.get("evidence_ids") or []), + "source": raw, + }) + return entities + + +def _tokens(value: object) -> set[str]: + return {token for token in re.findall(r"[a-z0-9]+", str(value or "").lower()) if len(token) > 1} + + +def _object_text(obj: dict) -> str: + return " ".join(str(obj.get(key) or "") for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt")) + + +def _center(obj: dict) -> tuple[float, float]: + box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + return float(box.get("x") or 0) + float(box.get("w") or 0) / 2, float(box.get("y") or 0) + float(box.get("h") or 0) / 2 + + +def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dict], list[dict]]: + objects = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + if len(objects) < len(entities): + seen = {str(item.get("id")) for item in objects} + objects.extend( + item for item in program.get("cards", []) + program.get("panels", []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and str(item.get("id")) not in seen + ) + objects.sort(key=lambda item: (_center(item)[0], _center(item)[1], str(item.get("id") or ""))) + available = list(objects) + mapping: dict[str, dict] = {} + records: list[dict] = [] + for entity in entities: + entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) + scored = [] + for obj in available: + obj_tokens = _tokens(_object_text(obj)) + score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) + scored.append((score, -_center(obj)[0], obj)) + if not scored: + records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) + continue + scored.sort(key=lambda value: (value[0], value[1]), reverse=True) + chosen = scored[0][2] if scored[0][0] > 0 else available[0] + available.remove(chosen) + mapping[entity["id"]] = chosen + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "object_id": chosen.get("id"), + "match_score": round(max(0.0, scored[0][0]), 4), + "status": "mapped", + }) + return mapping, records + + +def _inside(inner: dict, outer: dict) -> bool: + cx = float(inner.get("x") or 0) + float(inner.get("w") or 0) / 2 + cy = float(inner.get("y") or 0) + float(inner.get("h") or 0) / 2 + return float(outer.get("x") or 0) <= cx <= float(outer.get("x") or 0) + float(outer.get("w") or 0) and float(outer.get("y") or 0) <= cy <= float(outer.get("y") or 0) + float(outer.get("h") or 0) + + +def _label_bbox(obj: dict) -> dict[str, float]: + box = obj["bbox_percent"] + x, y, w, h = (float(box[key]) for key in ("x", "y", "w", "h")) + label_h = min(0.055, max(0.026, h * 0.18)) + return {"x": round(x, 4), "y": round(max(0.0, y), 4), "w": round(w, 4), "h": round(label_h, 4)} + + +def _anchor(source: dict, target: dict) -> list[list[float]]: + sx, sy = _center(source) + tx, ty = _center(target) + sb, tb = source["bbox_percent"], target["bbox_percent"] + if abs(tx - sx) >= abs(ty - sy): + start = [float(sb["x"]) + (float(sb["w"]) if tx >= sx else 0.0), sy] + end = [float(tb["x"]) + (0.0 if tx >= sx else float(tb["w"])), ty] + mid = (start[0] + end[0]) / 2 + points = [start, [mid, start[1]], [mid, end[1]], end] + else: + start = [sx, float(sb["y"]) + (float(sb["h"]) if ty >= sy else 0.0)] + end = [tx, float(tb["y"]) + (0.0 if ty >= sy else float(tb["h"]))] + mid = (start[1] + end[1]) / 2 + points = [start, [start[0], mid], [end[0], mid], end] + clean: list[list[float]] = [] + for point in points: + rounded = [round(max(0.0, min(1.0, point[0])), 4), round(max(0.0, min(1.0, point[1])), 4)] + if not clean or clean[-1] != rounded: + clean.append(rounded) + return clean + + +def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: + entities = _entities(contract) + mapping, mapping_records = _assign_entities(entities, program) + mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] + text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} + existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] + preserved = [item for item in existing if not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes)] + semantic_text = [] + for entity in entities: + obj = mapping.get(entity["id"]) + if not obj: + continue + box = _label_bbox(obj) + semantic_text.append({ + "id": f"semantic_text_{entity['id']}", + "text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_evidence_ids": entity["evidence_ids"], + "bbox_percent": box, + "center_percent": {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)}, + "width_percent": box["w"], + "height_percent": box["h"], + "font_size_pt": 10.0, + "color_hex": "#263747", + "bold": entity["role"] in {"module", "innovation"}, + "align": "center", + "font_family_guess": "Arial", + "fit_strategy": "paper_semantic_contract_exact_label", + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + text_program = { + **text_program, + "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", + "semantic_authority": "paper_ground_truth", + "items": preserved + semantic_text, + } + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + + spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + semantic_arrows = [] + skipped_relations = [] + for index, relation in enumerate(spec.get("relations") if isinstance(spec.get("relations"), list) else [], start=1): + if not isinstance(relation, dict): + continue + source_id = str(relation.get("source") or relation.get("source_id") or "") + target_id = str(relation.get("target") or relation.get("target_id") or "") + source, target = mapping.get(source_id), mapping.get(target_id) + if not source or not target: + skipped_relations.append({"source": source_id, "target": target_id, "reason": "unmapped_endpoint"}) + continue + semantic_arrows.append({ + "id": str(relation.get("id") or f"semantic_relation_{index:02d}"), + "source_id": str(source.get("id")), + "target_id": str(target.get("id")), + "semantic_source_id": source_id, + "semantic_target_id": target_id, + "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), + "label": str(relation.get("label") or relation.get("statement") or ""), + "evidence_ids": list(relation.get("evidence_ids") or []), + "path_percent": _anchor(source, target), + "render_style": "elbow_connector", + "route_intent": "paper_semantic_relation", + "stroke_color": "#3F5063", + "stroke_width_pt": 1.8, + "arrowhead": "triangle", + "editable_in": "pptx", + }) + if semantic_arrows: + program["arrows"] = semantic_arrows + + report = { + "summary": "Paper semantic contract application report.", + "status": "pass" if len(mapping) == len(entities) and not skipped_relations else "warning", + "semantic_authority": "paper_ground_truth", + "visual_authority": "generated_reference_image", + "entity_count": len(entities), + "mapped_entity_count": len(mapping), + "relation_count": len(spec.get("relations") or []), + "mapped_relation_count": len(semantic_arrows), + "mappings": mapping_records, + "skipped_relations": skipped_relations, + "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", + } + out = Path(out_dir) + write_json(out / "paper_semantic_contract.json", contract) + write_json(out / "semantic_binding_report.json", report) + write_json(out / "text_program.json", text_program) + return program, report diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 04f523d..05aa942 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -12,15 +12,14 @@ from PIL import Image, ImageChops, ImageDraw +from .composition import compile_ppt, render_rebuild_preview +from .contracts import apply_paper_semantic_contract from .control_localizer import localize_reference_controls +from .evaluation import run_rebuild_visual_quality_check from .layout_planner import dominant_palette, estimate_background, plan_reference_layout from .layout_semantic_planner import plan_slot_semantics -from .ppt_compiler import compile_ppt from .rebuild_design_planner import plan_rebuild_design -from .rebuild_preview_renderer import render_rebuild_preview from .rebuild_vlm_validation import build_rebuild_vlm_validation_report -from .rebuild_visual_critic import run_rebuild_visual_quality_check -from .semantic_contract import apply_paper_semantic_contract from .text_layer import build_text_layer from .utils import ensure_dir, read_json, write_json, write_text diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py new file mode 100644 index 0000000..13488ab --- /dev/null +++ b/rfs/evaluation/__init__.py @@ -0,0 +1,5 @@ +"""Deterministic and model-assisted quality evaluation APIs.""" + +from .rebuild_visual import run_rebuild_visual_quality_check + +__all__ = ["run_rebuild_visual_quality_check"] diff --git a/rfs/evaluation/rebuild_visual.py b/rfs/evaluation/rebuild_visual.py new file mode 100644 index 0000000..1b10818 --- /dev/null +++ b/rfs/evaluation/rebuild_visual.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..utils import write_json + + +def _bbox(item: dict) -> dict | None: + box = item.get("bbox_percent") + return box if isinstance(box, dict) else None + + +def _valid_bbox(box: dict | None) -> bool: + if not isinstance(box, dict): + return False + try: + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + except Exception: + return False + return 0 <= x <= 1 and 0 <= y <= 1 and w > 0 and h > 0 and x + w <= 1.0001 and y + h <= 1.0001 + + +def _overlap_area(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + return ix * iy + + +def _overlap_ratio(a: dict, b: dict) -> float: + area = _overlap_area(a, b) + if area <= 0: + return 0.0 + a_area = float(a["w"]) * float(a["h"]) + b_area = float(b["w"]) * float(b["h"]) + return area / max(min(a_area, b_area), 0.000001) + + +def _visible_text_items(program: dict) -> list[dict]: + text_program = program.get("text_program") + if not isinstance(text_program, dict): + return [] + return [item for item in text_program.get("items", []) if isinstance(item, dict) and item.get("visible", True)] + + +def _text_alignment_group_key(item: dict) -> tuple[str, str] | None: + for key in ("paragraph_id", "text_group_id", "group_id", "source_group_id"): + value = str(item.get(key) or "").strip() + if value: + return (key, value) + return None + + +def _detect_text_issues(program: dict) -> list[dict]: + issues = [] + items = _visible_text_items(program) + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": "text_bbox_out_of_bounds", "text_id": item.get("id"), "reason": "text bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > 0.18: + issues.append({ + "type": "text_overlap", + "text_id": left.get("id"), + "other_text_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": "visible text boxes overlap substantially", + }) + grouped: dict[tuple[str, str], list[dict]] = {} + for item in items: + key = _text_alignment_group_key(item) + if key: + grouped.setdefault(key, []).append(item) + for (group_key, group_id), group in grouped.items(): + if len(group) < 3: + continue + centers = [ + float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2 + for item in group + if _valid_bbox(_bbox(item)) + ] + if len(centers) >= 3 and max(centers) - min(centers) > 0.035: + issues.append({ + "type": "text_group_misaligned", + "group_key": group_key, + "group_id": group_id, + "text_ids": [item.get("id") for item in group], + "center_y_span": round(max(centers) - min(centers), 4), + "reason": "explicit text group is visually uneven", + }) + return issues + + +def _detect_object_issues(program: dict) -> list[dict]: + issues = [] + for collection_name in ("panels", "slots"): + items = [item for item in program.get(collection_name, []) if isinstance(item, dict)] + for item in items: + if not _valid_bbox(_bbox(item)): + issues.append({"type": f"{collection_name[:-1]}_bbox_out_of_bounds", "id": item.get("id"), "reason": "bbox is missing or outside canvas"}) + for index, left in enumerate(items): + lbox = _bbox(left) + if not _valid_bbox(lbox): + continue + for right in items[index + 1:]: + rbox = _bbox(right) + if not _valid_bbox(rbox): + continue + ratio = _overlap_ratio(lbox, rbox) + if ratio > (0.35 if collection_name == "slots" else 0.18): + issues.append({ + "type": f"{collection_name[:-1]}_overlap", + "id": left.get("id"), + "other_id": right.get("id"), + "overlap_ratio": round(ratio, 4), + "reason": f"{collection_name[:-1]} boxes overlap substantially", + }) + return issues + + +def _detect_arrow_issues(program: dict) -> list[dict]: + issues = [] + for arrow in program.get("arrows", []) or []: + if not isinstance(arrow, dict): + continue + path = arrow.get("path_percent") + if not isinstance(path, list) or len(path) < 2: + issues.append({"type": "arrow_missing_path", "arrow_id": arrow.get("id"), "reason": "arrow has no usable path_percent"}) + return issues + + +def _detect_ownership_issues(program: dict, ownership_report: dict | None) -> list[dict]: + issues = [] + text_ids = {str(item.get("source_reference_text_id") or item.get("id") or "") for item in _visible_text_items(program)} + report = ownership_report if isinstance(ownership_report, dict) else {} + for item in report.get("items", []) or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or "") + ownership = str(item.get("layer_ownership") or "") + included = bool(item.get("included_in_text_program")) + if ownership != "editable_text_layer" and text_id in text_ids: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "non-editable-owned text is present in text_program"}) + if ownership == "editable_text_layer" and not included: + issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "editable-owned text is missing from text_program"}) + return issues + + +def run_rebuild_visual_quality_check( + out_dir: str | Path, + program: dict, + reference_geometry: dict | None = None, + reference_controls: dict | None = None, + ownership_report: dict | None = None, + mode: str = "heuristic", +) -> dict[str, Any]: + text_issues = _detect_text_issues(program) + object_issues = _detect_object_issues(program) + arrow_issues = _detect_arrow_issues(program) + ownership_issues = _detect_ownership_issues(program, ownership_report) + issues = text_issues + object_issues + arrow_issues + ownership_issues + blocking_types = {"text_overlap", "text_bbox_out_of_bounds", "arrow_missing_path", "text_layer_ownership_conflict"} + blocking = [item for item in issues if item.get("type") in blocking_types] + report = { + "summary": "Deterministic rebuild visual quality report.", + "mode": mode, + "status": "blocked" if blocking else ("warning" if issues else "pass"), + "issue_count": len(issues), + "blocking_issue_count": len(blocking), + "text_issue_count": len(text_issues), + "object_issue_count": len(object_issues), + "arrow_issue_count": len(arrow_issues), + "ownership_issue_count": len(ownership_issues), + "issues": issues, + "reference_geometry_status": (reference_geometry or {}).get("status"), + "reference_controls_status": (reference_controls or {}).get("status"), + "policy": "deterministic check only; no program mutation", + } + write_json(Path(out_dir) / "rebuild_visual_quality_report.json", report) + return report diff --git a/rfs/generation/__init__.py b/rfs/generation/__init__.py new file mode 100644 index 0000000..d3a97a5 --- /dev/null +++ b/rfs/generation/__init__.py @@ -0,0 +1,6 @@ +"""Raster reference and slot-asset generation entry points.""" + +from ..asset_generator import generate_assets +from ..paper_to_image.generator import generate_and_select + +__all__ = ["generate_and_select", "generate_assets"] diff --git a/rfs/paper_to_editable.py b/rfs/paper_to_editable.py index 2d34caa..26d0c68 100644 --- a/rfs/paper_to_editable.py +++ b/rfs/paper_to_editable.py @@ -1,124 +1,8 @@ -from __future__ import annotations +"""Compatibility import for the paper-to-editable workflow. -from pathlib import Path -from typing import Any +New code should import from :mod:`rfs.workflows`. +""" -from .editable_rebuild import rebuild_editable -from .paper_to_image import run_paper_to_image -from .rebuild_vlm_adapters import build_rebuild_vlm_adapters -from .semantic_contract import load_paper_semantic_contract -from .utils import ensure_dir, write_json +from .workflows.paper_to_editable import run_paper_to_editable - -def run_paper_to_editable( - paper: str | Path, - out: str | Path, - preferences_path: str | Path | None = None, - positive_references: list[str] | None = None, - negative_references: list[str] | None = None, - planner_mode: str = "vlm", - planner_model: str | None = None, - image_asset_mode: str = "image2", - image_candidates: int = 3, - aspect_ratio: str | None = None, - language: str | None = None, - image_model: str | None = None, - image_retries: int = 2, - review_mode: str = "vlm", - review_model: str | None = None, - domain_profile: str = "auto", - template: str = "auto", - repair_rounds: int = 1, - rebuild_asset_mode: str = "api", - rebuild_asset_policy: str = "smart-api", - layout_mode: str = "hybrid", - control_mode: str = "hybrid", - text_mode: str = "ocr", - design_plan_mode: str = "vlm", - export_preview: bool = True, - allow_engineering_preview: bool = False, - ocr_engine: str = "auto", - ocr_lang: str = "en_ch", - critic_adapter=None, -) -> dict[str, Any]: - root = ensure_dir(out).resolve() - image_dir = root / "paper_to_image" - editable_dir = root / "editable" - image_result = run_paper_to_image( - paper=paper, - out=image_dir, - preferences_path=preferences_path, - positive_references=positive_references, - negative_references=negative_references, - planner_mode=planner_mode, - planner_model=planner_model, - asset_mode=image_asset_mode, - candidates=image_candidates, - aspect_ratio=aspect_ratio, - language=language, - image_model=image_model, - image_retries=image_retries, - review_mode=review_mode, - review_model=review_model, - domain_profile=domain_profile, - template=template, - repair_rounds=repair_rounds, - ocr_engine=ocr_engine, - ocr_lang=ocr_lang, - critic_adapter=critic_adapter, - ) - reference = image_result.get("selected_image") - reference_status = "production_selected_image" - if not reference and allow_engineering_preview: - reference = image_result.get("engineering_preview") - reference_status = "engineering_preview_explicitly_allowed" - if not reference: - blocked = { - "summary": "Paper-to-editable delivery blocked before rebuild because no production-approved image exists.", - "ok": False, - "delivery_blocked": True, - "out_dir": str(root), - "paper_to_image": image_result, - "policy": "failed image candidates are never promoted to production editable output", - } - write_json(root / "paper_to_editable_result.json", blocked) - return blocked - - contract = load_paper_semantic_contract(image_dir) - adapters = build_rebuild_vlm_adapters(editable_dir) - rebuild_ocr_engine = ocr_engine if ocr_engine in {"paddle", "easyocr", "off"} else "paddle" - editable_result = rebuild_editable( - reference=reference, - out=editable_dir, - asset_mode=rebuild_asset_mode, - asset_policy=rebuild_asset_policy, - text_mode=text_mode, - layout_mode=layout_mode, - control_mode=control_mode, - export_preview=export_preview, - ocr_engine=rebuild_ocr_engine, - ocr_lang=ocr_lang, - vlm_layout_adapter=adapters["layout"] if layout_mode in {"vlm", "hybrid"} else None, - control_adapter=adapters["control"] if control_mode in {"vlm", "hybrid"} else None, - semantic_adapter=adapters["semantic"] if layout_mode in {"vlm", "hybrid"} or control_mode in {"vlm", "hybrid"} else None, - design_plan_mode=design_plan_mode, - design_adapter=adapters["design"] if design_plan_mode == "vlm" else None, - semantic_contract=contract, - ) - result = { - "summary": "Paper-to-editable workflow completed with paper-grounded semantic bindings.", - "ok": bool(editable_result.get("ok")), - "delivery_blocked": False, - "out_dir": str(root), - "reference_status": reference_status, - "reference_image": str(reference), - "paper_to_image_dir": str(image_dir), - "editable_dir": str(editable_dir), - "pptx": editable_result.get("pptx"), - "preview": editable_result.get("preview"), - "semantic_binding_status": editable_result.get("semantic_binding_status"), - "paper_to_image": image_result, - "editable_rebuild": editable_result, - } - write_json(root / "paper_to_editable_result.json", result) - return result +__all__ = ["run_paper_to_editable"] diff --git a/rfs/planning/__init__.py b/rfs/planning/__init__.py new file mode 100644 index 0000000..643fe0f --- /dev/null +++ b/rfs/planning/__init__.py @@ -0,0 +1,6 @@ +"""Paper, design, and image planning entry points.""" + +from ..paper_to_image.planner import plan_paper_image, validate_plan_grounding +from ..rebuild_design_planner import plan_rebuild_design + +__all__ = ["plan_paper_image", "plan_rebuild_design", "validate_plan_grounding"] diff --git a/rfs/ppt_compiler.py b/rfs/ppt_compiler.py index 5436329..a029971 100644 --- a/rfs/ppt_compiler.py +++ b/rfs/ppt_compiler.py @@ -1,590 +1,6 @@ -from __future__ import annotations +"""Compatibility imports for PowerPoint composition. -from pathlib import Path +New code should import from :mod:`rfs.composition`. +""" -from PIL import Image -from pptx import Presentation -from pptx.dml.color import RGBColor -from pptx.enum.dml import MSO_LINE_DASH_STYLE -from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE -from pptx.enum.text import PP_ALIGN, MSO_ANCHOR -from pptx.oxml import parse_xml -from pptx.util import Inches, Pt - -from .utils import pct_to_inches, write_json - - -def _rgb(hex_color: str) -> RGBColor: - value = hex_color.strip().lstrip("#") or "000000" - return RGBColor(int(value[:2], 16), int(value[2:4], 16), int(value[4:6], 16)) - - -def _apply_arrow(connector, size: str = "sm") -> None: - ln = connector.line._get_or_add_ln() - ln.append(parse_xml(f'')) - - -def _apply_line_cap(connector, cap: str = "round") -> None: - value = {"round": "rnd", "square": "sq"}.get(str(cap).lower()) - if not value: - return - ln = connector.line._get_or_add_ln() - ln.set("cap", value) - - -def _connector_type_for_route(route_style: str): - if str(route_style).lower() in {"soft_curve", "dashed_loop", "dashed_spline_like"}: - return MSO_CONNECTOR.CURVE - return MSO_CONNECTOR.STRAIGHT - - -def _set_text(shape, text: str, font_size: float = 10, bold: bool = False, color: str = "#163B4D", align=PP_ALIGN.CENTER, font_family: str | None = None) -> None: - tf = shape.text_frame - tf.clear() - tf.word_wrap = True - tf.vertical_anchor = MSO_ANCHOR.MIDDLE - p = tf.paragraphs[0] - p.alignment = align - run = p.add_run() - run.text = text - run.font.size = Pt(float(font_size)) - run.font.bold = bold - if font_family: - run.font.name = str(font_family) - run.font.color.rgb = _rgb(color) - - -def _add_round_rect(slide, x: float, y: float, w: float, h: float, fill: str, stroke: str, width_pt: float = 1.2): - shape = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x), Inches(y), Inches(w), Inches(h)) - shape.fill.solid() - shape.fill.fore_color.rgb = _rgb(fill) - shape.line.color.rgb = _rgb(stroke) - shape.line.width = Pt(width_pt) - shape.shadow.inherit = False - return shape - - -def _add_card_frame(slide, card: dict, width_in: float, height_in: float): - x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) - shape_type = MSO_SHAPE.RECTANGLE if str(card.get("shape_kind") or "rounded_rect").lower() == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE - shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) - if float(card.get("fill_transparency", 1.0)) >= 1.0: - shape.fill.background() - else: - shape.fill.solid() - shape.fill.fore_color.rgb = _rgb(str(card.get("fill_color") or "#FFFFFF")) - shape.line.color.rgb = _rgb(str(card.get("stroke_color") or "#59AFCB")) - shape.line.width = Pt(float(card.get("stroke_width_pt") or 1.5)) - dash_style = str(card.get("dash_style") or "solid").lower() - if dash_style in {"dash", "dashed"}: - shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH - elif dash_style in {"dot", "dotted"}: - shape.line.dash_style = getattr(MSO_LINE_DASH_STYLE, "ROUND_DOT", MSO_LINE_DASH_STYLE.DASH) - shape.shadow.inherit = False - return shape - - -def _add_label(slide, text: str, x: float, y: float, w: float, h: float, font_size: float = 9, bold: bool = False, align=PP_ALIGN.CENTER, color: str = "#163B4D", font_family: str | None = None): - box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) - _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family) - return box - - -def _add_title_block(slide, program: dict, width_in: float, height_in: float) -> None: - title_block = program.get("title_block") - if not isinstance(title_block, dict): - return - title = str(title_block.get("title", "")).strip() - subtitle = str(title_block.get("subtitle", "")).strip() - bbox = title_block.get("bbox_percent") - if not title or not isinstance(bbox, dict): - return - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - title_h = min(h * 0.58, 0.42) - subtitle_h = max(0.18, min(h - title_h, 0.26)) - _add_label( - slide, - title, - x, - y, - w, - title_h, - font_size=int(title_block.get("title_font_size", 24)), - bold=True, - align=PP_ALIGN.LEFT, - ) - if subtitle: - subtitle_box = _add_label( - slide, - subtitle, - x, - y + title_h * 0.94, - w, - subtitle_h, - font_size=int(title_block.get("subtitle_font_size", 11)), - bold=False, - align=PP_ALIGN.LEFT, - ) - for paragraph in subtitle_box.text_frame.paragraphs: - for run in paragraph.runs: - run.font.color.rgb = _rgb(str(title_block.get("subtitle_color", "#333333"))) - - -def _short_caption(text: str) -> str: - replacements = { - "participant seated before 49-inch screen": "Participant setup", - "3D virtual interviewer on screen": "Virtual interviewer", - "wide-angle camera capturing full body": "Full-body camera", - "virtual interview room setup": "Interview room", - "interview question prompt on screen": "Question prompt", - "raw interview video recording": "Raw video", - "287 participant video collection": "287 participants", - "36 interview questions": "36 questions", - "full-body video thumbnails": "Full-body clips", - "participant clip grid": "Clip grid", - "FFmpeg extracts audio from video": "FFmpeg audio", - "audio waveform stream": "Audio stream", - "FunASR speech recognition": "FunASR ASR", - "spoken text with timestamps": "Text + timestamps", - "MTCNN face detection": "MTCNN face", - "face clips": "Face clips", - "AlphaPose full-body skeleton extraction": "AlphaPose pose", - "full-body pose skeleton stream": "Pose stream", - "frame sampling from video": "Frame sampling", - "sampled video frames": "Video frames", - "timestamp alignment clock": "Alignment clock", - "aligned audio and text streams": "Audio + text", - "aligned face and pose streams": "Face + pose", - "aligned frame stream": "Frame stream", - "face modality": "Face", - "frame modality": "Frame", - "pose modality": "Pose", - "audio modality": "Audio", - "text modality": "Text", - "NEO-FFI-3 questionnaire": "NEO-FFI-3", - "OCEAN score vector": "OCEAN scores", - "Openness score": "Openness", - "Conscientiousness score": "Conscientiousness", - "Extraversion score": "Extraversion", - "Agreeableness score": "Agreeableness", - "Neuroticism score": "Neuroticism", - } - if text in replacements: - return replacements[text] - if len(text) <= 22: - return text - words = text.replace("/", " ").split() - return " ".join(words[:3]) if len(words) > 3 else text[:22] - - -def _picture_contain_box(image_path: Path, x: float, y: float, w: float, h: float) -> tuple[float, float, float, float, float]: - with Image.open(image_path) as img: - iw, ih = img.size - image_ratio = iw / max(ih, 1) - slot_ratio = w / max(h, 0.001) - if image_ratio > slot_ratio: - fit_w = w - fit_h = w / image_ratio - else: - fit_h = h - fit_w = h * image_ratio - left = x + (w - fit_w) / 2 - top = y + (h - fit_h) / 2 - fill_percent = fit_w * fit_h / max(w * h, 0.001) * 100 - return left, top, fit_w, fit_h, fill_percent - - -def _add_picture_contain(slide, image_path: Path, x: float, y: float, w: float, h: float): - left, top, fit_w, fit_h, _fill_percent = _picture_contain_box(image_path, x, y, w, h) - return slide.shapes.add_picture(str(image_path), Inches(left), Inches(top), width=Inches(fit_w), height=Inches(fit_h)) - - -def _panel_map(program: dict) -> dict[str, dict]: - return {panel["id"]: panel for panel in program.get("panels", [])} - - -def _text_program_items(program: dict) -> list[dict]: - text_program = program.get("text_program") - if not isinstance(text_program, dict): - return [] - items = text_program.get("items", []) - return [item for item in items if isinstance(item, dict) and item.get("visible", True)] - - -def _text_item_for_target(program: dict, target_id: str, role: str) -> dict | None: - for item in _text_program_items(program): - if str(item.get("target_id")) == target_id and str(item.get("role")) == role: - return item - return None - - -def _align_from_text_item(item: dict): - value = str(item.get("align") or "").lower() - if value == "left": - return PP_ALIGN.LEFT - if value == "right": - return PP_ALIGN.RIGHT - return PP_ALIGN.CENTER - - -def _object_map(program: dict) -> dict[str, dict]: - objects = {panel["id"]: panel for panel in program.get("panels", [])} - objects.update({slot["id"]: slot for slot in program.get("slots", [])}) - return objects - - -def _bbox_center(bbox: dict, canvas_w: float, canvas_h: float) -> tuple[float, float]: - x, y, w, h = pct_to_inches(bbox, canvas_w, canvas_h) - return x + w / 2, y + h / 2 - - -def _arrow_endpoints(source: dict, target: dict, canvas_w: float, canvas_h: float) -> tuple[float, float, float, float]: - sbox = source["bbox_percent"] - tbox = target["bbox_percent"] - sx, sy = _bbox_center(sbox, canvas_w, canvas_h) - tx, ty = _bbox_center(tbox, canvas_w, canvas_h) - s_left, s_top, s_w, s_h = pct_to_inches(sbox, canvas_w, canvas_h) - t_left, t_top, t_w, t_h = pct_to_inches(tbox, canvas_w, canvas_h) - - dx = tx - sx - dy = ty - sy - if abs(dx) >= abs(dy): - if dx >= 0: - return s_left + s_w + 0.03, sy, t_left - 0.03, ty - return s_left - 0.03, sy, t_left + t_w + 0.03, ty - if dy >= 0: - return sx, s_top + s_h + 0.03, tx, t_top - 0.03 - return sx, s_top - 0.03, tx, t_top + t_h + 0.03 - - -def _arrow_points_from_path(path: list, width_in: float, height_in: float) -> list[tuple[float, float]]: - points = [] - for point in path: - if isinstance(point, list) and len(point) >= 2: - points.append((float(point[0]) * width_in, float(point[1]) * height_in)) - return points - - -def _draw_program_arrows(slide, program: dict, width_in: float, height_in: float) -> list[dict]: - objects_by_id = _object_map(program) - style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} - token_map = {str(item.get("token_id")): item for item in style.get("color_tokens", []) if isinstance(item, dict)} - rendered: list[dict] = [] - for arrow in program.get("arrows", []): - if arrow.get("type") == "custom_bus": - continue - source = objects_by_id.get(arrow.get("source") or arrow.get("source_id")) - target = objects_by_id.get(arrow.get("target") or arrow.get("target_id")) - path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - if len(path) >= 2 and all(isinstance(point, list) and len(point) >= 2 for point in path): - points = _arrow_points_from_path(path, width_in, height_in) - elif source and target: - x1, y1, x2, y2 = _arrow_endpoints(source, target, width_in, height_in) - points = [(x1, y1), (x2, y2)] - else: - continue - token = token_map.get(str(arrow.get("style_token_id"))) - arrow_color = str(token.get("hex")) if token else str(arrow.get("stroke_color") or "#1F6F8B") - control_kind = str(arrow.get("control_kind") or arrow.get("type", "")).lower() - dashed = control_kind in {"dashed_loop", "dashed", "loop"} - if str(arrow.get("line_pattern", "")).lower() in {"dash", "dashed"}: - dashed = True - line_width = float(arrow.get("stroke_width_pt") or style.get("arrow_weight_pt") or 1.7) - route_style = str(arrow.get("route_style") or "") - connector_type = _connector_type_for_route(route_style) - halo_width = float(arrow.get("halo_width_pt") or 0.0) - halo_color = str(arrow.get("halo_color") or "#FFFFFF") - arrowhead_size = str(arrow.get("arrowhead_size") or "sm").lower() - if arrowhead_size not in {"sm", "med", "lg"}: - arrowhead_size = "sm" - segment_count = 0 - for idx, ((x1, y1), (x2, y2)) in enumerate(zip(points[:-1], points[1:])): - if halo_width > 0: - halo = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) - halo.line.color.rgb = _rgb(halo_color) - halo.line.width = Pt(max(line_width + 1.2, halo_width)) - _apply_line_cap(halo, str(arrow.get("line_cap") or "round")) - if dashed: - halo.line.dash_style = MSO_LINE_DASH_STYLE.DASH - connector = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) - connector.line.color.rgb = _rgb(arrow_color) - connector.line.width = Pt(line_width) - _apply_line_cap(connector, str(arrow.get("line_cap") or "round")) - if dashed: - connector.line.dash_style = MSO_LINE_DASH_STYLE.DASH - if idx == len(points) - 2: - _apply_arrow(connector, arrowhead_size) - segment_count += 1 - rendered.append({ - "arrow_id": arrow.get("id"), - "control_kind": control_kind or "straight_arrow", - "semantic_role": arrow.get("semantic_role"), - "route_style": route_style, - "bundle_id": arrow.get("bundle_id"), - "lane_index": arrow.get("lane_index"), - "line_cap": arrow.get("line_cap", "round"), - "line_pattern": "dash" if dashed else "solid", - "connector_type": "curve" if connector_type == MSO_CONNECTOR.CURVE else "straight", - "halo_width_pt": halo_width, - "halo_color": halo_color if halo_width > 0 else None, - "stroke_width_pt": line_width, - "arrowhead_size": arrowhead_size, - "routing_algorithm": arrow.get("routing_algorithm"), - "route_generation_status": arrow.get("route_generation_status"), - "segment_count": segment_count, - "point_count": len(points), - "editable_in": "pptx", - "render_policy": "ppt_shape_not_image_asset", - "status": "ok" if segment_count else "not_rendered", - }) - return rendered - - -def _ocean_label(text: str) -> str: - low = text.lower() - if "openness" in low: - return "Openness" - if "conscientiousness" in low: - return "Conscientiousness" - if "extraversion" in low: - return "Extraversion" - if "agreeableness" in low: - return "Agreeableness" - if "neuroticism" in low: - return "Neuroticism" - return _short_caption(text) - - -def compile_ppt(program: dict, out_dir: str | Path) -> Path: - out = Path(out_dir) - canvas = program["canvas"] - width_in = float(canvas["width_in"]) - height_in = float(canvas["height_in"]) - style = program.get("style", {}) - palette = style.get("palette") or style.get("reference_palette") or ["#2D6FB7", "#E17721", "#6B57C8", "#1B9A94", "#4B9B52", "#D44E5D"] - panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} - - prs = Presentation() - prs.slide_width = Inches(width_in) - prs.slide_height = Inches(height_in) - slide = prs.slides.add_slide(prs.slide_layouts[6]) - - _add_title_block(slide, program, width_in, height_in) - - panels_by_id = _panel_map(program) - panel_shapes = {} - has_text_program = bool(_text_program_items(program)) - ocr_panel_title_targets = { - str(item.get("target_id")) - for item in _text_program_items(program) - if str(item.get("role")) == "panel_title" and "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() - } - for idx, panel in enumerate(program["panels"]): - x, y, w, h = pct_to_inches(panel["bbox_percent"], width_in, height_in) - local_style = panel_styles.get(panel["id"], {}) if isinstance(panel_styles.get(panel["id"], {}), dict) else {} - fill = local_style.get("fill_color") or palette[idx % len(palette)] - stroke = local_style.get("stroke_color") or palette[(idx + 1) % len(palette)] - header_color = local_style.get("header_color") or stroke - shape = _add_round_rect(slide, x, y, w, h, fill, stroke, width_pt=1.4) - panel_shapes[panel["id"]] = shape - header_h = min(0.34, h * 0.17) - header = _add_round_rect(slide, x, y, w, header_h, header_color, header_color, width_pt=0.8) - title_text_item = _text_item_for_target(program, panel["id"], "panel_title") - _set_text( - header, - "" if str(panel["id"]) in ocr_panel_title_targets else panel["title"], - font_size=float(title_text_item.get("font_size_pt")) if title_text_item else (9 if w < 2.0 else 10), - bold=True, - color=str(title_text_item.get("color_hex")) if title_text_item else "#FFFFFF", - font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, - ) - - rendered_cards = [] - for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index", 12))): - if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): - continue - _add_card_frame(slide, card, width_in, height_in) - x, y, w, h = pct_to_inches(card["bbox_percent"], width_in, height_in) - if card.get("title") and not has_text_program: - _add_label(slide, str(card.get("title")), x + 0.03, y + 0.03, max(0.05, w - 0.06), min(0.22, h * 0.32), font_size=7, bold=True) - rendered_cards.append({ - "card_id": card.get("id"), - "semantic_role": card.get("semantic_role"), - "shape_kind": card.get("shape_kind", "rounded_rect"), - "bbox_percent": card.get("bbox_percent"), - "dash_style": card.get("dash_style", "solid"), - "fill_transparency": float(card.get("fill_transparency", 1.0)), - "editable_in": "pptx", - "render_policy": "ppt_shape_not_image_asset", - "status": "ok", - }) - - rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) - - # Slot image layer and editable captions. - composition_items = [] - caption_queue = [] - ordered_slots = sorted(program["slots"], key=lambda item: int(item.get("z_index", 20))) - for slot in ordered_slots: - x, y, w, h = pct_to_inches(slot["bbox_percent"], width_in, height_in) - asset_path = out / "assets" / f"{slot['asset_id']}.png" - tile_added = False - if slot.get("composition_type") == "symbol_cutout": - if asset_path.exists(): - _add_picture_contain(slide, asset_path, x, y, w, h) - _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) - else: - fill_percent = 0.0 - if bool(slot.get("show_slot_caption", False)) and not has_text_program: - parent = panels_by_id.get(slot.get("panel_id")) - if parent: - px, py, pw, ph = pct_to_inches(parent["bbox_percent"], width_in, height_in) - label_x = x + w + 0.035 - label_w = max(0.28, min(0.86, px + pw - label_x - 0.03)) - caption_queue.append(( - _ocean_label(slot.get("display_label") or slot["paper_concept"]), - label_x, - y + h * 0.10, - label_w, - h * 0.80, - 5 if label_w < 0.55 else 6, - PP_ALIGN.LEFT, - )) - composition_items.append({ - "slot_id": slot["id"], - "asset_id": slot["asset_id"], - "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), - "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), - "tile_frame_added": tile_added, - "caption_inside_image_slot": False, - "slot_bbox_percent": slot["bbox_percent"], - "image_slot_area_fill_percent": round(fill_percent, 2), - "status": "ok" if fill_percent >= 95 else "image_area_below_95", - }) - continue - if asset_path.exists(): - _add_picture_contain(slide, asset_path, x, y, w, h) - _left, _top, fit_w, fit_h, fill_percent = _picture_contain_box(asset_path, x, y, w, h) - else: - fill_percent = 0.0 - if bool(slot.get("show_slot_caption", False)) and not has_text_program: - caption_h = min(0.20, h * 0.18) - caption = slot.get("display_label") or _short_caption(slot["paper_concept"]) - caption_queue.append((caption, x + w * 0.03, y + h + 0.01, w * 0.94, caption_h, 5 if w < 0.8 else 6, PP_ALIGN.CENTER)) - composition_items.append({ - "slot_id": slot["id"], - "asset_id": slot["asset_id"], - "slot_frame_policy": slot.get("slot_frame_policy", "frameless_slot"), - "picture_fill_policy": slot.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), - "tile_frame_added": tile_added, - "caption_inside_image_slot": False, - "slot_bbox_percent": slot["bbox_percent"], - "image_slot_area_fill_percent": round(fill_percent, 2), - "status": "ok" if fill_percent >= 95 else "image_area_below_95", - }) - - for caption, x, y, w, h, font_size, align in caption_queue: - _add_label(slide, caption, x, y, w, h, font_size=font_size, align=align) - - rendered_text_items = [] - for item in _text_program_items(program): - is_ocr_text = "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() - if str(item.get("role")) == "panel_title" and not is_ocr_text: - rendered_text_items.append({ - "text_id": item.get("id"), - "role": item.get("role"), - "target_id": item.get("target_id"), - "rendered_as": "panel_header_text", - "editable_in": "pptx", - }) - continue - bbox = item.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - _add_label( - slide, - str(item.get("text") or ""), - x, - y, - w, - h, - font_size=float(item.get("font_size_pt") or 6), - bold=bool(item.get("bold")), - align=_align_from_text_item(item), - color=str(item.get("color_hex") or "#263747"), - font_family=str(item.get("font_family_guess") or "") or None, - ) - rendered_text_items.append({ - "text_id": item.get("id"), - "role": item.get("role"), - "target_id": item.get("target_id"), - "source_reference_text_id": item.get("source_reference_text_id"), - "bbox_percent": item.get("bbox_percent"), - "font_size_pt": item.get("font_size_pt"), - "color_hex": item.get("color_hex"), - "font_family_guess": item.get("font_family_guess"), - "fit_strategy": item.get("fit_strategy"), - "ocr_confidence": item.get("ocr_confidence"), - "editable_in": "pptx", - "rendered_as": "ppt_textbox", - }) - - for label in program.get("labels", []): - bbox = label.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = pct_to_inches(bbox, width_in, height_in) - _add_label( - slide, - str(label.get("text") or ""), - x, - y, - w, - h, - font_size=float(label.get("font_size_pt") or 9), - bold=bool(label.get("bold", True)), - align=PP_ALIGN.LEFT if str(label.get("align")).lower() == "left" else PP_ALIGN.CENTER, - color=str(label.get("color_hex") or "#263747"), - ) - - # Shared resource bus as editable connectors. - shared = panels_by_id.get("shared_resource_library") - if shared: - sx, sy, sw, sh = pct_to_inches(shared["bbox_percent"], width_in, height_in) - bus_y = sy - 0.30 - line = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(sx + 0.1), Inches(bus_y), Inches(sx + sw - 0.1), Inches(bus_y)) - line.line.color.rgb = _rgb("#1F6F8B") - line.line.width = Pt(1.2) - for panel in program["panels"]: - if panel["id"] == "shared_resource_library": - continue - cx, _cy = _bbox_center(panel["bbox_percent"], width_in, height_in) - if not (sx + 0.1 <= cx <= sx + sw - 0.1): - continue - down = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(cx), Inches(bus_y), Inches(cx), Inches(sy)) - down.line.color.rgb = _rgb("#1F6F8B") - down.line.width = Pt(1.0) - if not has_text_program: - _add_label(slide, "shared resources", sx + 0.2, bus_y - 0.22, 1.5, 0.18, font_size=7, align=PP_ALIGN.LEFT) - - if program.get("show_visible_title"): - title = program.get("paper_brief", {}).get("title_guess") or "Research System Figure" - _add_label(slide, title[:90], width_in - 5.2, height_in - 0.50, 4.9, 0.38, font_size=7, bold=True, align=PP_ALIGN.RIGHT) - - pptx_path = out / "editable_composition.pptx" - prs.save(pptx_path) - write_json(out / "composition_quality_report.json", { - "summary": "PPT composition quality report checking frameless slot insertion, no extra white tiles, and image area fill inside each reference slot.", - "policy": { - "slot_frame_policy": "frameless_slot", - "picture_fill_policy": "direct_full_slot_contain_no_tile", - "min_image_slot_area_fill_percent": 95, - "caption_inside_image_slot_allowed": False, - }, - "slots": composition_items, - "cards": rendered_cards, - "arrows": rendered_arrows, - "text": rendered_text_items, - }) - return pptx_path +from .composition.pptx import * # noqa: F401,F403 diff --git a/rfs/professional_rebuild.py b/rfs/professional_rebuild.py index 8aa37fc..961f952 100644 --- a/rfs/professional_rebuild.py +++ b/rfs/professional_rebuild.py @@ -15,7 +15,7 @@ _ppt_package_counts, rebuild_editable, ) -from .ppt_compiler import compile_ppt +from .composition import compile_ppt from .professional_compiler import dsl_to_program from .professional_dsl import fallback_professional_dsl, validate_and_normalize_dsl from .professional_gap import build_professional_gap_report diff --git a/rfs/providers/__init__.py b/rfs/providers/__init__.py new file mode 100644 index 0000000..a9098dc --- /dev/null +++ b/rfs/providers/__init__.py @@ -0,0 +1,14 @@ +"""External model/provider integration entry points. + +Provider code must not contain repository paths or persist credentials. +""" + +from ..rebuild_vlm_adapters import build_rebuild_vlm_adapters +from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available + +__all__ = [ + "build_rebuild_vlm_adapters", + "call_vlm_json", + "resolve_vlm_model", + "vlm_credentials_available", +] diff --git a/rfs/rebuild_preview_renderer.py b/rfs/rebuild_preview_renderer.py index c2ecef9..8f707bc 100644 --- a/rfs/rebuild_preview_renderer.py +++ b/rfs/rebuild_preview_renderer.py @@ -1,176 +1,5 @@ -from __future__ import annotations +"""Compatibility import for rebuild preview rendering.""" -from pathlib import Path +from .composition.preview import render_rebuild_preview -from PIL import Image, ImageColor, ImageDraw, ImageFont - - -def _hex_color(value: object, fallback: str = "#000000") -> str: - if not isinstance(value, str): - return fallback - text = value.strip() - if not text: - return fallback - if not text.startswith("#"): - text = "#" + text - try: - ImageColor.getrgb(text) - return text - except ValueError: - return fallback - - -def _bbox_px(bbox: dict, width: int, height: int) -> tuple[int, int, int, int]: - x = int(round(float(bbox.get("x", 0)) * width)) - y = int(round(float(bbox.get("y", 0)) * height)) - w = int(round(float(bbox.get("w", 0)) * width)) - h = int(round(float(bbox.get("h", 0)) * height)) - return x, y, max(1, w), max(1, h) - - -def _font(size_px: int, bold: bool = False) -> ImageFont.ImageFont: - candidates = [ - "arialbd.ttf" if bold else "arial.ttf", - "calibrib.ttf" if bold else "calibri.ttf", - "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", - ] - for name in candidates: - try: - return ImageFont.truetype(name, max(4, int(size_px))) - except OSError: - continue - return ImageFont.load_default() - - -def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: int) -> None: - if not item.get("visible", True): - return - if str(item.get("layer_ownership") or "editable_text_layer") != "editable_text_layer": - return - bbox = item.get("bbox_percent") - if not isinstance(bbox, dict): - return - x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) - text = str(item.get("text") or "") - if not text: - return - font_size_pt = float(item.get("font_size_pt") or 8) - size_px = max(4, int(round(font_size_pt * canvas_h / 387.0))) - font = _font(size_px, bold=bool(item.get("bold"))) - color = _hex_color(item.get("color_hex"), "#263747") - align = str(item.get("align") or "center").lower() - text_bbox = draw.textbbox((0, 0), text, font=font) - text_w = max(1, text_bbox[2] - text_bbox[0]) - text_h = max(1, text_bbox[3] - text_bbox[1]) - if align == "left": - tx = x - elif align == "right": - tx = x + w - text_w - else: - tx = x + (w - text_w) / 2 - ty = y + (h - text_h) / 2 - draw.text((tx, ty), text, fill=color, font=font) - - -def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: int) -> None: - path = arrow.get("path_percent") - if not isinstance(path, list) or len(path) < 2: - return - points: list[tuple[int, int]] = [] - for point in path: - if isinstance(point, list) and len(point) >= 2: - points.append((int(round(float(point[0]) * canvas_w)), int(round(float(point[1]) * canvas_h)))) - if len(points) < 2: - return - color = _hex_color(arrow.get("stroke_color") or arrow.get("outline_color"), "#3F5063") - line_width = max(1, int(round(float(arrow.get("stroke_width_pt") or 1.5) * canvas_h / 277.0))) - render_style = str(arrow.get("render_style") or "").lower() - if render_style == "filled_block_arrow": - fill = _hex_color(arrow.get("fill_color"), "#AFC6DE") - start, end = points[0], points[-1] - thickness = max(8, int(round(float(arrow.get("block_arrow_thickness_percent") or 0.052) * canvas_h))) - if abs(end[0] - start[0]) >= abs(end[1] - start[1]): - y = int(round((start[1] + end[1]) / 2)) - left, right = sorted([start[0], end[0]]) - draw.rounded_rectangle((left, y - thickness // 2, right, y + thickness // 2), radius=thickness // 3, fill=fill, outline=color, width=line_width) - else: - x = int(round((start[0] + end[0]) / 2)) - top, bottom = sorted([start[1], end[1]]) - draw.rounded_rectangle((x - thickness // 2, top, x + thickness // 2, bottom), radius=thickness // 3, fill=fill, outline=color, width=line_width) - return - draw.line(points, fill=color, width=line_width, joint="curve") - - -def _draw_card(draw: ImageDraw.ImageDraw, card: dict, canvas_w: int, canvas_h: int) -> None: - bbox = card.get("bbox_percent") - if not isinstance(bbox, dict): - return - x, y, w, h = _bbox_px(bbox, canvas_w, canvas_h) - stroke = _hex_color(card.get("stroke_color"), "#59AFCB") - fill = None if float(card.get("fill_transparency", 1.0)) >= 1.0 else _hex_color(card.get("fill_color"), "#FFFFFF") - line_width = max(1, int(round(float(card.get("stroke_width_pt") or 1.5) * canvas_h / 360.0))) - radius = max(0, int(round(float(card.get("corner_radius") or 0.08) * min(w, h)))) - rect = (x, y, x + w, y + h) - if str(card.get("shape_kind") or "rounded_rect").lower() == "rect": - draw.rectangle(rect, fill=fill, outline=stroke, width=line_width) - else: - draw.rounded_rectangle(rect, radius=radius, fill=fill, outline=stroke, width=line_width) - - -def render_rebuild_preview(program: dict, out_dir: str | Path, preview_path: str | Path | None = None) -> Path: - out = Path(out_dir) - canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} - width = int(canvas.get("width_px") or 1600) - height = int(canvas.get("height_px") or 900) - background = _hex_color(canvas.get("background"), "#FFFFFF") - image = Image.new("RGB", (width, height), background) - draw = ImageDraw.Draw(image) - - style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} - panel_styles = style.get("panel_styles", {}) if isinstance(style.get("panel_styles"), dict) else {} - palette = style.get("palette") or ["#F7F8F7", "#D2DCDE", "#A09596"] - - for idx, panel in enumerate(program.get("panels", [])): - bbox = panel.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = _bbox_px(bbox, width, height) - local_style = panel_styles.get(panel.get("id"), {}) if isinstance(panel_styles.get(panel.get("id")), dict) else {} - fill = _hex_color(local_style.get("fill_color"), palette[idx % len(palette)]) - stroke = _hex_color(local_style.get("stroke_color"), "#FFFFFF") - draw.rounded_rectangle((x, y, x + w, y + h), radius=max(2, min(12, h // 18)), fill=fill, outline=stroke, width=2) - - for card in sorted(program.get("cards", []), key=lambda item: int(item.get("z_index") or 12)): - if isinstance(card, dict): - _draw_card(draw, card, width, height) - - asset_by_id = {str(asset.get("id")): asset for asset in program.get("assets", []) if isinstance(asset, dict)} - for slot in sorted(program.get("slots", []), key=lambda item: int(item.get("z_index") or 20)): - bbox = slot.get("bbox_percent") - if not isinstance(bbox, dict): - continue - x, y, w, h = _bbox_px(bbox, width, height) - asset = asset_by_id.get(str(slot.get("asset_id") or slot.get("id"))) - asset_path = out / str(asset.get("path")) if asset else out / "assets" / f"{slot.get('asset_id')}.png" - if asset_path.exists(): - with Image.open(asset_path) as asset_img: - asset_img = asset_img.convert("RGB") - asset_img.thumbnail((w, h), Image.Resampling.LANCZOS) - left = x + (w - asset_img.width) // 2 - top = y + (h - asset_img.height) // 2 - image.paste(asset_img, (left, top)) - - for arrow in program.get("arrows", []): - if isinstance(arrow, dict): - _draw_arrow(draw, arrow, width, height) - - text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} - text_items = text_program.get("items", []) if isinstance(text_program, dict) else [] - visible_items = [item for item in text_items if isinstance(item, dict)] - for item in sorted(visible_items, key=lambda item: (float(item.get("z_index") or 80), str(item.get("id") or ""))): - _draw_text(draw, item, width, height) - - target = Path(preview_path) if preview_path else out / "rebuild_preview.png" - target.parent.mkdir(parents=True, exist_ok=True) - image.save(target) - return target +__all__ = ["render_rebuild_preview"] diff --git a/rfs/rebuild_visual_critic.py b/rfs/rebuild_visual_critic.py index 5dfa2f2..95f531f 100644 --- a/rfs/rebuild_visual_critic.py +++ b/rfs/rebuild_visual_critic.py @@ -1,195 +1,6 @@ -from __future__ import annotations +"""Compatibility imports for rebuild visual evaluation. -from pathlib import Path -from typing import Any +New code should import from :mod:`rfs.evaluation`. +""" -from .utils import write_json - - -def _bbox(item: dict) -> dict | None: - box = item.get("bbox_percent") - return box if isinstance(box, dict) else None - - -def _valid_bbox(box: dict | None) -> bool: - if not isinstance(box, dict): - return False - try: - x = float(box["x"]) - y = float(box["y"]) - w = float(box["w"]) - h = float(box["h"]) - except Exception: - return False - return 0 <= x <= 1 and 0 <= y <= 1 and w > 0 and h > 0 and x + w <= 1.0001 and y + h <= 1.0001 - - -def _overlap_area(a: dict, b: dict) -> float: - ax0, ay0 = float(a["x"]), float(a["y"]) - ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) - bx0, by0 = float(b["x"]), float(b["y"]) - bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) - ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) - iy = max(0.0, min(ay1, by1) - max(ay0, by0)) - return ix * iy - - -def _overlap_ratio(a: dict, b: dict) -> float: - area = _overlap_area(a, b) - if area <= 0: - return 0.0 - a_area = float(a["w"]) * float(a["h"]) - b_area = float(b["w"]) * float(b["h"]) - return area / max(min(a_area, b_area), 0.000001) - - -def _visible_text_items(program: dict) -> list[dict]: - text_program = program.get("text_program") - if not isinstance(text_program, dict): - return [] - return [item for item in text_program.get("items", []) if isinstance(item, dict) and item.get("visible", True)] - - -def _text_alignment_group_key(item: dict) -> tuple[str, str] | None: - for key in ("paragraph_id", "text_group_id", "group_id", "source_group_id"): - value = str(item.get(key) or "").strip() - if value: - return (key, value) - return None - - -def _detect_text_issues(program: dict) -> list[dict]: - issues = [] - items = _visible_text_items(program) - for item in items: - if not _valid_bbox(_bbox(item)): - issues.append({"type": "text_bbox_out_of_bounds", "text_id": item.get("id"), "reason": "text bbox is missing or outside canvas"}) - for index, left in enumerate(items): - lbox = _bbox(left) - if not _valid_bbox(lbox): - continue - for right in items[index + 1:]: - rbox = _bbox(right) - if not _valid_bbox(rbox): - continue - ratio = _overlap_ratio(lbox, rbox) - if ratio > 0.18: - issues.append({ - "type": "text_overlap", - "text_id": left.get("id"), - "other_text_id": right.get("id"), - "overlap_ratio": round(ratio, 4), - "reason": "visible text boxes overlap substantially", - }) - grouped: dict[tuple[str, str], list[dict]] = {} - for item in items: - key = _text_alignment_group_key(item) - if key: - grouped.setdefault(key, []).append(item) - for (group_key, group_id), group in grouped.items(): - if len(group) < 3: - continue - centers = [ - float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2 - for item in group - if _valid_bbox(_bbox(item)) - ] - if len(centers) >= 3 and max(centers) - min(centers) > 0.035: - issues.append({ - "type": "text_group_misaligned", - "group_key": group_key, - "group_id": group_id, - "text_ids": [item.get("id") for item in group], - "center_y_span": round(max(centers) - min(centers), 4), - "reason": "explicit text group is visually uneven", - }) - return issues - - -def _detect_object_issues(program: dict) -> list[dict]: - issues = [] - for collection_name in ("panels", "slots"): - items = [item for item in program.get(collection_name, []) if isinstance(item, dict)] - for item in items: - if not _valid_bbox(_bbox(item)): - issues.append({"type": f"{collection_name[:-1]}_bbox_out_of_bounds", "id": item.get("id"), "reason": "bbox is missing or outside canvas"}) - for index, left in enumerate(items): - lbox = _bbox(left) - if not _valid_bbox(lbox): - continue - for right in items[index + 1:]: - rbox = _bbox(right) - if not _valid_bbox(rbox): - continue - ratio = _overlap_ratio(lbox, rbox) - if ratio > (0.35 if collection_name == "slots" else 0.18): - issues.append({ - "type": f"{collection_name[:-1]}_overlap", - "id": left.get("id"), - "other_id": right.get("id"), - "overlap_ratio": round(ratio, 4), - "reason": f"{collection_name[:-1]} boxes overlap substantially", - }) - return issues - - -def _detect_arrow_issues(program: dict) -> list[dict]: - issues = [] - for arrow in program.get("arrows", []) or []: - if not isinstance(arrow, dict): - continue - path = arrow.get("path_percent") - if not isinstance(path, list) or len(path) < 2: - issues.append({"type": "arrow_missing_path", "arrow_id": arrow.get("id"), "reason": "arrow has no usable path_percent"}) - return issues - - -def _detect_ownership_issues(program: dict, ownership_report: dict | None) -> list[dict]: - issues = [] - text_ids = {str(item.get("source_reference_text_id") or item.get("id") or "") for item in _visible_text_items(program)} - report = ownership_report if isinstance(ownership_report, dict) else {} - for item in report.get("items", []) or []: - if not isinstance(item, dict): - continue - text_id = str(item.get("text_id") or "") - ownership = str(item.get("layer_ownership") or "") - included = bool(item.get("included_in_text_program")) - if ownership != "editable_text_layer" and text_id in text_ids: - issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "non-editable-owned text is present in text_program"}) - if ownership == "editable_text_layer" and not included: - issues.append({"type": "text_layer_ownership_conflict", "text_id": text_id, "layer_ownership": ownership, "reason": "editable-owned text is missing from text_program"}) - return issues - - -def run_rebuild_visual_quality_check( - out_dir: str | Path, - program: dict, - reference_geometry: dict | None = None, - reference_controls: dict | None = None, - ownership_report: dict | None = None, - mode: str = "heuristic", -) -> dict[str, Any]: - text_issues = _detect_text_issues(program) - object_issues = _detect_object_issues(program) - arrow_issues = _detect_arrow_issues(program) - ownership_issues = _detect_ownership_issues(program, ownership_report) - issues = text_issues + object_issues + arrow_issues + ownership_issues - blocking_types = {"text_overlap", "text_bbox_out_of_bounds", "arrow_missing_path", "text_layer_ownership_conflict"} - blocking = [item for item in issues if item.get("type") in blocking_types] - report = { - "summary": "Deterministic rebuild visual quality report.", - "mode": mode, - "status": "blocked" if blocking else ("warning" if issues else "pass"), - "issue_count": len(issues), - "blocking_issue_count": len(blocking), - "text_issue_count": len(text_issues), - "object_issue_count": len(object_issues), - "arrow_issue_count": len(arrow_issues), - "ownership_issue_count": len(ownership_issues), - "issues": issues, - "reference_geometry_status": (reference_geometry or {}).get("status"), - "reference_controls_status": (reference_controls or {}).get("status"), - "policy": "deterministic check only; no program mutation", - } - write_json(Path(out_dir) / "rebuild_visual_quality_report.json", report) - return report +from .evaluation.rebuild_visual import * # noqa: F401,F403 diff --git a/rfs/semantic_contract.py b/rfs/semantic_contract.py index de807ad..756d331 100644 --- a/rfs/semantic_contract.py +++ b/rfs/semantic_contract.py @@ -1,231 +1,6 @@ -from __future__ import annotations +"""Compatibility imports for the semantic contract API. -import re -from pathlib import Path -from typing import Any +New code should import from :mod:`rfs.contracts`. +""" -from .utils import read_json, write_json - - -def load_paper_semantic_contract(run_dir: str | Path) -> dict[str, Any]: - root = Path(run_dir) - return { - "summary": "Paper-grounded semantic contract for editable figure reconstruction.", - "source_run_dir": str(root.resolve()), - "paper_review": read_json(root / "paper_review.json"), - "figure_specification": read_json(root / "figure_specification.json"), - "paper_summary": read_json(root / "paper_summary.json"), - "design_plan": read_json(root / "design_plan.json"), - "layout_intent": read_json(root / "layout_intent.json"), - "style_plan": read_json(root / "style_plan.json"), - } - - -def _text(item: dict, fallback: str = "") -> str: - for key in ("visible_label", "name", "text", "statement", "label", "id"): - value = str(item.get(key) or "").strip() - if value: - return value - return fallback - - -def _entities(contract: dict) -> list[dict]: - spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract - entities: list[dict] = [] - for field, role in (("inputs", "input"), ("modules", "module"), ("outputs", "output"), ("innovations", "innovation")): - values = spec.get(field) if isinstance(spec.get(field), list) else [] - for index, raw in enumerate(values, start=1): - if not isinstance(raw, dict): - continue - label = _text(raw) - if not label: - continue - entity_id = str(raw.get("id") or raw.get("name") or f"{role}_{index:02d}").strip() - entities.append({ - "id": entity_id, - "label": label, - "role": role, - "evidence_ids": list(raw.get("evidence_ids") or []), - "source": raw, - }) - return entities - - -def _tokens(value: object) -> set[str]: - return {token for token in re.findall(r"[a-z0-9]+", str(value or "").lower()) if len(token) > 1} - - -def _object_text(obj: dict) -> str: - return " ".join(str(obj.get(key) or "") for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt")) - - -def _center(obj: dict) -> tuple[float, float]: - box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} - return float(box.get("x") or 0) + float(box.get("w") or 0) / 2, float(box.get("y") or 0) + float(box.get("h") or 0) / 2 - - -def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dict], list[dict]]: - objects = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] - if len(objects) < len(entities): - seen = {str(item.get("id")) for item in objects} - objects.extend( - item for item in program.get("cards", []) + program.get("panels", []) - if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and str(item.get("id")) not in seen - ) - objects.sort(key=lambda item: (_center(item)[0], _center(item)[1], str(item.get("id") or ""))) - available = list(objects) - mapping: dict[str, dict] = {} - records: list[dict] = [] - for entity in entities: - entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) - scored = [] - for obj in available: - obj_tokens = _tokens(_object_text(obj)) - score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) - scored.append((score, -_center(obj)[0], obj)) - if not scored: - records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) - continue - scored.sort(key=lambda value: (value[0], value[1]), reverse=True) - chosen = scored[0][2] if scored[0][0] > 0 else available[0] - available.remove(chosen) - mapping[entity["id"]] = chosen - records.append({ - "entity_id": entity["id"], - "label": entity["label"], - "role": entity["role"], - "object_id": chosen.get("id"), - "match_score": round(max(0.0, scored[0][0]), 4), - "status": "mapped", - }) - return mapping, records - - -def _inside(inner: dict, outer: dict) -> bool: - cx = float(inner.get("x") or 0) + float(inner.get("w") or 0) / 2 - cy = float(inner.get("y") or 0) + float(inner.get("h") or 0) / 2 - return float(outer.get("x") or 0) <= cx <= float(outer.get("x") or 0) + float(outer.get("w") or 0) and float(outer.get("y") or 0) <= cy <= float(outer.get("y") or 0) + float(outer.get("h") or 0) - - -def _label_bbox(obj: dict) -> dict[str, float]: - box = obj["bbox_percent"] - x, y, w, h = (float(box[key]) for key in ("x", "y", "w", "h")) - label_h = min(0.055, max(0.026, h * 0.18)) - return {"x": round(x, 4), "y": round(max(0.0, y), 4), "w": round(w, 4), "h": round(label_h, 4)} - - -def _anchor(source: dict, target: dict) -> list[list[float]]: - sx, sy = _center(source) - tx, ty = _center(target) - sb, tb = source["bbox_percent"], target["bbox_percent"] - if abs(tx - sx) >= abs(ty - sy): - start = [float(sb["x"]) + (float(sb["w"]) if tx >= sx else 0.0), sy] - end = [float(tb["x"]) + (0.0 if tx >= sx else float(tb["w"])), ty] - mid = (start[0] + end[0]) / 2 - points = [start, [mid, start[1]], [mid, end[1]], end] - else: - start = [sx, float(sb["y"]) + (float(sb["h"]) if ty >= sy else 0.0)] - end = [tx, float(tb["y"]) + (0.0 if ty >= sy else float(tb["h"]))] - mid = (start[1] + end[1]) / 2 - points = [start, [start[0], mid], [end[0], mid], end] - clean: list[list[float]] = [] - for point in points: - rounded = [round(max(0.0, min(1.0, point[0])), 4), round(max(0.0, min(1.0, point[1])), 4)] - if not clean or clean[-1] != rounded: - clean.append(rounded) - return clean - - -def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: - entities = _entities(contract) - mapping, mapping_records = _assign_entities(entities, program) - mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] - text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} - existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] - preserved = [item for item in existing if not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes)] - semantic_text = [] - for entity in entities: - obj = mapping.get(entity["id"]) - if not obj: - continue - box = _label_bbox(obj) - semantic_text.append({ - "id": f"semantic_text_{entity['id']}", - "text": entity["label"], - "role": f"paper_{entity['role']}_label", - "target_id": obj.get("id"), - "semantic_entity_id": entity["id"], - "semantic_evidence_ids": entity["evidence_ids"], - "bbox_percent": box, - "center_percent": {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)}, - "width_percent": box["w"], - "height_percent": box["h"], - "font_size_pt": 10.0, - "color_hex": "#263747", - "bold": entity["role"] in {"module", "innovation"}, - "align": "center", - "font_family_guess": "Arial", - "fit_strategy": "paper_semantic_contract_exact_label", - "editable_in": "pptx", - "visible": True, - "layer_ownership": "editable_text_layer", - }) - text_program = { - **text_program, - "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", - "semantic_authority": "paper_ground_truth", - "items": preserved + semantic_text, - } - program["text_program"] = text_program - program["text_program_path"] = "text_program.json" - - spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract - semantic_arrows = [] - skipped_relations = [] - for index, relation in enumerate(spec.get("relations") if isinstance(spec.get("relations"), list) else [], start=1): - if not isinstance(relation, dict): - continue - source_id = str(relation.get("source") or relation.get("source_id") or "") - target_id = str(relation.get("target") or relation.get("target_id") or "") - source, target = mapping.get(source_id), mapping.get(target_id) - if not source or not target: - skipped_relations.append({"source": source_id, "target": target_id, "reason": "unmapped_endpoint"}) - continue - semantic_arrows.append({ - "id": str(relation.get("id") or f"semantic_relation_{index:02d}"), - "source_id": str(source.get("id")), - "target_id": str(target.get("id")), - "semantic_source_id": source_id, - "semantic_target_id": target_id, - "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), - "label": str(relation.get("label") or relation.get("statement") or ""), - "evidence_ids": list(relation.get("evidence_ids") or []), - "path_percent": _anchor(source, target), - "render_style": "elbow_connector", - "route_intent": "paper_semantic_relation", - "stroke_color": "#3F5063", - "stroke_width_pt": 1.8, - "arrowhead": "triangle", - "editable_in": "pptx", - }) - if semantic_arrows: - program["arrows"] = semantic_arrows - - report = { - "summary": "Paper semantic contract application report.", - "status": "pass" if len(mapping) == len(entities) and not skipped_relations else "warning", - "semantic_authority": "paper_ground_truth", - "visual_authority": "generated_reference_image", - "entity_count": len(entities), - "mapped_entity_count": len(mapping), - "relation_count": len(spec.get("relations") or []), - "mapped_relation_count": len(semantic_arrows), - "mappings": mapping_records, - "skipped_relations": skipped_relations, - "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", - } - out = Path(out_dir) - write_json(out / "paper_semantic_contract.json", contract) - write_json(out / "semantic_binding_report.json", report) - write_json(out / "text_program.json", text_program) - return program, report +from .contracts.semantic import * # noqa: F401,F403 diff --git a/rfs/workflow.py b/rfs/workflow.py index 7982b6a..f1be057 100644 --- a/rfs/workflow.py +++ b/rfs/workflow.py @@ -10,7 +10,7 @@ from .input_loader import load_text from .paper_analyzer import analyze_paper from .presentations_qa import run_presentations_qa -from .ppt_compiler import compile_ppt +from .composition import compile_ppt from .prompt_planner import plan_slot_prompts from .layout_locator import locate_layout from .program_builder import build_figure_program diff --git a/rfs/workflows/__init__.py b/rfs/workflows/__init__.py new file mode 100644 index 0000000..7318dfd --- /dev/null +++ b/rfs/workflows/__init__.py @@ -0,0 +1,5 @@ +"""Product-level workflow entry points.""" + +from .paper_to_editable import run_paper_to_editable + +__all__ = ["run_paper_to_editable"] diff --git a/rfs/workflows/paper_to_editable.py b/rfs/workflows/paper_to_editable.py new file mode 100644 index 0000000..556a54f --- /dev/null +++ b/rfs/workflows/paper_to_editable.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..contracts import load_paper_semantic_contract +from ..editable_rebuild import rebuild_editable +from ..paper_to_image import run_paper_to_image +from ..rebuild_vlm_adapters import build_rebuild_vlm_adapters +from ..utils import ensure_dir, write_json + + +def run_paper_to_editable( + paper: str | Path, + out: str | Path, + preferences_path: str | Path | None = None, + positive_references: list[str] | None = None, + negative_references: list[str] | None = None, + planner_mode: str = "vlm", + planner_model: str | None = None, + image_asset_mode: str = "image2", + image_candidates: int = 3, + aspect_ratio: str | None = None, + language: str | None = None, + image_model: str | None = None, + image_retries: int = 2, + review_mode: str = "vlm", + review_model: str | None = None, + domain_profile: str = "auto", + template: str = "auto", + repair_rounds: int = 1, + rebuild_asset_mode: str = "api", + rebuild_asset_policy: str = "smart-api", + layout_mode: str = "hybrid", + control_mode: str = "hybrid", + text_mode: str = "ocr", + design_plan_mode: str = "vlm", + export_preview: bool = True, + allow_engineering_preview: bool = False, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + critic_adapter=None, +) -> dict[str, Any]: + root = ensure_dir(out).resolve() + image_dir = root / "paper_to_image" + editable_dir = root / "editable" + image_result = run_paper_to_image( + paper=paper, + out=image_dir, + preferences_path=preferences_path, + positive_references=positive_references, + negative_references=negative_references, + planner_mode=planner_mode, + planner_model=planner_model, + asset_mode=image_asset_mode, + candidates=image_candidates, + aspect_ratio=aspect_ratio, + language=language, + image_model=image_model, + image_retries=image_retries, + review_mode=review_mode, + review_model=review_model, + domain_profile=domain_profile, + template=template, + repair_rounds=repair_rounds, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + critic_adapter=critic_adapter, + ) + reference = image_result.get("selected_image") + reference_status = "production_selected_image" + if not reference and allow_engineering_preview: + reference = image_result.get("engineering_preview") + reference_status = "engineering_preview_explicitly_allowed" + if not reference: + blocked = { + "summary": "Paper-to-editable delivery blocked before rebuild because no production-approved image exists.", + "ok": False, + "delivery_blocked": True, + "out_dir": str(root), + "paper_to_image": image_result, + "policy": "failed image candidates are never promoted to production editable output", + } + write_json(root / "paper_to_editable_result.json", blocked) + return blocked + + contract = load_paper_semantic_contract(image_dir) + adapters = build_rebuild_vlm_adapters(editable_dir) + rebuild_ocr_engine = ocr_engine if ocr_engine in {"paddle", "easyocr", "off"} else "paddle" + editable_result = rebuild_editable( + reference=reference, + out=editable_dir, + asset_mode=rebuild_asset_mode, + asset_policy=rebuild_asset_policy, + text_mode=text_mode, + layout_mode=layout_mode, + control_mode=control_mode, + export_preview=export_preview, + ocr_engine=rebuild_ocr_engine, + ocr_lang=ocr_lang, + vlm_layout_adapter=adapters["layout"] if layout_mode in {"vlm", "hybrid"} else None, + control_adapter=adapters["control"] if control_mode in {"vlm", "hybrid"} else None, + semantic_adapter=adapters["semantic"] if layout_mode in {"vlm", "hybrid"} or control_mode in {"vlm", "hybrid"} else None, + design_plan_mode=design_plan_mode, + design_adapter=adapters["design"] if design_plan_mode == "vlm" else None, + semantic_contract=contract, + ) + result = { + "summary": "Paper-to-editable workflow completed with paper-grounded semantic bindings.", + "ok": bool(editable_result.get("ok")), + "delivery_blocked": False, + "out_dir": str(root), + "reference_status": reference_status, + "reference_image": str(reference), + "paper_to_image_dir": str(image_dir), + "editable_dir": str(editable_dir), + "pptx": editable_result.get("pptx"), + "preview": editable_result.get("preview"), + "semantic_binding_status": editable_result.get("semantic_binding_status"), + "paper_to_image": image_result, + "editable_rebuild": editable_result, + } + write_json(root / "paper_to_editable_result.json", result) + return result diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..fcc8285 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,11 @@ +# Test layers + +- `unit/`: deterministic functions and local file/PPTX behavior. +- `integration/`: multi-component workflows using mocks, placeholders, or temporary files. +- `e2e/`: real paper-to-editable acceptance cases. This layer is intentionally empty until benchmark fixtures and production credentials are defined. + +Run all local tests with: + +```powershell +python -m unittest discover -s tests -q +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..1d1083a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""ResearchFigureStudio test suite.""" diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..1142560 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1 @@ +"""Real paper-to-editable acceptance tests and benchmark-backed smoke tests.""" diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..93b8465 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Tests spanning multiple ResearchFigureStudio engine components.""" diff --git a/tests/test_coevolution.py b/tests/integration/test_coevolution.py similarity index 100% rename from tests/test_coevolution.py rename to tests/integration/test_coevolution.py diff --git a/tests/test_paper_to_image.py b/tests/integration/test_paper_to_image.py similarity index 100% rename from tests/test_paper_to_image.py rename to tests/integration/test_paper_to_image.py diff --git a/tests/test_presentations_qa.py b/tests/integration/test_presentations_qa.py similarity index 100% rename from tests/test_presentations_qa.py rename to tests/integration/test_presentations_qa.py diff --git a/tests/test_rebuild_editable.py b/tests/integration/test_rebuild_editable.py similarity index 100% rename from tests/test_rebuild_editable.py rename to tests/integration/test_rebuild_editable.py diff --git a/tests/test_validator.py b/tests/integration/test_validator.py similarity index 100% rename from tests/test_validator.py rename to tests/integration/test_validator.py diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..61a6105 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Fast tests for isolated contracts, geometry, text, and composition behavior.""" diff --git a/tests/test_arrow_router.py b/tests/unit/test_arrow_router.py similarity index 100% rename from tests/test_arrow_router.py rename to tests/unit/test_arrow_router.py diff --git a/tests/unit/test_package_boundaries.py b/tests/unit/test_package_boundaries.py new file mode 100644 index 0000000..bbecc7b --- /dev/null +++ b/tests/unit/test_package_boundaries.py @@ -0,0 +1,43 @@ +import unittest + + +class PackageBoundaryTests(unittest.TestCase): + def test_stable_package_entry_points_are_importable(self): + from rfs.analysis import parse_paper, plan_reference_layout + from rfs.composition import compile_ppt, render_rebuild_preview + from rfs.contracts import apply_paper_semantic_contract + from rfs.evaluation import run_rebuild_visual_quality_check + from rfs.generation import generate_and_select + from rfs.planning import plan_paper_image + from rfs.providers import call_vlm_json + from rfs.workflows import run_paper_to_editable + + for value in ( + parse_paper, + plan_reference_layout, + compile_ppt, + render_rebuild_preview, + apply_paper_semantic_contract, + run_rebuild_visual_quality_check, + generate_and_select, + plan_paper_image, + call_vlm_json, + run_paper_to_editable, + ): + self.assertTrue(callable(value)) + + def test_legacy_imports_remain_compatible(self): + from rfs.composition import compile_ppt as stable_compile + from rfs.contracts import apply_paper_semantic_contract as stable_contract + from rfs.paper_to_editable import run_paper_to_editable as legacy_workflow + from rfs.ppt_compiler import compile_ppt as legacy_compile + from rfs.semantic_contract import apply_paper_semantic_contract as legacy_contract + from rfs.workflows import run_paper_to_editable as stable_workflow + + self.assertIs(legacy_compile, stable_compile) + self.assertIs(legacy_contract, stable_contract) + self.assertIs(legacy_workflow, stable_workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ppt_compiler.py b/tests/unit/test_ppt_compiler.py similarity index 100% rename from tests/test_ppt_compiler.py rename to tests/unit/test_ppt_compiler.py diff --git a/tests/test_rebuild_visual_critic.py b/tests/unit/test_rebuild_visual_critic.py similarity index 100% rename from tests/test_rebuild_visual_critic.py rename to tests/unit/test_rebuild_visual_critic.py diff --git a/tests/test_semantic_contract.py b/tests/unit/test_semantic_contract.py similarity index 100% rename from tests/test_semantic_contract.py rename to tests/unit/test_semantic_contract.py diff --git a/tests/test_text_grouping.py b/tests/unit/test_text_grouping.py similarity index 100% rename from tests/test_text_grouping.py rename to tests/unit/test_text_grouping.py diff --git a/tests/test_text_layer_ocr.py b/tests/unit/test_text_layer_ocr.py similarity index 100% rename from tests/test_text_layer_ocr.py rename to tests/unit/test_text_layer_ocr.py From 63ef1c4dda5f751920ccb6a6b5cefe0468274fd2 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Mon, 20 Jul 2026 20:09:32 +0800 Subject: [PATCH 14/74] Add dual scientific figure benchmark framework --- benchmarks/README.md | 46 +- .../image-to-ppt/annotation_template.json | 33 ++ .../cases/001_three_stage_layout/case.json | 26 + .../expected_objects.json | 14 + .../001_three_stage_layout/reference.ppm | 6 + .../cases/001_linear_pipeline/case.json | 26 + .../expected_semantics.json | 17 + .../cases/001_linear_pipeline/paper.md | 11 + .../001_linear_pipeline/preferences.json | 9 + .../paper-to-image/human_rating_template.json | 30 ++ benchmarks/schema/case.schema.json | 14 + .../schema/expected_objects.schema.json | 11 + .../schema/expected_semantics.schema.json | 12 + .../schema/paper_image_review.schema.json | 32 ++ docs/architecture.md | 1 + rfs/cli.py | 25 + rfs/evaluation/__init__.py | 9 +- rfs/evaluation/benchmarking.py | 470 ++++++++++++++++++ skills/research-figure-studio/SKILL.md | 12 + tests/unit/test_benchmarking.py | 78 +++ 20 files changed, 869 insertions(+), 13 deletions(-) create mode 100644 benchmarks/image-to-ppt/annotation_template.json create mode 100644 benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json create mode 100644 benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json create mode 100644 benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm create mode 100644 benchmarks/paper-to-image/cases/001_linear_pipeline/case.json create mode 100644 benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md create mode 100644 benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json create mode 100644 benchmarks/paper-to-image/human_rating_template.json create mode 100644 benchmarks/schema/case.schema.json create mode 100644 benchmarks/schema/expected_objects.schema.json create mode 100644 benchmarks/schema/expected_semantics.schema.json create mode 100644 benchmarks/schema/paper_image_review.schema.json create mode 100644 rfs/evaluation/benchmarking.py create mode 100644 tests/unit/test_benchmarking.py diff --git a/benchmarks/README.md b/benchmarks/README.md index a52a020..ef4298b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,16 +1,38 @@ -# Benchmarks +# ResearchFigureStudio Benchmarks -Benchmarks are the product-level acceptance layer for paper-to-editable figure generation. +ResearchFigureStudio uses two independent benchmark suites so failures can be attributed to either reference-image generation or editable reconstruction. -Each case should provide a paper, optional positive/negative visual references, a semantic expectation file, and expected production gates. Do not commit copyrighted papers or secrets; fixtures may point to locally supplied inputs. +## Suites -Recommended benchmark dimensions: +### `paper-to-image` -- exact paper-label preservation -- entity and relation coverage -- connector endpoint correctness -- editability of text, panels, cards, and arrows -- visual similarity to the selected reference image -- PPTX package validity and preview rendering -- cross-machine execution with no repository-specific absolute paths -- API cost and elapsed time +Measures scientific faithfulness, terminology, relation correctness, information coverage and density, clarity, aesthetics, reference compliance, hallucinations, and stability across repeated generations. + +Scientific errors are hard failures and cannot be offset by aesthetics. + +### `image-to-ppt` + +Measures rendered visual fidelity, object and relation reconstruction, text alignment, editable PowerPoint structure, anti-cheating rules, visual blockers, and eventually edit-mutation behavior. + +A full-slide copy of the reference image is a hard failure even if pixel similarity is perfect. + +## Commands + +```powershell +rfs benchmark list --root benchmarks --json +rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipeline --json +rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json +rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json +``` + +## Case policy + +Each case contains `case.json` plus suite-specific human-authored ground truth. Synthetic cases may be committed. Real papers and figures should only be committed when redistribution is permitted; otherwise use local paths or a private benchmark data repository. + +Generated runs and reports belong under `output/benchmarks/` and are not source fixtures. + +## Evaluation tiers + +- Offline contract tier: placeholder assets, deterministic validation, CI-safe. +- Production quality tier: real VLM/image models, frozen judge, repeated seeds, and human calibration. +- Human audit tier: blinded pairwise ratings for aesthetics, clarity, and information density. diff --git a/benchmarks/image-to-ppt/annotation_template.json b/benchmarks/image-to-ppt/annotation_template.json new file mode 100644 index 0000000..7edcc82 --- /dev/null +++ b/benchmarks/image-to-ppt/annotation_template.json @@ -0,0 +1,33 @@ +{ + "summary": "Independent human annotation template for image-to-PPT reconstruction.", + "case_id": "", + "canvas": {"width_px": 0, "height_px": 0}, + "objects": [ + { + "id": "object_01", + "type": "panel|card|asset|text|legend|formula", + "bbox_percent": {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0}, + "text": "", + "parent_id": null, + "z_index": 0, + "must_be_editable": true + } + ], + "relations": [ + { + "id": "relation_01", + "source": "object_01", + "target": "object_02", + "type": "data_flow", + "path_percent": [[0.0, 0.0], [0.0, 0.0]], + "must_be_editable": true + } + ], + "groups": [], + "mutation_tests": [ + {"operation": "replace_text", "target_id": "", "value": "Edited label"}, + {"operation": "move_object", "target_id": "", "delta_percent": {"x": 0.05, "y": 0.0}}, + {"operation": "change_fill", "target_id": "", "value": "#FFAA00"}, + {"operation": "delete_relation", "target_id": ""} + ] +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json b/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json new file mode 100644 index 0000000..0763927 --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/case.json @@ -0,0 +1,26 @@ +{ + "summary": "Synthetic three-stage image-to-editable-PPT reconstruction case.", + "case_id": "001_three_stage_layout", + "suite": "image-to-ppt", + "tier": "offline-contract", + "reference_image": "reference.ppm", + "expected_objects": "expected_objects.json", + "preview": "rebuild_preview.png", + "pptx": "editable_composition.pptx", + "run_config": { + "asset_mode": "placeholder", + "asset_policy": "smart-api", + "text_mode": "off", + "layout_mode": "heuristic", + "control_mode": "heuristic", + "design_plan_mode": "heuristic", + "ocr_engine": "off" + }, + "thresholds": { + "object_coverage": 0.8, + "relation_coverage": 1.0, + "editability_score": 0.66, + "full_slide_image_count": 0, + "blocking_visual_issue_count": 0 + } +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json b/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json new file mode 100644 index 0000000..91a070f --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/expected_objects.json @@ -0,0 +1,14 @@ +{ + "summary": "Human-authored object and relation annotations for a three-stage layout.", + "objects": [ + {"id": "input", "type": "card", "bbox_percent": {"x": 0.05, "y": 0.25, "w": 0.22, "h": 0.45}}, + {"id": "method", "type": "card", "bbox_percent": {"x": 0.39, "y": 0.20, "w": 0.22, "h": 0.55}}, + {"id": "output", "type": "card", "bbox_percent": {"x": 0.73, "y": 0.25, "w": 0.22, "h": 0.45}} + ], + "relations": [ + {"source": "input", "target": "method", "type": "data_flow"}, + {"source": "method", "target": "output", "type": "data_flow"} + ], + "required_editable_types": ["card", "text", "connector"], + "forbid_full_slide_reference_image": true +} diff --git a/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm b/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm new file mode 100644 index 0000000..de7242c --- /dev/null +++ b/benchmarks/image-to-ppt/cases/001_three_stage_layout/reference.ppm @@ -0,0 +1,6 @@ +P3 +6 3 +255 +250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 250 +80 150 210 80 150 210 250 250 250 70 180 160 70 180 160 220 150 80 +80 150 210 80 150 210 250 250 250 70 180 160 70 180 160 220 150 80 diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json new file mode 100644 index 0000000..0e31eb3 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/case.json @@ -0,0 +1,26 @@ +{ + "summary": "Synthetic three-stage pipeline for scientific and visual benchmark protocol validation.", + "case_id": "001_linear_pipeline", + "suite": "paper-to-image", + "tier": "offline-contract", + "paper": "paper.md", + "expected_semantics": "expected_semantics.json", + "preferences": "preferences.json", + "positive_references": [], + "negative_references": [], + "run_config": { + "planner_mode": "heuristic", + "image_asset_mode": "placeholder", + "image_candidates": 2, + "review_mode": "heuristic", + "aspect_ratio": "16:9", + "ocr_engine": "off" + }, + "thresholds": { + "entity_recall": 1.0, + "relation_recall": 1.0, + "exact_label_rate": 1.0, + "hallucination_count": 0, + "forbidden_content_count": 0 + } +} diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json new file mode 100644 index 0000000..05d6dcc --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/expected_semantics.json @@ -0,0 +1,17 @@ +{ + "summary": "Human-authored semantic ground truth for the ExampleFlow figure.", + "entities": [ + {"id": "raw_document", "label": "Raw Document", "role": "input", "required": true}, + {"id": "document_encoding", "label": "Document Encoding", "role": "module", "required": true}, + {"id": "evidence_aggregation", "label": "Evidence Aggregation", "role": "module", "required": true}, + {"id": "final_prediction", "label": "Final Prediction", "role": "output", "required": true} + ], + "relations": [ + {"source": "raw_document", "target": "document_encoding", "type": "data_flow", "required": true}, + {"source": "document_encoding", "target": "evidence_aggregation", "type": "data_flow", "required": true}, + {"source": "evidence_aggregation", "target": "final_prediction", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Agent", "Memory", "Retriever", "Knowledge Base"], + "required_innovation": "Evidence Aggregation", + "expected_reading_order": ["raw_document", "document_encoding", "evidence_aggregation", "final_prediction"] +} diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md b/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md new file mode 100644 index 0000000..352fc23 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/paper.md @@ -0,0 +1,11 @@ +# ExampleFlow: A Three-Stage Evidence Processing Framework + +## Abstract + +ExampleFlow transforms a Raw Document into a Final Prediction through Document Encoding and Evidence Aggregation. + +## Method + +The system takes a Raw Document as input. Document Encoding converts the Raw Document into contextual representations. Evidence Aggregation consumes the contextual representations and produces a Final Prediction. + +The processing order is Raw Document, Document Encoding, Evidence Aggregation, and Final Prediction. diff --git a/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json b/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json new file mode 100644 index 0000000..594d0a5 --- /dev/null +++ b/benchmarks/paper-to-image/cases/001_linear_pipeline/preferences.json @@ -0,0 +1,9 @@ +{ + "summary": "Visual preferences separated from scientific ground truth.", + "aspect_ratio": "16:9", + "language": "English", + "preferred_flow": "left_to_right", + "style_description": "clean academic framework figure", + "preferred_palette": ["blue", "teal", "warm orange accent"], + "avoid": ["commercial poster", "cyberpunk", "generic dashboard"] +} diff --git a/benchmarks/paper-to-image/human_rating_template.json b/benchmarks/paper-to-image/human_rating_template.json new file mode 100644 index 0000000..301efbc --- /dev/null +++ b/benchmarks/paper-to-image/human_rating_template.json @@ -0,0 +1,30 @@ +{ + "summary": "Blinded human rating record for one generated scientific figure.", + "case_id": "", + "candidate_id": "", + "reviewer_id": "anonymous_reviewer_01", + "reviewer_background": "paper_author|domain_expert|visual_design_expert|general_researcher", + "scores_1_to_5": { + "scientific_faithfulness": 0, + "information_completeness": 0, + "information_density_without_clutter": 0, + "reading_order_clarity": 0, + "arrow_and_grouping_clarity": 0, + "text_readability_at_target_size": 0, + "aesthetic_quality": 0, + "academic_professionalism": 0 + }, + "quick_comprehension": { + "input_identified": false, + "output_identified": false, + "core_innovation_identified": false, + "main_flow_direction_identified": false + }, + "hard_errors": [], + "pairwise_preference": { + "other_candidate_id": "", + "preferred": "this|other|tie", + "reason": "" + }, + "notes": "" +} diff --git a/benchmarks/schema/case.schema.json b/benchmarks/schema/case.schema.json new file mode 100644 index 0000000..076f0ae --- /dev/null +++ b/benchmarks/schema/case.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ResearchFigureStudio benchmark case", + "type": "object", + "required": ["summary", "case_id", "suite", "thresholds", "run_config"], + "properties": { + "summary": {"type": "string", "minLength": 1}, + "case_id": {"type": "string", "minLength": 1}, + "suite": {"enum": ["paper-to-image", "image-to-ppt"]}, + "thresholds": {"type": "object"}, + "run_config": {"type": "object"} + }, + "additionalProperties": true +} diff --git a/benchmarks/schema/expected_objects.schema.json b/benchmarks/schema/expected_objects.schema.json new file mode 100644 index 0000000..f0119ea --- /dev/null +++ b/benchmarks/schema/expected_objects.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Image-to-PPT object ground truth", + "type": "object", + "required": ["summary", "objects", "relations"], + "properties": { + "summary": {"type": "string"}, + "objects": {"type": "array", "items": {"type": "object", "required": ["id", "type", "bbox_percent"]}}, + "relations": {"type": "array", "items": {"type": "object", "required": ["source", "target", "type"]}} + } +} diff --git a/benchmarks/schema/expected_semantics.schema.json b/benchmarks/schema/expected_semantics.schema.json new file mode 100644 index 0000000..9d98d43 --- /dev/null +++ b/benchmarks/schema/expected_semantics.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Paper-to-image semantic ground truth", + "type": "object", + "required": ["summary", "entities", "relations", "forbidden_labels"], + "properties": { + "summary": {"type": "string"}, + "entities": {"type": "array", "items": {"type": "object", "required": ["id", "label", "role"]}}, + "relations": {"type": "array", "items": {"type": "object", "required": ["source", "target", "type"]}}, + "forbidden_labels": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/benchmarks/schema/paper_image_review.schema.json b/benchmarks/schema/paper_image_review.schema.json new file mode 100644 index 0000000..2ac8b84 --- /dev/null +++ b/benchmarks/schema/paper_image_review.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Frozen judge or human-calibrated paper-image review", + "type": "object", + "required": ["scientific", "ocr", "information", "clarity", "aesthetic", "stability"], + "properties": { + "scientific": { + "type": "object", + "required": ["score", "missing_modules", "missing_relations", "reversed_relations", "invented_items"] + }, + "ocr": { + "type": "object", + "required": ["score", "missing_labels", "misspelled_labels", "forbidden_labels_found"] + }, + "information": { + "type": "object", + "required": ["coverage", "density", "redundancy", "unsupported_information", "score"] + }, + "clarity": { + "type": "object", + "required": ["reading_order", "arrow_ambiguity", "grouping", "target_size_readability", "cognitive_load", "score"] + }, + "aesthetic": { + "type": "object", + "required": ["hierarchy", "balance", "whitespace", "color", "icon_consistency", "professionalism", "score"] + }, + "stability": { + "type": "object", + "required": ["seed_count", "mean_score", "worst_case_score", "standard_deviation", "production_pass_rate"] + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 54beeea..4a4a19f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,6 +67,7 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. - `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. - `paper-to-editable`: runs both stages and requires a production-approved image unless engineering preview use is explicitly enabled. - Plugin skill: selects and invokes workflows; it does not contain the engine or hardcoded repository paths. +- Benchmark layer: independently scores paper-to-image scientific/visual quality and image-to-PPT fidelity/editability. ## Production rules diff --git a/rfs/cli.py b/rfs/cli.py index 70883dc..6aa90d5 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -9,6 +9,7 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable +from .evaluation import list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case from .paper_to_image import run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro @@ -251,6 +252,15 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--run", required=True, help="Existing co-evolution output directory.") coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") + benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") + benchmark.add_argument("benchmark_action", choices=["list", "validate", "run", "score"]) + benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") + benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") + benchmark.add_argument("--case", help="Benchmark case directory for validate, run, or score.") + benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") + benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") + benchmark.add_argument("--json", action="store_true", help="Emit JSON.") + validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") validate.add_argument("--out", required=True, help="Output directory to validate.") validate.add_argument("--json", action="store_true", help="Emit JSON.") @@ -465,6 +475,21 @@ def main(argv: list[str] | None = None) -> int: ) elif args.command == "coevolution-report": result = analyze_coevolution_run(args.run) + elif args.command == "benchmark": + if args.benchmark_action == "list": + result = list_benchmark_cases(args.root, suite=args.suite) + elif args.benchmark_action == "validate": + if not args.case: + parser.error("benchmark validate requires --case") + result = validate_benchmark_case(args.case) + elif args.benchmark_action == "run": + if not args.case or not args.out: + parser.error("benchmark run requires --case and --out") + result = run_benchmark_case(args.case, args.out) + else: + if not args.case or not args.run_dir: + parser.error("benchmark score requires --case and --run") + result = score_benchmark_case(args.case, args.run_dir, args.out) elif args.command == "validate": result = validate_output(args.out) elif args.command == "presentations-qa": diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index 13488ab..c2c1e3c 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -1,5 +1,12 @@ """Deterministic and model-assisted quality evaluation APIs.""" from .rebuild_visual import run_rebuild_visual_quality_check +from .benchmarking import list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case -__all__ = ["run_rebuild_visual_quality_check"] +__all__ = [ + "list_benchmark_cases", + "run_rebuild_visual_quality_check", + "run_benchmark_case", + "score_benchmark_case", + "validate_benchmark_case", +] diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py new file mode 100644 index 0000000..2190769 --- /dev/null +++ b/rfs/evaluation/benchmarking.py @@ -0,0 +1,470 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +from PIL import Image, ImageChops, ImageFilter, ImageStat +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + +from ..utils import ensure_dir, read_json, write_json, write_text + + +SUITES = {"paper-to-image", "image-to-ppt"} + + +def _load(path: Path) -> dict: + if not path.exists(): + return {} + try: + value = read_json(path) + except Exception: + return {} + return value if isinstance(value, dict) else {} + + +def _mean(values: list[float]) -> float: + return round(sum(values) / max(1, len(values)), 4) + + +def _clamp(value: object) -> float: + try: + return round(max(0.0, min(1.0, float(value))), 4) + except Exception: + return 0.0 + + +def _threshold_failures(metrics: dict, thresholds: dict) -> list[str]: + maximum_metrics = {"hallucination_count", "forbidden_content_count", "full_slide_image_count", "blocking_visual_issue_count"} + failures = [] + for key, threshold in thresholds.items(): + value = metrics.get(key) + if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)): + continue + if key in maximum_metrics and float(value) > float(threshold): + failures.append(f"{key}={value} above maximum {threshold}") + elif key not in maximum_metrics and float(value) < float(threshold): + failures.append(f"{key}={value} below minimum {threshold}") + return failures + + +def validate_benchmark_case(case_dir: str | Path) -> dict[str, Any]: + root = Path(case_dir).resolve() + case = _load(root / "case.json") + errors: list[str] = [] + warnings: list[str] = [] + suite = str(case.get("suite") or "") + if suite not in SUITES: + errors.append(f"case.json suite must be one of {sorted(SUITES)}") + if not str(case.get("case_id") or "").strip(): + errors.append("case.json requires case_id") + if not isinstance(case.get("thresholds"), dict): + errors.append("case.json requires thresholds object") + + required_files = ["case.json"] + if suite == "paper-to-image": + required_files.extend([str(case.get("paper") or "paper.md"), str(case.get("expected_semantics") or "expected_semantics.json")]) + expected = _load(root / str(case.get("expected_semantics") or "expected_semantics.json")) + if not expected.get("entities"): + errors.append("expected_semantics.json requires entities") + if not isinstance(expected.get("relations"), list): + errors.append("expected_semantics.json requires relations list") + if not isinstance(expected.get("forbidden_labels", []), list): + errors.append("expected_semantics.json forbidden_labels must be a list") + elif suite == "image-to-ppt": + required_files.extend([str(case.get("reference_image") or "reference.png"), str(case.get("expected_objects") or "expected_objects.json")]) + expected = _load(root / str(case.get("expected_objects") or "expected_objects.json")) + if not isinstance(expected.get("objects"), list): + errors.append("expected_objects.json requires objects list") + if not isinstance(expected.get("relations", []), list): + errors.append("expected_objects.json relations must be a list") + + for relative in required_files: + if relative and not (root / relative).exists(): + errors.append(f"missing required file: {relative}") + references = case.get("positive_references", []) + case.get("negative_references", []) + for relative in references: + if not (root / str(relative)).exists(): + warnings.append(f"missing optional reference: {relative}") + + return { + "summary": "Benchmark case validation report.", + "ok": not errors, + "case_dir": str(root), + "case_id": case.get("case_id"), + "suite": suite, + "errors": errors, + "warnings": warnings, + } + + +def list_benchmark_cases(benchmarks_root: str | Path, suite: str | None = None) -> dict[str, Any]: + root = Path(benchmarks_root).resolve() + cases = [] + suites = [suite] if suite else sorted(SUITES) + for suite_name in suites: + case_root = root / suite_name / "cases" + if not case_root.exists(): + continue + for case_dir in sorted(path for path in case_root.iterdir() if path.is_dir()): + validation = validate_benchmark_case(case_dir) + cases.append({ + "case_id": validation.get("case_id") or case_dir.name, + "suite": suite_name, + "case_dir": str(case_dir), + "valid": validation["ok"], + "errors": validation["errors"], + }) + return {"summary": "Available ResearchFigureStudio benchmark cases.", "benchmarks_root": str(root), "cases": cases} + + +def _select_candidate_review(run: Path) -> dict: + report = _load(run / "candidate_review.json") + candidates = [item for item in report.get("candidates", []) if isinstance(item, dict)] + selected_id = report.get("selected_candidate_id") + selected = next((item for item in candidates if item.get("candidate_id") == selected_id), None) + if not selected and candidates: + selected = max(candidates, key=lambda item: float(item.get("score") or 0.0)) + return selected.get("review", {}) if isinstance(selected, dict) else {} + + +def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: + case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() + case = _load(case_root / "case.json") + expected = _load(case_root / str(case.get("expected_semantics") or "expected_semantics.json")) + review = _select_candidate_review(run) + benchmark_review = _load(run / "benchmark_review.json") + if benchmark_review: + review = {**review, **benchmark_review} + + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + clarity = review.get("clarity", {}) if isinstance(review.get("clarity"), dict) else {} + information = review.get("information", {}) if isinstance(review.get("information"), dict) else {} + stability = review.get("stability", {}) if isinstance(review.get("stability"), dict) else {} + + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] + missing_entities = scientific.get("missing_modules", []) or benchmark_review.get("missing_entities", []) + missing_relations = scientific.get("missing_relations", []) or benchmark_review.get("missing_relations", []) + invented = scientific.get("invented_items", []) or benchmark_review.get("invented_items", []) + forbidden_found = ocr.get("forbidden_labels_found", []) or benchmark_review.get("forbidden_labels_found", []) + missing_labels = ocr.get("missing_labels", []) or benchmark_review.get("missing_labels", []) + misspelled = ocr.get("misspelled_labels", []) or benchmark_review.get("misspelled_labels", []) + + entity_recall = _clamp((len(expected_entities) - len(missing_entities)) / max(1, len(expected_entities))) + relation_recall = _clamp((len(expected_relations) - len(missing_relations)) / max(1, len(expected_relations))) + exact_label_rate = _clamp((len(expected_entities) - len(missing_labels) - len(misspelled)) / max(1, len(expected_entities))) + scientific_score = _clamp(scientific.get("score", _mean([entity_recall, relation_recall]))) + aesthetic_dimensions = [aesthetic.get(key) for key in ("hierarchy", "balance", "whitespace", "color", "icon_consistency", "readability") if aesthetic.get(key) is not None] + aesthetic_score = _clamp(aesthetic.get("score", _mean([_clamp(value) for value in aesthetic_dimensions]))) + clarity_score = _clamp(clarity.get("score", aesthetic.get("readability", 0.0))) + information_score = _clamp(information.get("score", information.get("coverage", scientific_score))) + stability_score = _clamp(stability.get("production_pass_rate", stability.get("score", 0.0))) + hard_failures = list(review.get("hard_errors", [])) + if not scientific or not aesthetic or scientific.get("engineering_only") or aesthetic.get("engineering_only"): + hard_failures.append("frozen benchmark judge did not provide scientific and aesthetic sections") + if invented: + hard_failures.append("invented scientific content") + if forbidden_found: + hard_failures.append("forbidden labels found") + if relation_recall < 1.0: + hard_failures.append("required relations missing or incorrect") + if exact_label_rate < 1.0: + hard_failures.append("required labels missing or misspelled") + + metrics = { + "entity_recall": entity_recall, + "relation_recall": relation_recall, + "exact_label_rate": exact_label_rate, + "scientific_score": scientific_score, + "information_score": information_score, + "clarity_score": clarity_score, + "aesthetic_score": aesthetic_score, + "stability_score": stability_score, + "hallucination_count": len(invented), + "forbidden_content_count": len(forbidden_found), + } + thresholds = case.get("thresholds", {}) + threshold_failures = _threshold_failures(metrics, thresholds) + total_score = round(0.40 * scientific_score + 0.15 * information_score + 0.15 * clarity_score + 0.20 * aesthetic_score + 0.10 * stability_score, 4) + return { + "summary": "Paper-to-image benchmark score.", + "suite": "paper-to-image", + "case_id": case.get("case_id"), + "case_dir": str(case_root), + "run_dir": str(run), + "metrics": metrics, + "total_score": total_score, + "hard_failures": sorted(set(hard_failures)), + "threshold_failures": threshold_failures, + "passed": not hard_failures and not threshold_failures, + "policy": "scientific, relation, terminology, and hallucination failures cannot be offset by aesthetics", + } + + +def _image_similarity(reference: Path, preview: Path) -> dict[str, float | None]: + if not reference.exists() or not preview.exists(): + return {"pixel_similarity": None, "edge_similarity": None, "color_similarity": None} + with Image.open(reference) as ref_image, Image.open(preview) as out_image: + ref = ref_image.convert("RGB").resize((512, 512), Image.Resampling.LANCZOS) + out = out_image.convert("RGB").resize((512, 512), Image.Resampling.LANCZOS) + difference = ImageChops.difference(ref, out) + rms = math.sqrt(sum(value * value for value in ImageStat.Stat(difference).rms) / 3.0) + pixel_similarity = _clamp(1.0 - rms / 255.0) + ref_edge = ref.convert("L").filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > 32 else 0) + out_edge = out.convert("L").filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > 32 else 0) + edge_diff = ImageChops.difference(ref_edge, out_edge) + edge_similarity = _clamp(1.0 - ImageStat.Stat(edge_diff).mean[0] / 255.0) + ref_mean, out_mean = ImageStat.Stat(ref).mean, ImageStat.Stat(out).mean + color_error = sum(abs(a - b) for a, b in zip(ref_mean, out_mean)) / (3.0 * 255.0) + return {"pixel_similarity": pixel_similarity, "edge_similarity": edge_similarity, "color_similarity": _clamp(1.0 - color_error)} + + +def _ppt_editability(pptx_path: Path) -> dict[str, Any]: + if not pptx_path.exists(): + return {"status": "missing", "full_slide_image_count": 0, "text_shape_count": 0, "connector_count": 0, "shape_count": 0} + presentation = Presentation(str(pptx_path)) + full_slide_images = 0 + text_shapes = 0 + connectors = 0 + shapes = 0 + slide_width, slide_height = float(presentation.slide_width), float(presentation.slide_height) + for slide in presentation.slides: + for shape in slide.shapes: + shapes += 1 + if getattr(shape, "has_text_frame", False) and str(getattr(shape, "text", "")).strip(): + text_shapes += 1 + if shape.shape_type == MSO_SHAPE_TYPE.LINE: + connectors += 1 + if shape.shape_type == MSO_SHAPE_TYPE.PICTURE: + coverage = float(shape.width) * float(shape.height) / max(1.0, slide_width * slide_height) + if coverage >= 0.90: + full_slide_images += 1 + return { + "status": "pass" if full_slide_images == 0 else "blocked", + "full_slide_image_count": full_slide_images, + "text_shape_count": text_shapes, + "connector_count": connectors, + "shape_count": shapes, + } + + +def _bbox_iou(left: dict, right: dict) -> float: + lx0, ly0 = float(left.get("x", 0)), float(left.get("y", 0)) + lx1, ly1 = lx0 + float(left.get("w", 0)), ly0 + float(left.get("h", 0)) + rx0, ry0 = float(right.get("x", 0)), float(right.get("y", 0)) + rx1, ry1 = rx0 + float(right.get("w", 0)), ry0 + float(right.get("h", 0)) + intersection = max(0.0, min(lx1, rx1) - max(lx0, rx0)) * max(0.0, min(ly1, ry1) - max(ly0, ry0)) + union = max(0.000001, float(left.get("w", 0)) * float(left.get("h", 0)) + float(right.get("w", 0)) * float(right.get("h", 0)) - intersection) + return intersection / union + + +def _center_error(left: dict, right: dict) -> float: + lx = float(left.get("x", 0)) + float(left.get("w", 0)) / 2 + ly = float(left.get("y", 0)) + float(left.get("h", 0)) / 2 + rx = float(right.get("x", 0)) + float(right.get("w", 0)) / 2 + ry = float(right.get("y", 0)) + float(right.get("h", 0)) / 2 + return math.sqrt((lx - rx) ** 2 + (ly - ry) ** 2) + + +def _match_expected_objects(expected_objects: list[dict], geometry: dict) -> tuple[list[dict], dict[str, str]]: + actual = [] + for collection, object_type in (("panels", "panel"), ("cards", "card"), ("slots", "asset")): + actual.extend({**item, "benchmark_type": object_type} for item in geometry.get(collection, []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)) + unmatched = list(actual) + matches = [] + id_mapping: dict[str, str] = {} + for expected in expected_objects: + expected_box = expected.get("bbox_percent") if isinstance(expected.get("bbox_percent"), dict) else {} + expected_type = str(expected.get("type") or "") + same_id = next((item for item in unmatched if str(item.get("id")) == str(expected.get("id"))), None) + candidates = [item for item in unmatched if expected_type in {"", str(item.get("benchmark_type"))}] + if not candidates: + candidates = unmatched + chosen = same_id or max(candidates, key=lambda item: _bbox_iou(expected_box, item["bbox_percent"]), default=None) + if not chosen: + matches.append({"expected_id": expected.get("id"), "status": "unmatched", "iou": 0.0, "center_error": 1.0}) + continue + unmatched.remove(chosen) + iou = _bbox_iou(expected_box, chosen["bbox_percent"]) + center_error = _center_error(expected_box, chosen["bbox_percent"]) + id_mapping[str(expected.get("id"))] = str(chosen.get("id")) + matches.append({ + "expected_id": expected.get("id"), + "actual_id": chosen.get("id"), + "expected_type": expected_type, + "actual_type": chosen.get("benchmark_type"), + "iou": round(iou, 4), + "center_error": round(center_error, 4), + "status": "matched" if iou >= 0.30 else "low_overlap", + }) + return matches, id_mapping + + +def score_image_to_ppt(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: + case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() + case = _load(case_root / "case.json") + expected = _load(case_root / str(case.get("expected_objects") or "expected_objects.json")) + reference = case_root / str(case.get("reference_image") or "reference.png") + preview = run / str(case.get("preview") or "rebuild_preview.png") + pptx = run / str(case.get("pptx") or "editable_composition.pptx") + similarity = _image_similarity(reference, preview) + editability = _ppt_editability(pptx) + visual = _load(run / "rebuild_visual_quality_report.json") + semantic = _load(run / "semantic_binding_report.json") + composition = _load(run / "composition_quality_report.json") + alignment = _load(run / "text_alignment_report.json") + geometry = _load(run / "reference_geometry.json") + + expected_objects = [item for item in expected.get("objects", []) if isinstance(item, dict)] + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict)] + object_matches, id_mapping = _match_expected_objects(expected_objects, geometry) + object_coverage = _clamp(sum(1 for item in object_matches if item["status"] == "matched") / max(1, len(expected_objects))) if expected_objects else 1.0 + mean_object_iou = _mean([float(item["iou"]) for item in object_matches]) if object_matches else 1.0 + mean_center_error = _mean([float(item["center_error"]) for item in object_matches]) if object_matches else 0.0 + actual_relations = { + (str(item.get("source_id") or item.get("source") or ""), str(item.get("target_id") or item.get("target") or "")) + for item in composition.get("arrows", []) + if isinstance(item, dict) + } + matched_relations = 0 + for relation in expected_relations: + source = id_mapping.get(str(relation.get("source"))) + target = id_mapping.get(str(relation.get("target"))) + if source and target and (source, target) in actual_relations: + matched_relations += 1 + relation_coverage = _clamp(matched_relations / max(1, len(expected_relations))) if expected_relations else 1.0 + max_text_delta = max((float(item.get("center_delta_percent") or 0.0) for item in alignment.get("items", []) if isinstance(item, dict)), default=0.0) + text_alignment_score = _clamp(1.0 - max_text_delta / 0.10) + blocking_visual_issues = int(visual.get("blocking_issue_count") or 0) + semantic_binding_score = _clamp(semantic.get("mapped_entity_count", 0) / max(1, semantic.get("entity_count", 0))) if semantic else 1.0 + editable_score = 0.0 if editability["status"] == "blocked" else _clamp((min(1, editability["text_shape_count"]) + min(1, editability["connector_count"]) + 1) / 3) + fidelity_values = [value for value in similarity.values() if isinstance(value, float)] + render_fidelity = _mean(fidelity_values) if fidelity_values else 0.0 + metrics = { + **similarity, + "render_fidelity": render_fidelity, + "object_coverage": object_coverage, + "mean_object_iou": mean_object_iou, + "mean_center_error": mean_center_error, + "relation_coverage": relation_coverage, + "text_alignment_score": text_alignment_score, + "semantic_binding_score": semantic_binding_score, + "editability_score": editable_score, + "full_slide_image_count": editability["full_slide_image_count"], + "blocking_visual_issue_count": blocking_visual_issues, + } + hard_failures = [] + if editability["full_slide_image_count"]: + hard_failures.append("full-slide reference image detected") + if blocking_visual_issues: + hard_failures.append("blocking visual issues remain") + thresholds = case.get("thresholds", {}) + threshold_failures = _threshold_failures(metrics, thresholds) + total_score = round(0.40 * render_fidelity + 0.20 * object_coverage + 0.15 * relation_coverage + 0.10 * text_alignment_score + 0.15 * editable_score, 4) + return { + "summary": "Image-to-editable-PPT benchmark score.", + "suite": "image-to-ppt", + "case_id": case.get("case_id"), + "case_dir": str(case_root), + "run_dir": str(run), + "metrics": metrics, + "editability": editability, + "object_matches": object_matches, + "total_score": total_score, + "hard_failures": hard_failures, + "threshold_failures": threshold_failures, + "passed": not hard_failures and not threshold_failures, + "limitations": [ + "pixel, edge, and color similarity are baseline metrics; calibrated SSIM/LPIPS and region annotations remain future work", + "mutation-based editability testing requires a dedicated render-after-edit harness", + ], + } + + +def score_benchmark_case(case_dir: str | Path, run_dir: str | Path, out: str | Path | None = None) -> dict[str, Any]: + validation = validate_benchmark_case(case_dir) + if not validation["ok"]: + return {**validation, "passed": False} + result = score_paper_to_image(case_dir, run_dir) if validation["suite"] == "paper-to-image" else score_image_to_ppt(case_dir, run_dir) + target = ensure_dir(out or run_dir) + write_json(target / "benchmark_result.json", result) + lines = [ + "# Benchmark Result", + "", + f"- Suite: {result.get('suite')}", + f"- Case: {result.get('case_id')}", + f"- Passed: {result.get('passed')}", + f"- Total score: {result.get('total_score')}", + "", + "## Metrics", + ] + lines.extend(f"- {key}: {value}" for key, value in result.get("metrics", {}).items()) + lines.extend(["", "## Hard failures"]) + lines.extend(f"- {item}" for item in result.get("hard_failures", [])) + lines.extend(["", "## Threshold failures"]) + lines.extend(f"- {item}" for item in result.get("threshold_failures", [])) + write_text(target / "benchmark_result.md", "\n".join(lines) + "\n") + return result + + +def run_benchmark_case(case_dir: str | Path, out: str | Path) -> dict[str, Any]: + validation = validate_benchmark_case(case_dir) + if not validation["ok"]: + return {**validation, "passed": False} + root = Path(case_dir).resolve() + target = ensure_dir(out).resolve() + case = _load(root / "case.json") + config = case.get("run_config", {}) if isinstance(case.get("run_config"), dict) else {} + if validation["suite"] == "paper-to-image": + from ..paper_to_image import run_paper_to_image + + workflow_result = run_paper_to_image( + paper=root / str(case.get("paper") or "paper.md"), + out=target, + preferences_path=(root / str(case["preferences"])) if case.get("preferences") else None, + positive_references=[str(root / str(path)) for path in case.get("positive_references", [])], + negative_references=[str(root / str(path)) for path in case.get("negative_references", [])], + planner_mode=str(config.get("planner_mode") or "heuristic"), + asset_mode=str(config.get("image_asset_mode") or "placeholder"), + candidates=int(config.get("image_candidates") or 1), + aspect_ratio=str(config.get("aspect_ratio") or "auto"), + language=str(config.get("language") or "English"), + review_mode=str(config.get("review_mode") or "heuristic"), + domain_profile=str(config.get("domain_profile") or "auto"), + template=str(config.get("template") or "auto"), + repair_rounds=int(config.get("repair_rounds") or 0), + ocr_engine=str(config.get("ocr_engine") or "off"), + ocr_lang=str(config.get("ocr_lang") or "en_ch"), + ) + else: + from ..editable_rebuild import rebuild_editable + + workflow_result = rebuild_editable( + reference=root / str(case.get("reference_image") or "reference.png"), + out=target, + asset_mode=str(config.get("asset_mode") or "placeholder"), + asset_policy=str(config.get("asset_policy") or "smart-api"), + text_mode=str(config.get("text_mode") or "off"), + layout_mode=str(config.get("layout_mode") or "heuristic"), + control_mode=str(config.get("control_mode") or "heuristic"), + design_plan_mode=str(config.get("design_plan_mode") or "heuristic"), + export_preview=True, + ocr_engine=str(config.get("ocr_engine") or "off"), + ) + score = score_benchmark_case(root, target, target) + result = { + "summary": "Benchmark workflow and scoring completed.", + "ok": bool(workflow_result.get("ok")), + "suite": validation["suite"], + "case_id": validation["case_id"], + "out_dir": str(target), + "workflow_result": workflow_result, + "benchmark_result": score, + } + write_json(target / "benchmark_run.json", result) + return result diff --git a/skills/research-figure-studio/SKILL.md b/skills/research-figure-studio/SKILL.md index f5bc4af..5d5719c 100644 --- a/skills/research-figure-studio/SKILL.md +++ b/skills/research-figure-studio/SKILL.md @@ -78,3 +78,15 @@ The normal paper-to-editable output should contain: - `editable/rebuild_preview.png` when preview export is requested If production gates fail, return the engineering artifacts and the blocker; do not claim that the figure is ready for paper submission. + +## Benchmark commands + +Use the two independent suites to locate failures before changing the workflow: + +```powershell +rfs benchmark list --root benchmarks --json +rfs benchmark run --case --out --json +rfs benchmark score --case --run --json +``` + +`paper-to-image` scientific and terminology errors are hard failures. `image-to-ppt` must reject full-slide reference images even when raster similarity is high. diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py new file mode 100644 index 0000000..24de27e --- /dev/null +++ b/tests/unit/test_benchmarking.py @@ -0,0 +1,78 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from PIL import Image +from pptx import Presentation +from pptx.util import Inches + +from rfs.evaluation.benchmarking import ( + list_benchmark_cases, + score_benchmark_case, + validate_benchmark_case, +) + + +ROOT = Path(__file__).resolve().parents[2] + + +class BenchmarkingTests(unittest.TestCase): + def test_bundled_cases_validate_and_list(self): + p2i = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + i2p = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" + + self.assertTrue(validate_benchmark_case(p2i)["ok"]) + self.assertTrue(validate_benchmark_case(i2p)["ok"]) + listing = list_benchmark_cases(ROOT / "benchmarks") + self.assertEqual(len(listing["cases"]), 2) + + def test_paper_to_image_score_uses_hard_scientific_gates(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.95, "review": review}], + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertTrue(result["passed"]) + self.assertEqual(result["metrics"]["relation_recall"], 1.0) + + def test_full_slide_picture_is_detected_as_editability_cheating(self): + case_dir = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + reference = case_dir / "reference.ppm" + with Image.open(reference) as image: + image.save(run / "rebuild_preview.png") + image.save(run / "reference.png") + prs = Presentation() + prs.slide_width = Inches(10) + prs.slide_height = Inches(5.625) + slide = prs.slides.add_slide(prs.slide_layouts[6]) + slide.shapes.add_picture(str(run / "reference.png"), 0, 0, width=prs.slide_width, height=prs.slide_height) + prs.save(run / "editable_composition.pptx") + (run / "composition_quality_report.json").write_text(json.dumps({"slots": [], "cards": [], "arrows": []}), encoding="utf-8") + (run / "rebuild_visual_quality_report.json").write_text(json.dumps({"blocking_issue_count": 0}), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertEqual(result["metrics"]["full_slide_image_count"], 1) + self.assertIn("full-slide reference image detected", result["hard_failures"]) + + +if __name__ == "__main__": + unittest.main() From 09a933b202141c5c204981bbe2f8793ecf80c2e2 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Tue, 21 Jul 2026 16:18:51 +0800 Subject: [PATCH 15/74] Add top-conference paper benchmark cases --- .gitignore | 1 + benchmarks/README.md | 3 + .../cases/101_vit_linear/case.json | 15 ++ .../101_vit_linear/expected_semantics.json | 25 +++ .../cases/101_vit_linear/source.json | 12 ++ .../cases/102_mask_rcnn_branch/case.json | 14 ++ .../expected_semantics.json | 24 +++ .../cases/102_mask_rcnn_branch/source.json | 8 + .../cases/103_self_refine_feedback/case.json | 8 + .../expected_semantics.json | 25 +++ .../103_self_refine_feedback/source.json | 7 + .../cases/104_imagebind_multimodal/case.json | 8 + .../expected_semantics.json | 27 ++++ .../104_imagebind_multimodal/source.json | 8 + .../cases/105_sam_dense_multiframe/case.json | 8 + .../expected_semantics.json | 32 ++++ .../105_sam_dense_multiframe/source.json | 8 + .../preferences/top_conference.json | 9 ++ rfs/cli.py | 11 +- rfs/evaluation/__init__.py | 3 +- rfs/evaluation/benchmarking.py | 152 +++++++++++++++++- skills/research-figure-studio/SKILL.md | 1 + tests/unit/test_benchmarking.py | 39 ++++- 23 files changed, 441 insertions(+), 7 deletions(-) create mode 100644 benchmarks/paper-to-image/cases/101_vit_linear/case.json create mode 100644 benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/101_vit_linear/source.json create mode 100644 benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json create mode 100644 benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json create mode 100644 benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json create mode 100644 benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json create mode 100644 benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json create mode 100644 benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json create mode 100644 benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json create mode 100644 benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json create mode 100644 benchmarks/paper-to-image/preferences/top_conference.json diff --git a/.gitignore b/.gitignore index 913afc4..0045bb2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ tmp/codex-presentations/ tmp/pdfs/*/ tmp/pdfs/ tmp/paper-to-editable-smoke*/ +benchmarks/**/inputs/ *.key *.pem diff --git a/benchmarks/README.md b/benchmarks/README.md index ef4298b..1d44455 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -21,6 +21,7 @@ A full-slide copy of the reference image is a hard failure even if pixel similar ```powershell rfs benchmark list --root benchmarks --json rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipeline --json +rfs benchmark fetch --case benchmarks/paper-to-image/cases/101_vit_linear --json rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json ``` @@ -31,6 +32,8 @@ Each case contains `case.json` plus suite-specific human-authored ground truth. Generated runs and reports belong under `output/benchmarks/` and are not source fixtures. +Real-paper cases commit `source.json` and human-authored Ground Truth, while `benchmark fetch` downloads the PDF into an ignored local `inputs/` directory. This keeps the public repository reproducible without redistributing publisher files. + ## Evaluation tiers - Offline contract tier: placeholder assets, deterministic validation, CI-safe. diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/case.json b/benchmarks/paper-to-image/cases/101_vit_linear/case.json new file mode 100644 index 0000000..3ac8ca6 --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ICLR 2021 Vision Transformer case for a clean linear architecture figure.", + "case_id": "101_vit_linear", + "suite": "paper-to-image", + "tier": "real-paper-offline-contract", + "topology": "linear_pipeline", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json new file mode 100644 index 0000000..dcad857 --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json @@ -0,0 +1,25 @@ +{ + "summary": "Human-authored semantic contract for the Vision Transformer architecture.", + "entities": [ + {"id": "input_image", "label": "Input Image", "role": "input", "required": true}, + {"id": "image_patches", "label": "Image Patches", "role": "module", "required": true}, + {"id": "linear_projection", "label": "Linear Projection of Flattened Patches", "role": "module", "required": true}, + {"id": "position_embedding", "label": "Position Embedding", "role": "module", "required": true}, + {"id": "class_token", "label": "Class Token", "role": "module", "required": true}, + {"id": "transformer_encoder", "label": "Transformer Encoder", "role": "module", "required": true}, + {"id": "mlp_head", "label": "MLP Head", "role": "module", "required": true}, + {"id": "class_prediction", "label": "Class Prediction", "role": "output", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "image_patches", "type": "data_flow", "required": true}, + {"source": "image_patches", "target": "linear_projection", "type": "data_flow", "required": true}, + {"source": "linear_projection", "target": "transformer_encoder", "type": "data_flow", "required": true}, + {"source": "position_embedding", "target": "transformer_encoder", "type": "conditioning", "required": true}, + {"source": "class_token", "target": "transformer_encoder", "type": "conditioning", "required": true}, + {"source": "transformer_encoder", "target": "mlp_head", "type": "data_flow", "required": true}, + {"source": "mlp_head", "target": "class_prediction", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["CNN Backbone", "Recurrent Network", "Decoder", "Object Detection Head"], + "expected_reading_order": ["input_image", "image_patches", "linear_projection", "transformer_encoder", "mlp_head", "class_prediction"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 0, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/source.json b/benchmarks/paper-to-image/cases/101_vit_linear/source.json new file mode 100644 index 0000000..0b126fc --- /dev/null +++ b/benchmarks/paper-to-image/cases/101_vit_linear/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale", + "venue": "ICLR", + "year": 2021, + "arxiv_id": "2010.11929", + "official_url": "https://openreview.net/forum?id=YicbFdNTTy", + "paper_urls": ["https://arxiv.org/pdf/2010.11929", "https://export.arxiv.org/pdf/2010.11929"], + "target_figure": "Figure 1", + "verified_caption": "Model overview. The image is split into fixed-size patches, linearly embedded, combined with position embeddings and processed by a Transformer encoder.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json new file mode 100644 index 0000000..771007a --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json @@ -0,0 +1,14 @@ +{ + "summary": "ICCV 2017 Mask R-CNN case for shared-backbone multi-head branching.", + "case_id": "102_mask_rcnn_branch", + "suite": "paper-to-image", + "tier": "real-paper-offline-contract", + "topology": "branching_architecture", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json new file mode 100644 index 0000000..c2187d0 --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json @@ -0,0 +1,24 @@ +{ + "summary": "Human-authored semantic contract for Mask R-CNN.", + "entities": [ + {"id": "input_image", "label": "Input Image", "role": "input", "required": true}, + {"id": "backbone", "label": "Backbone", "role": "module", "required": true}, + {"id": "region_proposals", "label": "Region Proposals", "role": "module", "required": true}, + {"id": "roi_align", "label": "RoIAlign", "role": "module", "required": true}, + {"id": "classification_head", "label": "Classification", "role": "output", "required": true}, + {"id": "box_head", "label": "Bounding-box Regression", "role": "output", "required": true}, + {"id": "mask_head", "label": "Mask Branch", "role": "output", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "backbone", "type": "data_flow", "required": true}, + {"source": "backbone", "target": "region_proposals", "type": "data_flow", "required": true}, + {"source": "region_proposals", "target": "roi_align", "type": "data_flow", "required": true}, + {"source": "backbone", "target": "roi_align", "type": "feature_flow", "required": true}, + {"source": "roi_align", "target": "classification_head", "type": "branch", "required": true}, + {"source": "roi_align", "target": "box_head", "type": "branch", "required": true}, + {"source": "roi_align", "target": "mask_head", "type": "branch", "required": true} + ], + "forbidden_labels": ["Semantic Segmentation Only", "Single Output Head", "Transformer Decoder"], + "expected_reading_order": ["input_image", "backbone", "region_proposals", "roi_align"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 3, "convergence_count": 1, "feedback_loop_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json new file mode 100644 index 0000000..06453b2 --- /dev/null +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Mask R-CNN", "venue": "ICCV", "year": 2017, "arxiv_id": "1703.06870", + "official_url": "https://openaccess.thecvf.com/content_ICCV_2017/html/He_Mask_R-CNN_ICCV_2017_paper.html", + "paper_urls": ["https://arxiv.org/pdf/1703.06870", "https://export.arxiv.org/pdf/1703.06870"], + "target_figure": "Figure 1", "verified_caption": "The Mask R-CNN framework for instance segmentation.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json new file mode 100644 index 0000000..9c3d8e0 --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json @@ -0,0 +1,8 @@ +{ + "summary": "NeurIPS 2023 Self-Refine case for an explicit iterative feedback loop.", + "case_id": "103_self_refine_feedback", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "feedback_loop", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json new file mode 100644 index 0000000..8a10482 --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json @@ -0,0 +1,25 @@ +{ + "summary": "Human-authored semantic contract for Self-Refine.", + "entities": [ + {"id": "task_input", "label": "Input", "role": "input", "required": true}, + {"id": "initial_generator", "label": "Generate", "role": "module", "required": true}, + {"id": "initial_output", "label": "Initial Output", "role": "module", "required": true}, + {"id": "feedback_module", "label": "Feedback", "role": "module", "required": true}, + {"id": "feedback_text", "label": "Feedback", "role": "control", "required": true}, + {"id": "refinement_module", "label": "Refine", "role": "module", "required": true}, + {"id": "refined_output", "label": "Refined Output", "role": "output", "required": true} + ], + "relations": [ + {"source": "task_input", "target": "initial_generator", "type": "data_flow", "required": true}, + {"source": "initial_generator", "target": "initial_output", "type": "data_flow", "required": true}, + {"source": "initial_output", "target": "feedback_module", "type": "evaluation", "required": true}, + {"source": "feedback_module", "target": "feedback_text", "type": "feedback", "required": true}, + {"source": "initial_output", "target": "refinement_module", "type": "revision_input", "required": true}, + {"source": "feedback_text", "target": "refinement_module", "type": "feedback", "required": true}, + {"source": "refinement_module", "target": "refined_output", "type": "data_flow", "required": true}, + {"source": "refined_output", "target": "feedback_module", "type": "feedback_loop", "required": true} + ], + "forbidden_labels": ["Gradient Update", "Human Annotator Required", "External Reward Model", "Supervised Fine-tuning"], + "expected_reading_order": ["task_input", "initial_generator", "initial_output", "feedback_module", "refinement_module", "refined_output"], + "topology_requirements": {"primary_flow": "cyclic", "feedback_loop_count": 1, "iteration_must_be_visible": true} +} diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json new file mode 100644 index 0000000..b0c34da --- /dev/null +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/source.json @@ -0,0 +1,7 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Self-Refine: Iterative Refinement with Self-Feedback", "venue": "NeurIPS", "year": 2023, "arxiv_id": "2303.17651", + "official_url": "https://arxiv.org/abs/2303.17651", "paper_urls": ["https://arxiv.org/pdf/2303.17651", "https://export.arxiv.org/pdf/2303.17651"], + "target_figure": "Figure 1", "verified_caption": "Iterative generation, feedback and refinement without additional supervised training.", + "license_note": "Use the arXiv license shown on the source page; downloaded inputs remain local." +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json new file mode 100644 index 0000000..9531de9 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json @@ -0,0 +1,8 @@ +{ + "summary": "CVPR 2023 ImageBind case for six-modality shared-embedding alignment.", + "case_id": "104_imagebind_multimodal", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "multimodal_system", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json new file mode 100644 index 0000000..93cd3c5 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json @@ -0,0 +1,27 @@ +{ + "summary": "Human-authored semantic contract for ImageBind.", + "entities": [ + {"id": "image", "label": "Image", "role": "input", "required": true}, + {"id": "text", "label": "Text", "role": "input", "required": true}, + {"id": "audio", "label": "Audio", "role": "input", "required": true}, + {"id": "depth", "label": "Depth", "role": "input", "required": true}, + {"id": "thermal", "label": "Thermal", "role": "input", "required": true}, + {"id": "imu", "label": "IMU", "role": "input", "required": true}, + {"id": "modality_encoders", "label": "Modality Encoders", "role": "module", "required": true}, + {"id": "joint_embedding", "label": "Joint Embedding Space", "role": "module", "required": true}, + {"id": "emergent_alignment", "label": "Emergent Cross-modal Alignment", "role": "output", "required": true} + ], + "relations": [ + {"source": "image", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "text", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "audio", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "depth", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "thermal", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "imu", "target": "modality_encoders", "type": "encoding", "required": true}, + {"source": "modality_encoders", "target": "joint_embedding", "type": "alignment", "required": true}, + {"source": "joint_embedding", "target": "emergent_alignment", "type": "enables", "required": true} + ], + "forbidden_labels": ["All Modality Pairs Are Directly Paired", "Single Shared Raw Encoder", "Late Fusion Classifier"], + "expected_reading_order": ["image", "text", "audio", "depth", "thermal", "imu", "modality_encoders", "joint_embedding", "emergent_alignment"], + "topology_requirements": {"primary_flow": "many_to_one_to_many", "input_modality_count": 6, "shared_space_required": true} +} diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json new file mode 100644 index 0000000..726abe3 --- /dev/null +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "ImageBind: One Embedding Space To Bind Them All", "venue": "CVPR", "year": 2023, "arxiv_id": "2305.05665", + "official_url": "https://openaccess.thecvf.com/content/CVPR2023/html/Girdhar_ImageBind_One_Embedding_Space_To_Bind_Them_All_CVPR_2023_paper.html", + "paper_urls": ["https://arxiv.org/pdf/2305.05665", "https://export.arxiv.org/pdf/2305.05665"], + "target_figure": "Figure 2", "verified_caption": "Different naturally aligned modality pairs are linked through image data into a common embedding space.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json new file mode 100644 index 0000000..dbe7ad7 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json @@ -0,0 +1,8 @@ +{ + "summary": "ICCV 2023 Segment Anything case for a dense three-part task-model-data-engine overview.", + "case_id": "105_sam_dense_multiframe", "suite": "paper-to-image", "tier": "real-paper-offline-contract", "topology": "dense_multi_panel", + "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", + "positive_references": [], "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json new file mode 100644 index 0000000..394caaa --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json @@ -0,0 +1,32 @@ +{ + "summary": "Human-authored semantic contract for the Segment Anything overview.", + "entities": [ + {"id": "promptable_task", "label": "Promptable Segmentation Task", "role": "panel", "required": true}, + {"id": "image", "label": "Image", "role": "input", "required": true}, + {"id": "prompt", "label": "Prompt", "role": "input", "required": true}, + {"id": "valid_mask", "label": "Valid Segmentation Mask", "role": "output", "required": true}, + {"id": "sam_model", "label": "Segment Anything Model", "role": "panel", "required": true}, + {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, + {"id": "prompt_encoder", "label": "Prompt Encoder", "role": "module", "required": true}, + {"id": "mask_decoder", "label": "Mask Decoder", "role": "module", "required": true}, + {"id": "data_engine", "label": "Data Engine", "role": "panel", "required": true}, + {"id": "assisted_manual", "label": "Assisted-manual", "role": "module", "required": true}, + {"id": "semi_automatic", "label": "Semi-automatic", "role": "module", "required": true}, + {"id": "fully_automatic", "label": "Fully Automatic", "role": "module", "required": true}, + {"id": "sa1b", "label": "SA-1B", "role": "output", "required": true} + ], + "relations": [ + {"source": "image", "target": "image_encoder", "type": "data_flow", "required": true}, + {"source": "prompt", "target": "prompt_encoder", "type": "data_flow", "required": true}, + {"source": "image_encoder", "target": "mask_decoder", "type": "feature_flow", "required": true}, + {"source": "prompt_encoder", "target": "mask_decoder", "type": "conditioning", "required": true}, + {"source": "mask_decoder", "target": "valid_mask", "type": "data_flow", "required": true}, + {"source": "assisted_manual", "target": "semi_automatic", "type": "stage_transition", "required": true}, + {"source": "semi_automatic", "target": "fully_automatic", "type": "stage_transition", "required": true}, + {"source": "fully_automatic", "target": "sa1b", "type": "data_generation", "required": true}, + {"source": "sam_model", "target": "data_engine", "type": "annotation_support", "required": true} + ], + "forbidden_labels": ["Text-to-Image Generation", "Object Detector Only", "Single-stage Manual Annotation", "Language Model"], + "expected_panels": ["promptable_task", "sam_model", "data_engine"], + "topology_requirements": {"panel_count": 3, "dense_multi_panel": true, "feedback_between_model_and_data_engine": true} +} diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json new file mode 100644 index 0000000..1065723 --- /dev/null +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/source.json @@ -0,0 +1,8 @@ +{ + "summary": "Publication metadata and local-fetch sources.", + "paper_title": "Segment Anything", "venue": "ICCV", "year": 2023, "arxiv_id": "2304.02643", + "official_url": "https://openaccess.thecvf.com/content/ICCV2023/html/Kirillov_Segment_Anything_ICCV_2023_paper.html", + "paper_urls": ["https://arxiv.org/pdf/2304.02643", "https://export.arxiv.org/pdf/2304.02643"], + "target_figure": "Figure 1", "verified_caption": "Three interconnected components: promptable segmentation task, Segment Anything Model and data engine.", + "license_note": "Verify redistribution rights; keep downloaded inputs local." +} diff --git a/benchmarks/paper-to-image/preferences/top_conference.json b/benchmarks/paper-to-image/preferences/top_conference.json new file mode 100644 index 0000000..bbdf8c7 --- /dev/null +++ b/benchmarks/paper-to-image/preferences/top_conference.json @@ -0,0 +1,9 @@ +{ + "summary": "Shared visual preferences for computer-science top-conference benchmark cases.", + "aspect_ratio": "16:9", + "language": "English", + "style_description": "clean top-conference method figure, compact academic layout, restrained color, clear arrows and grouping", + "preferred_palette": ["cool blue", "teal", "muted orange accent", "light neutral background"], + "visual_density": "high but readable", + "avoid": ["commercial poster", "photorealistic decoration", "cyberpunk", "generic robot imagery", "fake equations", "fake charts", "long paragraphs"] +} diff --git a/rfs/cli.py b/rfs/cli.py index 6aa90d5..c123543 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -9,7 +9,7 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable -from .evaluation import list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case +from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case from .paper_to_image import run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro @@ -253,12 +253,13 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") - benchmark.add_argument("benchmark_action", choices=["list", "validate", "run", "score"]) + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "run", "score"]) benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") - benchmark.add_argument("--case", help="Benchmark case directory for validate, run, or score.") + benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") + benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") benchmark.add_argument("--json", action="store_true", help="Emit JSON.") validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") @@ -482,6 +483,10 @@ def main(argv: list[str] | None = None) -> int: if not args.case: parser.error("benchmark validate requires --case") result = validate_benchmark_case(args.case) + elif args.benchmark_action == "fetch": + if not args.case: + parser.error("benchmark fetch requires --case") + result = fetch_benchmark_case(args.case, force=args.force) elif args.benchmark_action == "run": if not args.case or not args.out: parser.error("benchmark run requires --case and --out") diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index c2c1e3c..7e86ec0 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -1,10 +1,11 @@ """Deterministic and model-assisted quality evaluation APIs.""" from .rebuild_visual import run_rebuild_visual_quality_check -from .benchmarking import list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case +from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case __all__ = [ "list_benchmark_cases", + "fetch_benchmark_case", "run_rebuild_visual_quality_check", "run_benchmark_case", "score_benchmark_case", diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 2190769..b38cf54 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -1,13 +1,16 @@ from __future__ import annotations +import hashlib import json import math +import re from pathlib import Path from typing import Any from PIL import Image, ImageChops, ImageFilter, ImageStat from pptx import Presentation from pptx.enum.shapes import MSO_SHAPE_TYPE +import requests from ..utils import ensure_dir, read_json, write_json, write_text @@ -37,7 +40,13 @@ def _clamp(value: object) -> float: def _threshold_failures(metrics: dict, thresholds: dict) -> list[str]: - maximum_metrics = {"hallucination_count", "forbidden_content_count", "full_slide_image_count", "blocking_visual_issue_count"} + maximum_metrics = { + "hallucination_count", + "forbidden_content_count", + "plan_forbidden_content_count", + "full_slide_image_count", + "blocking_visual_issue_count", + } failures = [] for key, threshold in thresholds.items(): value = metrics.get(key) @@ -50,6 +59,70 @@ def _threshold_failures(metrics: dict, thresholds: dict) -> list[str]: return failures +def _normalized_text(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + + +def _score_planning_contract(expected: dict, specification: dict) -> dict[str, Any]: + if not specification: + return {"available": False} + actual_entities = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for index, item in enumerate(specification.get(field, []) if isinstance(specification.get(field), list) else []): + if not isinstance(item, dict): + continue + label = next((str(item.get(key) or "").strip() for key in ("visible_label", "name", "text", "statement", "label", "id") if str(item.get(key) or "").strip()), "") + actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label)}) + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] + mapping: dict[str, str] = {} + matched = [] + used_actual_ids: set[str] = set() + for item in expected_entities: + candidates = [str(item.get("label") or "")] + [str(value) for value in item.get("aliases", []) if str(value).strip()] + normalized = [_normalized_text(value) for value in candidates if _normalized_text(value)] + match = next( + ( + actual + for actual in actual_entities + if actual["id"] not in used_actual_ids + and actual["normalized"] + and any( + value == actual["normalized"] + or value in actual["normalized"] + or actual["normalized"] in value + for value in normalized + ) + ), + None, + ) + if match: + mapping[str(item.get("id"))] = match["id"] + used_actual_ids.add(match["id"]) + matched.append(str(item.get("id"))) + expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] + actual_relations = { + (str(item.get("source") or item.get("source_id") or ""), str(item.get("target") or item.get("target_id") or "")) + for item in specification.get("relations", []) + if isinstance(item, dict) + } + relation_matches = 0 + for relation in expected_relations: + source = mapping.get(str(relation.get("source"))) + target = mapping.get(str(relation.get("target"))) + if source and target and (source, target) in actual_relations: + relation_matches += 1 + actual_text = " ".join(item["label"] for item in actual_entities).casefold() + forbidden = [label for label in expected.get("forbidden_labels", []) if str(label).casefold() in actual_text] + return { + "available": True, + "entity_recall": _clamp(len(matched) / max(1, len(expected_entities))), + "relation_recall": _clamp(relation_matches / max(1, len(expected_relations))), + "matched_entity_ids": matched, + "missing_entity_ids": [str(item.get("id")) for item in expected_entities if str(item.get("id")) not in matched], + "forbidden_labels_found": forbidden, + } + + def validate_benchmark_case(case_dir: str | Path) -> dict[str, Any]: root = Path(case_dir).resolve() case = _load(root / "case.json") @@ -65,7 +138,14 @@ def validate_benchmark_case(case_dir: str | Path) -> dict[str, Any]: required_files = ["case.json"] if suite == "paper-to-image": - required_files.extend([str(case.get("paper") or "paper.md"), str(case.get("expected_semantics") or "expected_semantics.json")]) + paper_relative = str(case.get("paper") or "paper.md") + required_files.append(str(case.get("expected_semantics") or "expected_semantics.json")) + if not (root / paper_relative).exists(): + source_path = root / str(case.get("source") or "source.json") + if source_path.exists(): + warnings.append(f"paper input not fetched yet: {paper_relative}") + else: + errors.append(f"missing required file: {paper_relative}; no source.json is available") expected = _load(root / str(case.get("expected_semantics") or "expected_semantics.json")) if not expected.get("entities"): errors.append("expected_semantics.json requires entities") @@ -100,6 +180,55 @@ def validate_benchmark_case(case_dir: str | Path) -> dict[str, Any]: } +def fetch_benchmark_case(case_dir: str | Path, force: bool = False) -> dict[str, Any]: + root = Path(case_dir).resolve() + case = _load(root / "case.json") + source_path = root / str(case.get("source") or "source.json") + source = _load(source_path) + if str(case.get("suite") or "") != "paper-to-image": + return {"summary": "Benchmark source fetch is only required for paper-to-image cases.", "ok": True, "case_dir": str(root), "status": "not_required"} + if not source: + return {"summary": "Benchmark source metadata is missing.", "ok": False, "case_dir": str(root), "status": "missing_source", "error": str(source_path)} + target = root / str(case.get("paper") or "inputs/paper.pdf") + if target.exists() and not force: + return { + "summary": "Benchmark paper source already exists.", + "ok": True, + "case_dir": str(root), + "status": "reused", + "paper": str(target), + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + } + urls = source.get("paper_urls") if isinstance(source.get("paper_urls"), list) else [] + errors = [] + for url in urls: + try: + response = requests.get(str(url), headers={"User-Agent": "ResearchFigureStudio benchmark fetch/0.1"}, timeout=120) + response.raise_for_status() + content = response.content + if not content.startswith(b"%PDF"): + raise ValueError("downloaded response is not a PDF") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + report = { + "summary": "Benchmark paper source fetched for local use.", + "ok": True, + "case_dir": str(root), + "status": "downloaded", + "paper": str(target), + "source_url": str(url), + "bytes": len(content), + "sha256": digest, + "license_note": source.get("license_note"), + } + write_json(target.parent / "fetch_report.json", report) + return report + except Exception as exc: + errors.append({"url": str(url), "error": str(exc)}) + return {"summary": "Unable to fetch benchmark paper source.", "ok": False, "case_dir": str(root), "status": "failed", "errors": errors} + + def list_benchmark_cases(benchmarks_root: str | Path, suite: str | None = None) -> dict[str, Any]: root = Path(benchmarks_root).resolve() cases = [] @@ -145,6 +274,7 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, clarity = review.get("clarity", {}) if isinstance(review.get("clarity"), dict) else {} information = review.get("information", {}) if isinstance(review.get("information"), dict) else {} stability = review.get("stability", {}) if isinstance(review.get("stability"), dict) else {} + planning_contract = _score_planning_contract(expected, _load(run / "figure_specification.json")) expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] @@ -175,6 +305,12 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, hard_failures.append("required relations missing or incorrect") if exact_label_rate < 1.0: hard_failures.append("required labels missing or misspelled") + if planning_contract.get("available") and float(planning_contract.get("entity_recall", 0.0)) < 1.0: + hard_failures.append("paper planning missed required benchmark entities") + if planning_contract.get("available") and float(planning_contract.get("relation_recall", 0.0)) < 1.0: + hard_failures.append("paper planning missed required benchmark relations") + if planning_contract.get("forbidden_labels_found"): + hard_failures.append("paper planning introduced forbidden benchmark labels") metrics = { "entity_recall": entity_recall, @@ -187,6 +323,9 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "stability_score": stability_score, "hallucination_count": len(invented), "forbidden_content_count": len(forbidden_found), + "plan_entity_recall": planning_contract.get("entity_recall"), + "plan_relation_recall": planning_contract.get("relation_recall"), + "plan_forbidden_content_count": len(planning_contract.get("forbidden_labels_found", [])), } thresholds = case.get("thresholds", {}) threshold_failures = _threshold_failures(metrics, thresholds) @@ -198,6 +337,7 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "case_dir": str(case_root), "run_dir": str(run), "metrics": metrics, + "planning_contract": planning_contract, "total_score": total_score, "hard_failures": sorted(set(hard_failures)), "threshold_failures": threshold_failures, @@ -413,6 +553,14 @@ def score_benchmark_case(case_dir: str | Path, run_dir: str | Path, out: str | P def run_benchmark_case(case_dir: str | Path, out: str | Path) -> dict[str, Any]: + case_root = Path(case_dir).resolve() + case_preflight = _load(case_root / "case.json") + if str(case_preflight.get("suite") or "") == "paper-to-image": + paper_path = case_root / str(case_preflight.get("paper") or "paper.md") + if not paper_path.exists(): + fetched = fetch_benchmark_case(case_root) + if not fetched.get("ok"): + return {**fetched, "passed": False} validation = validate_benchmark_case(case_dir) if not validation["ok"]: return {**validation, "passed": False} diff --git a/skills/research-figure-studio/SKILL.md b/skills/research-figure-studio/SKILL.md index 5d5719c..35cee63 100644 --- a/skills/research-figure-studio/SKILL.md +++ b/skills/research-figure-studio/SKILL.md @@ -85,6 +85,7 @@ Use the two independent suites to locate failures before changing the workflow: ```powershell rfs benchmark list --root benchmarks --json +rfs benchmark fetch --case --json rfs benchmark run --case --out --json rfs benchmark score --case --run --json ``` diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 24de27e..8825e4b 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -25,7 +25,14 @@ def test_bundled_cases_validate_and_list(self): self.assertTrue(validate_benchmark_case(p2i)["ok"]) self.assertTrue(validate_benchmark_case(i2p)["ok"]) listing = list_benchmark_cases(ROOT / "benchmarks") - self.assertEqual(len(listing["cases"]), 2) + self.assertGreaterEqual(len(listing["cases"]), 7) + + def test_real_paper_case_is_valid_with_source_metadata(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "101_vit_linear" + validation = validate_benchmark_case(case_dir) + + self.assertTrue(validation["ok"]) + self.assertTrue((case_dir / "source.json").exists()) def test_paper_to_image_score_uses_hard_scientific_gates(self): case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" @@ -50,6 +57,36 @@ def test_paper_to_image_score_uses_hard_scientific_gates(self): self.assertTrue(result["passed"]) self.assertEqual(result["metrics"]["relation_recall"], 1.0) + def test_paper_to_image_score_checks_planning_contract(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.95, "review": review}], + }), encoding="utf-8") + (run / "figure_specification.json").write_text(json.dumps({ + "modules": [{"id": "only_module", "name": "Feature Encoder"}], + "relations": [], + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertLess(result["metrics"]["plan_entity_recall"], 1.0) + self.assertEqual(result["metrics"]["plan_relation_recall"], 0.0) + self.assertIn("paper planning missed required benchmark entities", result["hard_failures"]) + self.assertIn("paper planning missed required benchmark relations", result["hard_failures"]) + def test_full_slide_picture_is_detected_as_editability_cheating(self): case_dir = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" with tempfile.TemporaryDirectory() as tmp: From 2ef221d3352b19dfea3bcb13e3a4cc96c58dd63b Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Tue, 21 Jul 2026 16:50:24 +0800 Subject: [PATCH 16/74] Add structured PDF document extraction --- rfs/paper_to_image/analyzer.py | 557 ++++++++++++++++++++++++------ tests/unit/test_paper_analyzer.py | 64 ++++ 2 files changed, 508 insertions(+), 113 deletions(-) create mode 100644 tests/unit/test_paper_analyzer.py diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 6980b77..41ede06 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -1,167 +1,498 @@ from __future__ import annotations +import hashlib import re +import shutil +import subprocess +import tempfile +import time +import unicodedata +from difflib import SequenceMatcher from pathlib import Path -from typing import Any +from typing import Any, Callable + + +MOJIBAKE_MARKERS = ("�", "Ã", "Â", "鈭", "鈥", "尾", "卤", "绗", "鍥", "琛") +SECTION_ALIASES = { + "abstract": ("abstract",), + "introduction": ("introduction", "background"), + "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework"), + "experiments": ("experiment", "experiments", "evaluation", "results"), + "conclusion": ("conclusion", "conclusions", "discussion"), +} def _clean(text: str) -> str: - return re.sub(r"[ \t]+", " ", str(text or "")).strip() + value = unicodedata.normalize("NFKC", str(text or "")).replace("\u00ad", "") + value = value.replace("\r\n", "\n").replace("\r", "\n") + value = re.sub(r"[ \t]+", " ", value) + value = re.sub(r"\n{3,}", "\n\n", value) + return value.strip() -def _read_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any]], str]: - suffix = path.suffix.lower() - pages: list[dict[str, Any]] = [] - loader = "plain" - if suffix == ".pdf": - loader = "pymupdf" +def _normalized_compare_text(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "", _clean(text).casefold()) + + +def _replacement_rate(text: str) -> float: + return round(text.count("�") / max(1, len(text)), 6) + + +def _mojibake_rate(text: str) -> float: + return round(sum(text.count(marker) for marker in MOJIBAKE_MARKERS) / max(1, len(text)), 6) + + +def _block_kind(text: str, width_ratio: float = 0.0) -> str: + value = _clean(text) + if re.match(r"^(figure|fig\.|table)\s*[a-z0-9]+", value, re.IGNORECASE): + return "caption" if value.casefold().startswith(("figure", "fig.")) else "table" + if re.match(r"^(abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)(.{2,100})$", value, re.IGNORECASE) and len(value) <= 140: + return "heading" + if len(value) <= 180 and width_ratio >= 0.55 and not value.endswith((".", ",", ";")): + return "title" + if re.search(r"(?:[A-Za-z][A-Za-z0-9_{}^\-]*\s*=\s*[^=\n]{3,}|∑|∫|≤|≥|≈)", value): + return "formula" + return "paragraph" + + +def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[list[dict[str, Any]], float]: + if not blocks: + return [], 0.0 + usable = [item for item in blocks if _clean(item.get("text", ""))] + if not usable: + return [], 0.0 + midpoint = page_width / 2.0 + spanning = [item for item in usable if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.62] + left = [item for item in usable if item not in spanning and (float(item["bbox"][0]) + float(item["bbox"][2])) / 2 <= midpoint] + right = [item for item in usable if item not in spanning and item not in left] + two_column = len(left) >= 2 and len(right) >= 2 + if not two_column: + ordered = sorted(usable, key=lambda item: (round(float(item["bbox"][1]), 1), float(item["bbox"][0]))) + confidence = 0.96 + else: + boundaries = sorted(spanning, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) + ordered = [] + cursor = float("-inf") + for boundary in boundaries + [None]: + limit = float(boundary["bbox"][1]) if boundary else float("inf") + segment = [item for item in usable if item not in spanning and cursor <= float(item["bbox"][1]) < limit] + ordered.extend(sorted((item for item in segment if item in left), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) + ordered.extend(sorted((item for item in segment if item in right), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) + if boundary: + ordered.append(boundary) + cursor = float(boundary["bbox"][3]) + seen: set[int] = set() + ordered = [item for item in ordered if not (id(item) in seen or seen.add(id(item)))] + confidence = 0.88 if spanning else 0.82 + for index, item in enumerate(ordered, 1): + item["reading_order"] = index + return ordered, confidence + + +def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: + records = [] + payload = page.get_text("dict") + for block in payload.get("blocks", []): + if int(block.get("type", 0)) != 0: + continue + for line in block.get("lines", []): + spans = [span for span in line.get("spans", []) if _clean(span.get("text", ""))] + if not spans: + continue + spans.sort(key=lambda span: float(span.get("bbox", [0, 0, 0, 0])[0])) + text = _clean(" ".join(str(span.get("text") or "") for span in spans)) + bbox_values = [span.get("bbox", [0, 0, 0, 0]) for span in spans] + bbox = [ + round(min(float(value[0]) for value in bbox_values), 3), + round(min(float(value[1]) for value in bbox_values), 3), + round(max(float(value[2]) for value in bbox_values), 3), + round(max(float(value[3]) for value in bbox_values), 3), + ] + records.append({"bbox": bbox, "text": text, "source": "pymupdf", "confidence": 1.0}) + return records + + +def _poppler_pages(path: Path, timeout: int = 30) -> tuple[list[str], str | None]: + executable = shutil.which("pdftotext") + if not executable: + return [], "pdftotext unavailable" + try: + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / "paper.txt" + subprocess.run([executable, "-layout", str(path), str(target)], check=True, timeout=timeout, capture_output=True) + return target.read_text(encoding="utf-8", errors="replace").split("\f"), None + except Exception as exc: + return [], str(exc) + + +def _pdf_metadata(path: Path) -> dict[str, Any]: + try: + from pypdf import PdfReader + + reader = PdfReader(str(path)) + if reader.is_encrypted: + try: + unlocked = reader.decrypt("") + except Exception: + unlocked = 0 + if not unlocked: + raise ValueError("PDF is encrypted and cannot be opened without a password") + return {"page_count": len(reader.pages), "encrypted": bool(reader.is_encrypted), "metadata": {str(key): str(value) for key, value in (reader.metadata or {}).items()}} + except ImportError: + return {"page_count": None, "encrypted": None, "metadata": {}, "warning": "pypdf unavailable"} + + +def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None) -> tuple[list[dict[str, Any]], str]: + if adapter: + return list(adapter(image_path, lang) or []), "adapter" + from ..reference_text_extractor import run_easyocr, run_paddle_ocr + + requested = engine + if requested == "auto": + requested = "easyocr" try: - import fitz - - document = fitz.open(str(path)) - used = 0 - for index, page in enumerate(document, 1): - text = page.get_text("text") - remaining = max_chars - used - if remaining <= 0: - break - text = text[:remaining] - pages.append({"page": index, "text": text}) - used += len(text) - document.close() - except Exception as exc: - raise RuntimeError(f"PDF extraction failed: {exc}") from exc - elif suffix == ".docx": + import easyocr # noqa: F401 + except Exception: + requested = "paddle" + if requested == "easyocr": + return run_easyocr(image_path, lang), "easyocr" + if requested == "paddle": + return run_paddle_ocr(image_path, lang), "paddle" + return [], "off" + + +def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None) -> tuple[list[dict[str, Any]], str, str | None]: + try: + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / f"page_{page_number:03d}.png" + pixmap = page.get_pixmap(dpi=200, alpha=False) + pixmap.save(str(target)) + records, used_engine = _ocr_records(target, engine, lang, adapter) + blocks = [] + scale_x = float(page.rect.width) / max(1, pixmap.width) + scale_y = float(page.rect.height) / max(1, pixmap.height) + for record in records: + quad = record.get("quad") or [] + if not quad: + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + blocks.append({ + "bbox": [min(xs) * scale_x, min(ys) * scale_y, max(xs) * scale_x, max(ys) * scale_y], + "text": _clean(record.get("text", "")), + "source": used_engine, + "confidence": round(float(record.get("confidence") or 0.0), 4), + }) + return blocks, used_engine, None + except Exception as exc: + return [], engine, str(exc) + + +def _read_pdf_pages( + path: Path, + max_chars: int, + deadline_at: float | None, + ocr_engine: str, + ocr_lang: str, + ocr_adapter: Callable | None, + max_ocr_pages: int, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + try: + import fitz + except Exception as exc: + raise RuntimeError(f"PyMuPDF is required for PDF extraction: {exc}") from exc + + metadata = _pdf_metadata(path) + started = time.monotonic() + document = fitz.open(str(path)) + poppler_text, poppler_warning = _poppler_pages(path) + pages: list[dict[str, Any]] = [] + warnings: list[str] = [poppler_warning] if poppler_warning else [] + used_chars = 0 + ocr_candidates: list[int] = [] + try: + for index, page in enumerate(document, 1): + raw_blocks = _pymupdf_line_blocks(page) + ordered, order_confidence = _reading_order(raw_blocks, float(page.rect.width)) + for block_index, block in enumerate(ordered, 1): + block["id"] = f"P{index:03d}_B{block_index:03d}" + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) + page_text = "\n\n".join(item["text"] for item in ordered) + poppler_page = poppler_text[index - 1] if index - 1 < len(poppler_text) else "" + native_norm = _normalized_compare_text(page_text) + poppler_norm = _normalized_compare_text(poppler_page) + agreement = round(SequenceMatcher(None, native_norm[:20000], poppler_norm[:20000]).ratio(), 4) if native_norm and poppler_norm else None + abnormal = len(native_norm) < 80 or _replacement_rate(page_text) > 0.005 or _mojibake_rate(page_text) > 0.005 or (agreement is not None and agreement < 0.35) + if abnormal: + ocr_candidates.append(index) + pages.append({ + "page": index, + "width": round(float(page.rect.width), 3), + "height": round(float(page.rect.height), 3), + "blocks": ordered, + "text": page_text, + "char_count": len(page_text), + "replacement_character_rate": _replacement_rate(page_text), + "mojibake_rate": _mojibake_rate(page_text), + "poppler_agreement": agreement, + "reading_order_confidence": order_confidence, + "used_ocr": False, + }) + used_chars += len(page_text) + if used_chars >= max_chars and "max_chars reached; remaining pages retain metadata but not full evidence text" not in warnings: + warnings.append("max_chars reached; remaining pages retain metadata but not full evidence text") + finally: + pass + + ocr_pages: list[int] = [] + if ocr_engine != "off": + prioritized = ocr_candidates[:max_ocr_pages] + for page_number in prioritized: + if deadline_at is not None and time.monotonic() >= deadline_at - 20: + warnings.append("OCR stopped to preserve deadline validation budget") + break + page = document[page_number - 1] + blocks, used_engine, error = _ocr_page(page, page_number, ocr_engine, ocr_lang, ocr_adapter) + if error: + warnings.append(f"page {page_number} OCR unavailable: {error}") + continue + ordered, order_confidence = _reading_order(blocks, float(page.rect.width)) + for block_index, block in enumerate(ordered, 1): + block["id"] = f"P{page_number:03d}_B{block_index:03d}" + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) + ocr_text = "\n\n".join(item["text"] for item in ordered) + current = pages[page_number - 1] + if len(_normalized_compare_text(ocr_text)) > len(_normalized_compare_text(current["text"])): + current.update({ + "blocks": ordered, + "text": ocr_text, + "char_count": len(ocr_text), + "replacement_character_rate": _replacement_rate(ocr_text), + "mojibake_rate": _mojibake_rate(ocr_text), + "reading_order_confidence": order_confidence, + "used_ocr": True, + "ocr_engine": used_engine, + }) + ocr_pages.append(page_number) + document.close() + elapsed = round(time.monotonic() - started, 3) + return pages, { + "metadata": metadata, + "ocr_candidate_pages": ocr_candidates, + "ocr_pages": ocr_pages, + "warnings": warnings, + "poppler_available": bool(poppler_text), + "elapsed_seconds": elapsed, + } + + +def _read_non_pdf_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any]], str]: + suffix = path.suffix.lower() + if suffix == ".docx": loader = "python-docx" try: import docx document = docx.Document(str(path)) text = "\n".join(paragraph.text for paragraph in document.paragraphs)[:max_chars] - pages.append({"page": 1, "text": text}) except Exception as exc: raise RuntimeError(f"DOCX extraction failed: {exc}") from exc else: - text = path.read_text(encoding="utf-8", errors="ignore")[:max_chars] - pages.append({"page": 1, "text": text}) - return pages, loader - - -def _chunk_page(text: str, target_chars: int = 2200) -> list[str]: - paragraphs = [_clean(item) for item in re.split(r"\n\s*\n", text) if _clean(item)] - if not paragraphs: - paragraphs = [_clean(item) for item in text.splitlines() if _clean(item)] - chunks: list[str] = [] - current: list[str] = [] - size = 0 + loader = "plain" + text = path.read_text(encoding="utf-8", errors="replace")[:max_chars] + clean = _clean(text) + paragraphs = [_clean(value) for value in re.split(r"\n\s*\n", clean) if _clean(value)] + chunks = [] for paragraph in paragraphs: - if current and size + len(paragraph) > target_chars: - chunks.append("\n".join(current)) - current = [] - size = 0 - current.append(paragraph) - size += len(paragraph) + 1 - if current: - chunks.append("\n".join(current)) - return chunks - - -def _heading_candidates(text: str) -> list[str]: - headings: list[str] = [] - for raw in text.splitlines(): - line = _clean(raw) - if not 3 <= len(line) <= 100: - continue - if re.match(r"^(?:\d+(?:\.\d+)*[.)]?\s+|[IVX]+[.)]\s+|第[一二三四五六七八九十]+[章节])", line): - value = re.sub(r"^(?:\d+(?:\.\d+)*[.)]?\s+|[IVX]+[.)]\s+)", "", line).strip() - if value and value not in headings: - headings.append(value) - return headings[:30] + lines = paragraph.splitlines() + if len(lines) > 1 and re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)", lines[0], re.IGNORECASE): + chunks.extend([_clean(lines[0]), _clean("\n".join(lines[1:]))]) + else: + chunks.append(paragraph) + blocks = [] + for index, chunk in enumerate(chunks or [clean], 1): + blocks.append({"id": f"P001_B{index:03d}", "bbox": None, "text": chunk, "kind": _block_kind(chunk), "source": loader, "confidence": 1.0, "reading_order": index}) + return [{"page": 1, "width": None, "height": None, "blocks": blocks, "text": clean, "char_count": len(clean), "replacement_character_rate": _replacement_rate(clean), "mojibake_rate": _mojibake_rate(clean), "poppler_agreement": None, "reading_order_confidence": 1.0, "used_ocr": False}], loader -def _document_index(pages: list[dict[str, Any]], headings: list[str]) -> dict: - sections = [] - formulas = [] - tables = [] +def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: + headings = [] + seen = set() + for page in pages: + for block in page.get("blocks", []): + lines = [_clean(value) for value in str(block.get("text", "")).splitlines() if _clean(value)] + for line_index, line in enumerate(lines): + if not 2 <= len(line) <= 140: + continue + explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) + numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) + known = any(re.match(rf"^{re.escape(alias)}\b", line, re.IGNORECASE) for aliases in SECTION_ALIASES.values() for alias in aliases) + if numbered: + title_candidate = numbered.group(2).strip() + first_alpha = next((char for char in title_candidate if char.isalpha()), "") + if not first_alpha or not first_alpha.isupper() or re.search(r"[=±∑∫]", title_candidate): + numbered = None + if not (explicit or numbered or known): + continue + normalized = numbered.group(2).strip() if numbered else line.strip() + key = normalized.casefold() + if normalized and key not in seen: + seen.add(key) + headings.append({"id": f"section_{len(headings) + 1:03d}", "title": normalized, "page": page["page"], "block_id": block["id"], "line_index": line_index}) + return headings[:80] + + +def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, Any]: figures = [] - seen_sections = set() - formula_pattern = re.compile(r"(?:[A-Za-zΑ-Ωα-ω][A-Za-z0-9_{}^\-]*\s*=\s*[^=\n]{3,}|[∑∫∂∇≈≤≥]\s*[^\n]{3,}|\([^\n]{2,80}\)\s*\(\d+\)\s*$)") - table_pattern = re.compile(r"^(?:Table|TABLE|表)\s*[A-Za-z0-9一二三四五六七八九十.:: -]+", re.IGNORECASE) - figure_pattern = re.compile(r"^(?:Figure|FIGURE|Fig\.|图)\s*[A-Za-z0-9一二三四五六七八九十.:: -]+", re.IGNORECASE) + tables = [] + formulas = [] for page in pages: - for raw in page.get("text", "").splitlines(): - line = _clean(raw) - if not line: - continue - for heading in headings: - if heading.lower() in line.lower() and heading not in seen_sections: - sections.append({"id": f"section_{len(sections) + 1:03d}", "title": heading, "page": page["page"]}) - seen_sections.add(heading) - break - if len(formulas) < 120 and 4 <= len(line) <= 240 and formula_pattern.search(line): - formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "text": line}) - if len(tables) < 80 and table_pattern.match(line): - tables.append({"id": f"table_{len(tables) + 1:03d}", "page": page["page"], "caption": line}) - if len(figures) < 80 and figure_pattern.match(line): - figures.append({"id": f"figure_{len(figures) + 1:03d}", "page": page["page"], "caption": line}) - return { - "summary": "Page-aware section, formula, table-caption, and figure-caption index.", - "sections": sections, - "formulas": formulas, - "tables": tables, - "figures": figures, - } + for block in page.get("blocks", []): + record = {"page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "caption": block.get("text", "")} + if block.get("kind") == "caption": + figures.append({"id": f"figure_{len(figures) + 1:03d}", **record}) + elif block.get("kind") == "table": + tables.append({"id": f"table_{len(tables) + 1:03d}", **record}) + elif block.get("kind") == "formula": + formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "text": block.get("text", "")}) + return {"summary": "Page-aware section, formula, table-caption, and figure-caption index.", "sections": sections, "formulas": formulas[:120], "tables": tables[:80], "figures": figures[:80]} -def parse_paper(path: str | Path, max_chars: int = 90000) -> dict: - source = Path(path).resolve() - if not source.exists(): - raise FileNotFoundError(f"Paper does not exist: {source}") - pages, loader = _read_pages(source, max_chars=max_chars) - evidence: list[dict[str, Any]] = [] +def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, bool]: + candidates = ("\n".join(item["title"] for item in sections) + "\n" + "\n".join(page.get("text", "") for page in pages)).casefold() + return {name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) for alias in aliases) for name, aliases in SECTION_ALIASES.items()} + + +def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]: + section_positions = sorted(sections, key=lambda item: (item["page"], item.get("block_id", ""))) + evidence = [] + used = 0 for page in pages: - for chunk in _chunk_page(page["text"]): - if not chunk: + for block in page.get("blocks", []): + text = _clean(block.get("text", "")) + if not text or used >= max_chars: continue - evidence_id = f"E{len(evidence) + 1:04d}" + text = text[: max(0, max_chars - used)] + section = next((item["title"] for item in reversed(section_positions) if item["page"] <= page["page"]), None) evidence.append({ - "id": evidence_id, + "id": f"E{len(evidence) + 1:04d}", "page": page["page"], - "section_hint": None, - "text": chunk, - "char_count": len(chunk), + "block_id": block.get("id"), + "bbox": block.get("bbox"), + "section_hint": section, + "kind": block.get("kind"), + "source": block.get("source"), + "confidence": block.get("confidence", 1.0), + "text": text, + "char_count": len(text), }) + used += len(text) + return evidence + + +def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[str, Any], details: dict[str, Any]) -> dict[str, Any]: + readable = [page for page in pages if len(_normalized_compare_text(page.get("text", ""))) >= 80] + readable_ratio = round(len(readable) / max(1, len(pages)), 4) + ocr_count = sum(1 for page in pages if page.get("used_ocr")) + pdf_type = "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" + coverage = _section_coverage(pages, index.get("sections", [])) + warnings = list(details.get("warnings", [])) + if readable_ratio < 0.6: + warnings.append("fewer than 60% of pages contain reliable text") + if not coverage.get("abstract") or not coverage.get("method"): + warnings.append("abstract or method-like section was not confidently located") + agreements = [page["poppler_agreement"] for page in pages if isinstance(page.get("poppler_agreement"), (int, float))] + report_status = "fail" if readable_ratio < 0.6 else "warning" if warnings else "pass" + return { + "summary": "PDF extraction quality and fallback report.", + "status": report_status, + "source_path": str(source), + "pdf_type": pdf_type, + "page_count": len(pages), + "readable_page_count": len(readable), + "readable_page_ratio": readable_ratio, + "empty_or_low_text_pages": [page["page"] for page in pages if page not in readable], + "ocr_candidate_pages": details.get("ocr_candidate_pages", []), + "ocr_pages": details.get("ocr_pages", []), + "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), + "mojibake_rate": round(sum(page.get("mojibake_rate", 0.0) for page in pages) / max(1, len(pages)), 6), + "cross_extractor_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, + "reading_order_confidence": round(sum(page.get("reading_order_confidence", 0.0) for page in pages) / max(1, len(pages)), 4), + "section_coverage": coverage, + "figure_caption_count": len(index.get("figures", [])), + "table_caption_count": len(index.get("tables", [])), + "poppler_available": details.get("poppler_available", False), + "warnings": warnings, + "elapsed_seconds": details.get("elapsed_seconds", 0.0), + } - all_text = "\n".join(page["text"] for page in pages) - headings = _heading_candidates(all_text) - document_index = _document_index(pages, headings) - for item in evidence: - low = item["text"].lower() - item["section_hint"] = next((heading for heading in headings if heading.lower() in low), None) +def parse_paper( + path: str | Path, + max_chars: int = 90000, + deadline_at: float | None = None, + ocr_engine: str = "off", + ocr_lang: str = "en_ch", + ocr_adapter: Callable | None = None, + max_ocr_pages: int = 6, +) -> dict[str, Any]: + source = Path(path).resolve() + if not source.exists(): + raise FileNotFoundError(f"Paper does not exist: {source}") + source_hash = hashlib.sha256(source.read_bytes()).hexdigest() + if source.suffix.lower() == ".pdf": + pages, details = _read_pdf_pages(source, max_chars, deadline_at, ocr_engine, ocr_lang, ocr_adapter, max_ocr_pages) + loader = "structured-pdf" + else: + pages, loader = _read_non_pdf_pages(source, max_chars) + details = {"warnings": [], "ocr_candidate_pages": [], "ocr_pages": [], "poppler_available": False, "elapsed_seconds": 0.0} + sections = _heading_candidates(pages) + document_index = _document_index(pages, sections) + evidence = _build_evidence(pages, sections, max_chars) + all_text = "\n\n".join(page.get("text", "") for page in pages) if not evidence: raise ValueError("No readable paper text was extracted") + extraction_report = _extraction_report(source, pages, document_index, details) return { - "summary": "Paper parsed into page-aware evidence chunks.", + "summary": "Paper parsed into a structured, page-aware evidence model.", "source_path": str(source), "source_name": source.name, "source_type": source.suffix.lower(), + "source_sha256": source_hash, "loader": loader, "page_count": len(pages), "char_count": len(all_text), - "headings": headings, + "headings": [item["title"] for item in sections], + "sections": sections, + "pages": pages, "document_index": document_index, "evidence": evidence, + "extraction_report": extraction_report, } +def paper_markdown(parsed: dict[str, Any]) -> str: + parts = [] + for page in parsed.get("pages", []): + parts.append(f"\n\n{page.get('text', '')}") + return "\n\n".join(parts).strip() + "\n" + + def evidence_excerpt(parsed: dict, max_chars: int = 58000) -> str: + prioritized = sorted( + parsed.get("evidence", []), + key=lambda item: ( + 0 if any(term in str(item.get("section_hint") or "").casefold() for term in ("abstract", "method", "approach", "architecture", "framework", "conclusion")) else 1, + int(item.get("page") or 0), + str(item.get("block_id") or ""), + ), + ) lines: list[str] = [] used = 0 - for item in parsed.get("evidence", []): - block = f"[{item['id']} | page {item['page']} | {item.get('section_hint') or 'unknown section'}]\n{item['text']}" + for item in prioritized: + block = f"[{item['id']} | page {item['page']} | {item.get('section_hint') or 'unknown section'} | {item.get('block_id') or 'no block'}]\n{item['text']}" if used + len(block) > max_chars: - break + continue lines.append(block) used += len(block) return "\n\n".join(lines) diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py new file mode 100644 index 0000000..b605843 --- /dev/null +++ b/tests/unit/test_paper_analyzer.py @@ -0,0 +1,64 @@ +import tempfile +import unittest +from pathlib import Path + +import fitz + +from rfs.paper_to_image.analyzer import parse_paper + + +class PaperAnalyzerTests(unittest.TestCase): + def _pdf(self, draw) -> Path: + root = Path(tempfile.mkdtemp()) + target = root / "paper.pdf" + document = fitz.open() + page = document.new_page(width=600, height=800) + draw(page) + document.save(target) + document.close() + self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) + return target + + def test_structured_pdf_preserves_page_blocks_and_captions(self): + def draw(page): + page.insert_text((50, 60), "1 Introduction", fontsize=14) + page.insert_textbox((50, 90, 550, 180), "A readable scientific paper paragraph with enough text for extraction quality checks.", fontsize=10) + page.insert_textbox((50, 220, 550, 280), "Figure 1: Overview of the proposed framework.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + + self.assertEqual(parsed["page_count"], 1) + self.assertTrue(parsed["source_sha256"]) + self.assertTrue(parsed["pages"][0]["blocks"]) + self.assertEqual(parsed["document_index"]["figures"][0]["page"], 1) + self.assertIn("Introduction", parsed["headings"]) + self.assertTrue(all(item.get("block_id") for item in parsed["evidence"])) + + def test_two_column_blocks_are_sorted_left_then_right(self): + def draw(page): + page.insert_textbox((330, 100, 560, 180), "RIGHT COLUMN SECOND with enough words to form a complete paragraph.", fontsize=10) + page.insert_textbox((40, 100, 270, 180), "LEFT COLUMN FIRST with enough words to form a complete paragraph.", fontsize=10) + page.insert_textbox((40, 40, 560, 75), "2 Method", fontsize=14) + page.insert_textbox((330, 220, 560, 300), "RIGHT COLUMN FOURTH with enough words to form a complete paragraph.", fontsize=10) + page.insert_textbox((40, 220, 270, 300), "LEFT COLUMN THIRD with enough words to form a complete paragraph.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + text = parsed["pages"][0]["text"] + + self.assertLess(text.index("LEFT COLUMN FIRST"), text.index("LEFT COLUMN THIRD")) + self.assertLess(text.index("LEFT COLUMN THIRD"), text.index("RIGHT COLUMN SECOND")) + self.assertGreaterEqual(parsed["pages"][0]["reading_order_confidence"], 0.8) + + def test_unicode_normalization_keeps_math_symbols(self): + def draw(page): + page.insert_text((50, 60), "3 Method", fontsize=14) + page.insert_textbox((50, 100, 550, 180), "Model accuracy is approximately 95 percent and beta parameters are preserved.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + + self.assertEqual(parsed["extraction_report"]["replacement_character_rate"], 0.0) + self.assertEqual(parsed["extraction_report"]["mojibake_rate"], 0.0) + + +if __name__ == "__main__": + unittest.main() From 18be9a85ea84de12cb234c5dc5e1697c2fa5d519 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Tue, 21 Jul 2026 17:01:37 +0800 Subject: [PATCH 17/74] Add PDF extraction quality gates and OCR fallback --- pyproject.toml | 2 + rfs/cli.py | 29 ++++++++- rfs/paper_to_image/__init__.py | 5 +- rfs/paper_to_image/analyzer.py | 44 ++++++++++++- rfs/paper_to_image/inspection.py | 78 ++++++++++++++++++++++++ tests/integration/test_paper_to_image.py | 7 +++ tests/unit/test_paper_analyzer.py | 27 ++++++++ 7 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 rfs/paper_to_image/inspection.py diff --git a/pyproject.toml b/pyproject.toml index a78f4df..ea2fe29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ dependencies = [ "pillow", "python-pptx", "PyMuPDF", + "pypdf", + "pdfplumber", "requests", "opencv-python-headless", "pywin32; platform_system == 'Windows'" diff --git a/rfs/cli.py b/rfs/cli.py index c123543..ce635e0 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -1,8 +1,10 @@ from __future__ import annotations import argparse +import importlib.util import json import os +import shutil import sys from pathlib import Path @@ -10,7 +12,7 @@ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case -from .paper_to_image import run_paper_to_image +from .paper_to_image import inspect_paper, run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro from .professional_repair import vlm_professional_repair_adapter @@ -28,7 +30,7 @@ def _json_print(data: dict) -> None: def _doctor() -> dict: deps = {} - for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: + for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("pypdf", "pypdf"), ("pdfplumber", "pdfplumber"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: try: __import__(module) deps[name] = {"available": True} @@ -61,6 +63,12 @@ def _doctor() -> dict: "RFS_IMAGE_EDIT_URL": {"present": env_present("RFS_IMAGE_EDIT_URL"), "value": os.getenv("RFS_IMAGE_EDIT_URL") if env_present("RFS_IMAGE_EDIT_URL") else None}, "MODEL_VLM": {"present": env_present("MODEL_VLM"), "value": os.getenv("MODEL_VLM") if env_present("MODEL_VLM") else None}, } + optional_pdf = { + "pdftotext": {"available": bool(shutil.which("pdftotext")), "path": shutil.which("pdftotext")}, + "pdftoppm": {"available": bool(shutil.which("pdftoppm")), "path": shutil.which("pdftoppm")}, + "easyocr": {"available": importlib.util.find_spec("easyocr") is not None}, + "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, + } ok = all(item["available"] for item in deps.values()) return { "summary": "ResearchFigureStudio doctor report.", @@ -68,6 +76,7 @@ def _doctor() -> dict: "version": __version__, "python": sys.executable, "dependencies": deps, + "pdf_tools": optional_pdf, "powerpoint": powerpnt, "auth": auth, "notes": [ @@ -94,6 +103,14 @@ def build_parser() -> argparse.ArgumentParser: doctor = sub.add_parser("doctor", help="Check dependencies, PowerPoint, and auth env vars.") doctor.add_argument("--json", action="store_true", help="Emit JSON.") + inspect_pdf = sub.add_parser("inspect-pdf", help="Extract and diagnose a paper without calling a model.") + inspect_pdf.add_argument("--paper", required=True, help="Paper PDF or supported document path.") + inspect_pdf.add_argument("--out", required=True, help="Output directory for the document model and extraction report.") + inspect_pdf.add_argument("--deadline", type=int, default=180, help="Soft processing deadline in seconds. Default: 180.") + inspect_pdf.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "off"], default="auto") + inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") + make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") make.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") make.add_argument("--reference", required=True, help="User-provided visual reference image path.") @@ -313,6 +330,14 @@ def main(argv: list[str] | None = None) -> int: try: if args.command == "doctor": result = _doctor() + elif args.command == "inspect-pdf": + result = inspect_paper( + paper=args.paper, + out=args.out, + deadline_seconds=args.deadline, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + ) elif args.command == "make-framework": result = make_framework( paper=args.paper, diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index 0836ed1..3f10052 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,5 +1,6 @@ -"""Paper-to-image workflow without PPTX generation.""" +"""Paper analysis and paper-to-image workflows.""" +from .inspection import inspect_paper from .workflow import run_paper_to_image -__all__ = ["run_paper_to_image"] +__all__ = ["inspect_paper", "run_paper_to_image"] diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 41ede06..744f033 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -125,6 +125,36 @@ def _poppler_pages(path: Path, timeout: int = 30) -> tuple[list[str], str | None return [], str(exc) +def _pdfplumber_blocks(path: Path, page_number: int) -> list[dict[str, Any]]: + try: + import pdfplumber + + with pdfplumber.open(str(path)) as document: + page = document.pages[page_number - 1] + words = page.extract_words(use_text_flow=False, keep_blank_chars=False) or [] + rows: list[list[dict[str, Any]]] = [] + for word in sorted(words, key=lambda item: (float(item.get("top", 0.0)), float(item.get("x0", 0.0)))): + row = next((items for items in rows if abs(float(items[0].get("top", 0.0)) - float(word.get("top", 0.0))) <= 3.0), None) + if row is None: + row = [] + rows.append(row) + row.append(word) + blocks = [] + for row in rows: + row.sort(key=lambda item: float(item.get("x0", 0.0))) + text = _clean(" ".join(str(item.get("text") or "") for item in row)) + if text: + blocks.append({ + "bbox": [min(float(item["x0"]) for item in row), min(float(item["top"]) for item in row), max(float(item["x1"]) for item in row), max(float(item["bottom"]) for item in row)], + "text": text, + "source": "pdfplumber", + "confidence": 0.92, + }) + return blocks + except Exception: + return [] + + def _pdf_metadata(path: Path) -> dict[str, Any]: try: from pypdf import PdfReader @@ -214,6 +244,11 @@ def _read_pdf_pages( for index, page in enumerate(document, 1): raw_blocks = _pymupdf_line_blocks(page) ordered, order_confidence = _reading_order(raw_blocks, float(page.rect.width)) + if len(_normalized_compare_text(" ".join(item["text"] for item in ordered))) < 80: + fallback_blocks = _pdfplumber_blocks(path, index) + fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, float(page.rect.width)) + if len(_normalized_compare_text(" ".join(item["text"] for item in fallback_ordered))) > len(_normalized_compare_text(" ".join(item["text"] for item in ordered))): + ordered, order_confidence = fallback_ordered, min(fallback_confidence, 0.9) for block_index, block in enumerate(ordered, 1): block["id"] = f"P{index:03d}_B{block_index:03d}" block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) @@ -326,7 +361,8 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: continue explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) - known = any(re.match(rf"^{re.escape(alias)}\b", line, re.IGNORECASE) for aliases in SECTION_ALIASES.values() for alias in aliases) + compact_line = re.sub(r"[^a-z]", "", line.casefold()) + known = any(compact_line.startswith(re.sub(r"[^a-z]", "", alias.casefold())) for aliases in SECTION_ALIASES.values() for alias in aliases) if numbered: title_candidate = numbered.group(2).strip() first_alpha = next((char for char in title_candidate if char.isalpha()), "") @@ -360,7 +396,11 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, bool]: candidates = ("\n".join(item["title"] for item in sections) + "\n" + "\n".join(page.get("text", "") for page in pages)).casefold() - return {name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) for alias in aliases) for name, aliases in SECTION_ALIASES.items()} + compact = re.sub(r"[^a-z]", "", candidates) + return { + name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) or re.sub(r"[^a-z]", "", alias.casefold()) in compact for alias in aliases) + for name, aliases in SECTION_ALIASES.items() + } def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]: diff --git a/rfs/paper_to_image/inspection.py b/rfs/paper_to_image/inspection.py new file mode 100644 index 0000000..1df10bc --- /dev/null +++ b/rfs/paper_to_image/inspection.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import hashlib +import shutil +import time +from pathlib import Path +from typing import Any, Callable + +from ..utils import ensure_dir, read_json, write_json, write_text +from .analyzer import paper_markdown, parse_paper + + +def _source_hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def inspect_paper( + paper: str | Path, + out: str | Path, + deadline_seconds: int = 180, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + ocr_adapter: Callable | None = None, + archive_input: bool = False, +) -> dict[str, Any]: + started = time.monotonic() + source = Path(paper).resolve() + if not source.exists(): + raise FileNotFoundError(f"Paper does not exist: {source}") + root = ensure_dir(out).resolve() + active_source = source + if archive_input: + inputs = ensure_dir(root / "inputs") + active_source = inputs / f"paper{source.suffix.lower()}" + if active_source.resolve() != source: + shutil.copyfile(source, active_source) + digest = _source_hash(active_source) + model_path = root / "document_model.json" + if model_path.exists(): + cached = read_json(model_path) + if isinstance(cached, dict) and cached.get("source_sha256") == digest: + report = dict(cached.get("extraction_report") or {}) + elapsed = round(time.monotonic() - started, 3) + return { + "summary": "PDF inspection reused a matching structured document cache.", + "ok": report.get("status") != "fail", + "status": "cached", + "out_dir": str(root), + "source_sha256": digest, + "extraction_report": str(root / "extraction_report.json"), + "document_model": str(model_path), + "elapsed_seconds": elapsed, + } + deadline = max(30, int(deadline_seconds)) + parsed = parse_paper( + active_source, + deadline_at=time.monotonic() + deadline, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + ocr_adapter=ocr_adapter, + ) + write_json(model_path, parsed) + write_json(root / "extraction_report.json", parsed["extraction_report"]) + write_json(root / "section_index.json", parsed["document_index"]) + write_text(root / "paper.md", paper_markdown(parsed)) + elapsed = round(time.monotonic() - started, 3) + return { + "summary": "Paper inspection completed without model calls.", + "ok": parsed["extraction_report"].get("status") != "fail", + "status": parsed["extraction_report"].get("status"), + "out_dir": str(root), + "source_sha256": digest, + "page_count": parsed["page_count"], + "evidence_count": len(parsed["evidence"]), + "extraction_report": str(root / "extraction_report.json"), + "document_model": str(model_path), + "elapsed_seconds": elapsed, + } diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 95e51d9..e6e884b 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -120,6 +120,13 @@ def test_cli_production_defaults(self): self.assertEqual(args.template, "auto") self.assertEqual(args.repair_rounds, 1) + def test_inspect_pdf_cli_defaults(self): + parser = build_parser() + args = parser.parse_args(["inspect-pdf", "--paper", "paper.pdf", "--out", "output/inspect"]) + self.assertEqual(args.command, "inspect-pdf") + self.assertEqual(args.deadline, 180) + self.assertEqual(args.ocr_engine, "auto") + def test_domain_profiles_detect_method_and_survey(self): method = {"evidence": [{"text": "A neural model is optimized with a training loss and used during inference."}]} survey = {"evidence": [{"text": "This systematic survey presents a taxonomy and review of the landscape."}]} diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index b605843..d7fe9c9 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -5,6 +5,7 @@ import fitz from rfs.paper_to_image.analyzer import parse_paper +from rfs.paper_to_image.inspection import inspect_paper class PaperAnalyzerTests(unittest.TestCase): @@ -59,6 +60,32 @@ def draw(page): self.assertEqual(parsed["extraction_report"]["replacement_character_rate"], 0.0) self.assertEqual(parsed["extraction_report"]["mojibake_rate"], 0.0) + def test_low_text_page_uses_local_ocr_adapter(self): + path = self._pdf(lambda _page: None) + + parsed = parse_paper( + path, + ocr_engine="easyocr", + ocr_adapter=lambda _image, _lang: [{ + "text": "Abstract Method Image Encoder produces the final representation with reliable evidence.", + "confidence": 0.93, + "quad": [[10, 10], [900, 10], [900, 80], [10, 80]], + }], + ) + + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1]) + self.assertTrue(parsed["pages"][0]["used_ocr"]) + self.assertIn("Image Encoder", parsed["pages"][0]["text"]) + + def test_inspection_reuses_matching_document_cache(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "Abstract Method A sufficiently long scientific paragraph for deterministic PDF inspection.", fontsize=10)) + with tempfile.TemporaryDirectory() as temp: + first = inspect_paper(path, temp, ocr_engine="off") + second = inspect_paper(path, temp, ocr_engine="off") + + self.assertTrue(first["document_model"]) + self.assertEqual(second["status"], "cached") + if __name__ == "__main__": unittest.main() From 497f2d782b57522774cc52e347dac86d67e9dfda Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Tue, 21 Jul 2026 17:09:59 +0800 Subject: [PATCH 18/74] Add fast framework prompt workflow --- rfs/cli.py | 30 ++- rfs/paper_to_image/__init__.py | 3 +- rfs/paper_to_image/analyzer.py | 10 +- rfs/paper_to_image/planner.py | 36 ++- rfs/paper_to_image/preparation.py | 319 +++++++++++++++++++++++ rfs/paper_to_image/review.py | 4 +- rfs/paper_to_image/workflow.py | 118 +++------ tests/integration/test_paper_to_image.py | 34 ++- 8 files changed, 459 insertions(+), 95 deletions(-) create mode 100644 rfs/paper_to_image/preparation.py diff --git a/rfs/cli.py b/rfs/cli.py index ce635e0..6cb5b60 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -12,7 +12,7 @@ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case -from .paper_to_image import inspect_paper, run_paper_to_image +from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro from .professional_repair import vlm_professional_repair_adapter @@ -111,6 +111,20 @@ def build_parser() -> argparse.ArgumentParser: inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") + fast_prompt = sub.add_parser("fast-framework-prompt", help="Compile a paper-grounded framework prompt without generating images.") + fast_prompt.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + fast_prompt.add_argument("--out", required=True, help="Output directory for evidence, semantic contract, prompt, and overlay specification.") + fast_prompt.add_argument("--deadline", type=int, default=180, help="Soft end-to-end deadline in seconds. Default: 180.") + fast_prompt.add_argument("--preferences", help="Optional JSON style and output preferences.") + fast_prompt.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + fast_prompt.add_argument("--planner-model") + fast_prompt.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + fast_prompt.add_argument("--aspect-ratio", default="16:9") + fast_prompt.add_argument("--language", default="English") + fast_prompt.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "off"], default="auto") + fast_prompt.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + fast_prompt.add_argument("--json", action="store_true", help="Emit JSON.") + make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") make.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") make.add_argument("--reference", required=True, help="User-provided visual reference image path.") @@ -338,6 +352,20 @@ def main(argv: list[str] | None = None) -> int: ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, ) + elif args.command == "fast-framework-prompt": + result = run_fast_framework_prompt( + paper=args.paper, + out=args.out, + deadline_seconds=args.deadline, + planner_mode=args.planner_mode, + planner_model=args.planner_model, + ocr_engine=args.ocr_engine, + ocr_lang=args.ocr_lang, + preferences_path=args.preferences, + aspect_ratio=args.aspect_ratio, + language=args.language, + domain_profile=args.domain_profile, + ) elif args.command == "make-framework": result = make_framework( paper=args.paper, diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index 3f10052..b3d42a8 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,6 +1,7 @@ """Paper analysis and paper-to-image workflows.""" from .inspection import inspect_paper +from .preparation import prepare_paper_figure_contract, run_fast_framework_prompt from .workflow import run_paper_to_image -__all__ = ["inspect_paper", "run_paper_to_image"] +__all__ = ["inspect_paper", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 744f033..f79f904 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -25,6 +25,12 @@ def _clean(text: str) -> str: value = unicodedata.normalize("NFKC", str(text or "")).replace("\u00ad", "") value = value.replace("\r\n", "\n").replace("\r", "\n") + previous = None + while previous != value: + previous = value + value = re.sub(r"\b([A-Z])\s+([A-Z]{2,})\b", r"\1\2", value) + value = re.sub(r"\bANIMAGE\b", "AN IMAGE", value) + value = re.sub(r"\(\s*([A-Z])\s+([A-Z])\s+([A-Z])\s*\)", r"(\1\2\3)", value) value = re.sub(r"[ \t]+", " ", value) value = re.sub(r"\n{3,}", "\n\n", value) return value.strip() @@ -362,11 +368,11 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) compact_line = re.sub(r"[^a-z]", "", line.casefold()) - known = any(compact_line.startswith(re.sub(r"[^a-z]", "", alias.casefold())) for aliases in SECTION_ALIASES.values() for alias in aliases) + known = len(line) <= 60 and not line.endswith((".", ",", ";")) and any(compact_line.startswith(re.sub(r"[^a-z]", "", alias.casefold())) for aliases in SECTION_ALIASES.values() for alias in aliases) if numbered: title_candidate = numbered.group(2).strip() first_alpha = next((char for char in title_candidate if char.isalpha()), "") - if not first_alpha or not first_alpha.isupper() or re.search(r"[=±∑∫]", title_candidate): + if not first_alpha or not first_alpha.isupper() or re.search(r"[=±∑∫]", title_candidate) or title_candidate.endswith(".") or len(title_candidate) > 80 or len(title_candidate.split()) > 10: numbered = None if not (explicit or numbered or known): continue diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 000c887..b86a3d6 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -66,6 +66,8 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = "figure_specification": {{ "summary": "Scientific contract for the figure.", "figure_goal": "...", + "research_problem": {{"text": "...", "evidence_ids": ["E0001"]}}, + "central_claim": {{"text": "...", "evidence_ids": ["E0001"]}}, "storyline": [], "must_show": [{{"text": "...", "evidence_ids": ["E0001"]}}], "modules": [{{"id": "...", "name": "...", "role": "...", "evidence_ids": ["E0001"]}}], @@ -73,9 +75,15 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = "inputs": [], "outputs": [], "innovations": [], + "feedback_loops": [], + "training_flow": [], + "inference_flow": [], + "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", + "required_labels": [], "visual_priorities": [], "terminology": {{}}, - "forbidden_inventions": [] + "forbidden_inventions": [], + "uncertainties": [] }}, "design_plan": {{ "summary": "Information narrative for one complete image.", @@ -252,13 +260,13 @@ def normalize_plan(raw: dict, preferences: dict) -> dict: return result -def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None) -> tuple[dict, dict]: +def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1) -> tuple[dict, dict]: prompt = _planner_prompt(parsed, preferences, paper_review=paper_review) metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=240, retries=1) + raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries))) metadata["model"] = resolved return normalize_plan(raw, preferences), metadata except Exception as exc: @@ -367,6 +375,12 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: endpoint_id = str(item.get("id") or item.get("name") or item.get("visible_label") or item.get("text") or "").strip() if endpoint_id: endpoint_ids.add(endpoint_id) + evidence_ids = item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [] + if not evidence_ids: + warnings.append(f"{field}[{index}] has no evidence_ids") + invalid = [value for value in evidence_ids if value not in valid_evidence] + if invalid: + errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] for index, relation in enumerate(relations): @@ -379,7 +393,7 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: errors.append(f"relation {index + 1} has unknown endpoint: {source} -> {target}") evidence_ids = relation.get("evidence_ids") if isinstance(relation.get("evidence_ids"), list) else [] if not evidence_ids: - warnings.append(f"relation {source} -> {target} has no evidence_ids") + errors.append(f"relation {source} -> {target} has no evidence_ids") invalid = [item for item in evidence_ids if item not in valid_evidence] if invalid: errors.append(f"relation {source} -> {target} references unknown evidence ids: {invalid}") @@ -394,6 +408,20 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: if invalid: errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") + for field in ("research_problem", "central_claim"): + item = spec.get(field) + if not isinstance(item, dict): + errors.append(f"figure_specification.{field} is missing") + continue + text = str(item.get("text") or item.get("statement") or "").strip() + evidence_ids = item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [] + explicitly_unknown = item.get("status") == "unknown" or text.casefold() == "unknown" or "require" in text.casefold() and "review" in text.casefold() + if not explicitly_unknown and not evidence_ids: + errors.append(f"figure_specification.{field} has no evidence_ids") + invalid = [value for value in evidence_ids if value not in valid_evidence] + if invalid: + errors.append(f"figure_specification.{field} references unknown evidence ids: {invalid}") + return { "summary": "Scientific grounding and relation-contract validation.", "ok": not errors, diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py new file mode 100644 index 0000000..3c76373 --- /dev/null +++ b/rfs/paper_to_image/preparation.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import json +import shutil +import time +from pathlib import Path +from typing import Any, Callable + +from ..utils import ensure_dir, write_json, write_text +from .analyzer import paper_markdown, parse_paper +from .planner import compile_image_prompt, merge_preferences, plan_paper_image, validate_plan_grounding +from .review import build_paper_review, detect_domain_profile, validate_review_coverage + + +def _load_preferences(path: str | Path | None) -> dict[str, Any]: + if not path: + return {} + source = Path(path).resolve() + if not source.exists(): + raise FileNotFoundError(f"Preferences file does not exist: {source}") + raw = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("Preferences file must contain a JSON object") + return raw + + +def _archive_file(source: str | Path, target_dir: Path, target_name: str | None = None) -> str: + path = Path(source).resolve() + if not path.exists(): + raise FileNotFoundError(f"Input file does not exist: {path}") + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / (target_name or path.name) + if target.resolve() != path: + shutil.copyfile(path, target) + return str(target) + + +def _section_summary_markdown(parsed: dict[str, Any]) -> str: + grouped: dict[str, list[str]] = {} + for item in parsed.get("evidence", []): + section = str(item.get("section_hint") or "Unclassified") + grouped.setdefault(section, []).append(str(item.get("text") or "")) + lines = ["# Section Summary", ""] + for section, values in grouped.items(): + combined = " ".join(value.replace("\n", " ") for value in values) + sentences = [value.strip() for value in combined.split(". ") if value.strip()] + summary = ". ".join(sentences[:3])[:900].strip() + lines.extend([f"## {section}", "", summary or "No reliable summary extracted.", ""]) + return "\n".join(lines).strip() + "\n" + + +def _collect_evidence_ids(value: Any) -> set[str]: + found: set[str] = set() + if isinstance(value, dict): + for key, child in value.items(): + if key == "evidence_ids" and isinstance(child, list): + found.update(str(item) for item in child if str(item).strip()) + else: + found.update(_collect_evidence_ids(child)) + elif isinstance(value, list): + for child in value: + found.update(_collect_evidence_ids(child)) + return found + + +def _item_id(item: dict[str, Any], field: str, index: int) -> str: + return str(item.get("id") or item.get("name") or item.get("visible_label") or item.get("text") or item.get("statement") or f"{field}_{index}").strip() + + +def _item_label(item: dict[str, Any]) -> str: + return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() + + +def _infer_topology(spec: dict[str, Any]) -> str: + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + if any("feedback" in str(item.get("type") or "").casefold() for item in relations): + return "feedback" + inputs = [item for item in spec.get("inputs", []) if isinstance(item, dict)] + labels = " ".join(_item_label(item).casefold() for item in inputs) + modalities = sum(term in labels for term in ("image", "text", "audio", "video", "depth", "thermal", "imu")) + if len(inputs) >= 3 or modalities >= 3: + return "multimodal" + outgoing: dict[str, int] = {} + for relation in relations: + source = str(relation.get("source") or relation.get("source_id") or "") + outgoing[source] = outgoing.get(source, 0) + 1 + if any(value > 1 for value in outgoing.values()): + return "branch" + total = sum(len(spec.get(field, []) if isinstance(spec.get(field), list) else []) for field in ("inputs", "modules", "outputs", "innovations")) + if total >= 12: + return "dense_multiframe" + return "linear" if relations else "unknown" + + +def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} + spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + spec.setdefault("research_problem", summary.get("research_problem") or {"text": "unknown", "evidence_ids": []}) + spec.setdefault("central_claim", summary.get("central_claim") or {"text": "unknown", "evidence_ids": []}) + if not isinstance(spec.get("inputs"), list) or not spec.get("inputs"): + spec["inputs"] = summary.get("inputs") if isinstance(summary.get("inputs"), list) else [] + if not isinstance(spec.get("outputs"), list) or not spec.get("outputs"): + spec["outputs"] = summary.get("outputs") if isinstance(summary.get("outputs"), list) else [] + spec.setdefault("training_flow", summary.get("training_flow") if isinstance(summary.get("training_flow"), list) else []) + spec.setdefault("inference_flow", summary.get("inference_flow") if isinstance(summary.get("inference_flow"), list) else []) + spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) + spec.setdefault("topology", _infer_topology(spec)) + labels = [] + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + labels.extend(str(value).strip() for value in terminology.values() if str(value).strip()) + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if isinstance(item, dict) and _item_label(item): + labels.append(_item_label(item)) + spec["required_labels"] = list(dict.fromkeys(labels)) + raw_uncertainties = list(summary.get("unknowns", []) if isinstance(summary.get("unknowns"), list) else []) + uncertainties = [ + str(item.get("statement") or item.get("text") or item.get("id") or "unknown") if isinstance(item, dict) else str(item) + for item in raw_uncertainties + ] + evidence_by_id = {item["id"]: item for item in parsed.get("evidence", [])} + for field in ("inputs", "modules", "outputs", "relations", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + evidence = [evidence_by_id.get(value) for value in item.get("evidence_ids", [])] + if evidence and all(float(record.get("confidence", 1.0)) < 0.75 for record in evidence if record): + uncertainties.append(f"{_item_id(item, field, 0)} relies only on low-confidence OCR evidence") + spec["uncertainties"] = list(dict.fromkeys(str(item) for item in uncertainties if str(item).strip())) + spec.setdefault("forbidden_inventions", []) + plan["figure_specification"] = spec + return spec + + +def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: + spec = plan.get("figure_specification", {}) + labels = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for index, item in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): + if not isinstance(item, dict): + continue + text = _item_label(item) + if text: + labels.append({"target_id": _item_id(item, field, index), "text": text, "role": field.rstrip("s")}) + connectors = [] + for item in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if isinstance(item, dict): + connectors.append({ + "source": str(item.get("source") or item.get("source_id") or ""), + "target": str(item.get("target") or item.get("target_id") or ""), + "type": str(item.get("type") or item.get("relation_type") or "data_flow"), + "label": str(item.get("label") or ""), + }) + return { + "summary": "Exact editable labels and directed connectors derived from the paper contract.", + "labels": labels, + "connectors": connectors, + "groups": list(plan.get("design_plan", {}).get("groups", []) if isinstance(plan.get("design_plan"), dict) else []), + "reading_order": list(plan.get("design_plan", {}).get("reading_order", []) if isinstance(plan.get("design_plan"), dict) else []), + } + + +def prepare_paper_figure_contract( + paper: str | Path, + out: str | Path, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "auto", + ocr_lang: str = "en_ch", + preferences_path: str | Path | None = None, + positive_references: list[str] | None = None, + negative_references: list[str] | None = None, + aspect_ratio: str | None = None, + language: str | None = None, + domain_profile: str = "auto", + ocr_adapter: Callable | None = None, +) -> dict[str, Any]: + started = time.monotonic() + root = ensure_dir(out).resolve() + inputs = ensure_dir(root / "inputs") + positive_dir = ensure_dir(inputs / "positive_references") + negative_dir = ensure_dir(inputs / "negative_references") + archived_paper = _archive_file(paper, inputs, f"paper{Path(paper).suffix.lower()}") + archived_positive = [_archive_file(path, positive_dir) for path in (positive_references or [])] + archived_negative = [_archive_file(path, negative_dir) for path in (negative_references or [])] + raw_preferences = _load_preferences(preferences_path) + preferences = merge_preferences(raw_preferences, aspect_ratio=aspect_ratio, language=language) + preferences["positive_references"] = archived_positive + preferences["negative_references"] = archived_negative + write_json(root / "preferences.json", preferences) + write_json(root / "input_manifest.json", { + "summary": "Archived inputs for paper figure contract preparation.", + "paper_original": str(Path(paper).resolve()), + "paper_archived": archived_paper, + "preferences_original": str(Path(preferences_path).resolve()) if preferences_path else None, + "positive_references": archived_positive, + "negative_references": archived_negative, + }) + + deadline = max(30, int(deadline_seconds)) + deadline_at = started + deadline + parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter) + write_json(root / "document_model.json", parsed) + write_json(root / "extraction_report.json", parsed["extraction_report"]) + write_json(root / "document_index.json", parsed["document_index"]) + write_json(root / "section_index.json", parsed["document_index"]) + write_json(root / "evidence_map.json", {"summary": "Page-aware evidence map.", "source_path": parsed["source_path"], "page_count": parsed["page_count"], "char_count": parsed["char_count"], "headings": parsed["headings"], "evidence": parsed["evidence"]}) + write_text(root / "paper.md", paper_markdown(parsed)) + write_text(root / "section_summary.md", _section_summary_markdown(parsed)) + + if parsed["extraction_report"].get("status") == "fail": + result = { + "summary": "Paper extraction failed the minimum readable-page gate.", + "ok": False, + "status": "extraction_failed", + "out_dir": str(root), + "paper": archived_paper, + "elapsed_seconds": round(time.monotonic() - started, 3), + "errors": parsed["extraction_report"].get("warnings", []), + } + write_json(root / "run_report.json", result) + return {**result, "root": root, "parsed": parsed, "preferences": preferences} + + selected_domain = detect_domain_profile(parsed, explicit=domain_profile) + write_json(root / "domain_profile.json", selected_domain) + remaining = max(10, int(deadline_at - time.monotonic() - 15)) + effective_mode = planner_mode if remaining >= 30 else "heuristic" + paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=effective_mode, model=planner_model, timeout_seconds=min(75, remaining), retries=0) + review_prompt = review_metadata.pop("prompt") + write_text(root / "prompts" / "paper_review_prompt.txt", review_prompt) + write_json(root / "paper_review_metadata.json", review_metadata) + write_json(root / "paper_review.json", paper_review) + coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=False) + write_json(root / "review_coverage_report.json", coverage) + + remaining = max(10, int(deadline_at - time.monotonic() - 15)) + planning_mode = effective_mode if remaining >= 25 else "heuristic" + plan, planner_metadata = plan_paper_image( + parsed, + preferences, + mode=planning_mode, + model=planner_model, + reference_images=archived_positive + archived_negative, + paper_review=paper_review, + timeout_seconds=min(70, remaining), + retries=0, + ) + planning_prompt = planner_metadata.pop("prompt") + write_text(root / "prompts" / "planning_prompt.txt", planning_prompt) + write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) + normalize_figure_contract(plan, parsed) + for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): + write_json(root / f"{name}.json", plan[name]) + planning_validation = validate_plan_grounding(plan, parsed) + write_json(root / "planning_validation_report.json", planning_validation) + overlay = build_overlay_spec(plan) + write_json(root / "overlay_spec.json", overlay) + final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1) + write_text(root / "image_prompt.md", final_prompt) + write_text(root / "image_prompt.txt", final_prompt) + referenced_ids = _collect_evidence_ids({"paper_review": paper_review, "figure_specification": plan["figure_specification"]}) + write_json(root / "key_evidence.json", { + "summary": "Evidence records referenced by the paper review and figure contract.", + "evidence": [item for item in parsed["evidence"] if item["id"] in referenced_ids], + }) + + elapsed = round(time.monotonic() - started, 3) + deadline_reached = time.monotonic() >= deadline_at + production_ready = bool( + review_metadata.get("mode") == "vlm" + and planner_metadata.get("mode") == "vlm" + and planning_validation.get("ok") + and not deadline_reached + ) + status = "complete" if production_ready else "completed_with_warnings" + result = { + "summary": "Fast paper-to-framework contract preparation completed.", + "ok": bool(planning_validation.get("ok")), + "status": status, + "production_ready": production_ready, + "engineering_only": not production_ready, + "out_dir": str(root), + "paper": archived_paper, + "source_sha256": parsed["source_sha256"], + "deadline_seconds": deadline, + "deadline_reached": deadline_reached, + "elapsed_seconds": elapsed, + "planner_mode": planner_metadata.get("mode"), + "paper_review_mode": review_metadata.get("mode"), + "planner_warning": planner_metadata.get("warning"), + "review_warning": review_metadata.get("warning"), + "topology": plan["figure_specification"].get("topology"), + "module_count": len(plan["figure_specification"].get("modules", [])), + "relation_count": len(plan["figure_specification"].get("relations", [])), + "uncertainties": plan["figure_specification"].get("uncertainties", []), + "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], + } + write_json(root / "run_report.json", result) + return { + **result, + "root": root, + "parsed": parsed, + "preferences": preferences, + "paper_review": paper_review, + "review_metadata": review_metadata, + "selected_domain": selected_domain, + "plan": plan, + "planner_metadata": planner_metadata, + "archived_positive": archived_positive, + "archived_negative": archived_negative, + "planning_validation": planning_validation, + } + + +def run_fast_framework_prompt(**kwargs: Any) -> dict[str, Any]: + prepared = prepare_paper_figure_contract(**kwargs) + internal = {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"} + return {key: value for key, value in prepared.items() if key not in internal} diff --git a/rfs/paper_to_image/review.py b/rfs/paper_to_image/review.py index 57d26f3..3200ffc 100644 --- a/rfs/paper_to_image/review.py +++ b/rfs/paper_to_image/review.py @@ -326,13 +326,13 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s } -def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None) -> tuple[dict, dict]: +def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1) -> tuple[dict, dict]: prompt = _review_prompt(parsed, domain_profile) metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_REVIEW_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, [], model=resolved, timeout=300, retries=1) + raw = call_vlm_json(prompt, [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries))) metadata["model"] = resolved review = normalize_review(raw, domain_profile) _expand_evidence(review, parsed) diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 9690f4c..957b494 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -1,41 +1,17 @@ from __future__ import annotations -import json -import shutil import time from pathlib import Path from typing import Any -from ..utils import ensure_dir, write_json, write_text -from .analyzer import parse_paper +from ..utils import write_json, write_text from .generator import generate_and_select -from .planner import compile_image_prompt, merge_preferences, plan_paper_image, validate_plan_grounding -from .review import build_paper_review, detect_domain_profile, validate_review_coverage +from .planner import compile_image_prompt +from .preparation import prepare_paper_figure_contract +from .review import validate_review_coverage from .templates import build_template_profiles, render_layout_blueprint, select_template -def _load_preferences(path: str | Path | None) -> dict: - if not path: - return {} - source = Path(path).resolve() - if not source.exists(): - raise FileNotFoundError(f"Preferences file does not exist: {source}") - raw = json.loads(source.read_text(encoding="utf-8")) - if not isinstance(raw, dict): - raise ValueError("Preferences file must contain a JSON object") - return raw - - -def _archive_file(source: str | Path, target_dir: Path, target_name: str | None = None) -> str: - path = Path(source).resolve() - if not path.exists(): - raise FileNotFoundError(f"Input file does not exist: {path}") - target_dir.mkdir(parents=True, exist_ok=True) - target = target_dir / (target_name or path.name) - shutil.copyfile(path, target) - return str(target) - - def run_paper_to_image( paper: str | Path, out: str | Path, @@ -61,47 +37,36 @@ def run_paper_to_image( critic_adapter=None, ) -> dict: started = time.time() - root = ensure_dir(out).resolve() - inputs = ensure_dir(root / "inputs") - positive_dir = ensure_dir(inputs / "positive_references") - negative_dir = ensure_dir(inputs / "negative_references") - - archived_paper = _archive_file(paper, inputs, f"paper{Path(paper).suffix.lower()}") - archived_positive = [_archive_file(path, positive_dir) for path in (positive_references or [])] - archived_negative = [_archive_file(path, negative_dir) for path in (negative_references or [])] - raw_preferences = _load_preferences(preferences_path) - preferences = merge_preferences(raw_preferences, aspect_ratio=aspect_ratio, language=language) - preferences["positive_references"] = archived_positive - preferences["negative_references"] = archived_negative - write_json(root / "preferences.json", preferences) - input_manifest = { - "summary": "Archived inputs for the paper-to-image run.", - "paper_original": str(Path(paper).resolve()), - "paper_archived": archived_paper, - "preferences_original": str(Path(preferences_path).resolve()) if preferences_path else None, - "positive_references": archived_positive, - "negative_references": archived_negative, - } - write_json(root / "input_manifest.json", input_manifest) - - parsed = parse_paper(archived_paper) - evidence_map = { - "summary": "Page-aware evidence map used by paper summary and figure planning.", - "source_path": parsed["source_path"], - "page_count": parsed["page_count"], - "char_count": parsed["char_count"], - "headings": parsed["headings"], - "evidence": parsed["evidence"], - } - write_json(root / "evidence_map.json", evidence_map) - write_json(root / "document_index.json", parsed["document_index"]) - - selected_domain = detect_domain_profile(parsed, explicit=domain_profile) - write_json(root / "domain_profile.json", selected_domain) - paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=planner_mode, model=planner_model) - write_text(root / "prompts" / "paper_review_prompt.txt", review_metadata.pop("prompt")) - write_json(root / "paper_review_metadata.json", review_metadata) - write_json(root / "paper_review.json", paper_review) + prepared = prepare_paper_figure_contract( + paper=paper, + out=out, + deadline_seconds=600, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine if ocr_engine in {"auto", "paddle", "easyocr", "off"} else "auto", + ocr_lang=ocr_lang, + preferences_path=preferences_path, + positive_references=positive_references, + negative_references=negative_references, + aspect_ratio=aspect_ratio, + language=language, + domain_profile=domain_profile, + ocr_adapter=ocr_adapter, + ) + if not prepared.get("ok"): + raise ValueError(f"Paper contract preparation failed: {prepared.get('errors') or prepared.get('planning_validation', {}).get('errors')}") + root = prepared["root"] + archived_paper = prepared["paper"] + archived_positive = prepared["archived_positive"] + archived_negative = prepared["archived_negative"] + preferences = prepared["preferences"] + parsed = prepared["parsed"] + selected_domain = prepared["selected_domain"] + paper_review = prepared["paper_review"] + review_metadata = prepared["review_metadata"] + plan = prepared["plan"] + planner_metadata = prepared["planner_metadata"] + planning_validation = prepared["planning_validation"] production_mode = asset_mode == "image2" coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=production_mode) write_json(root / "review_coverage_report.json", coverage) @@ -122,25 +87,10 @@ def run_paper_to_image( blueprint_report = render_layout_blueprint(selected_template, root / "layout_blueprint.png", target_ratio=preferences["aspect_ratio"]) write_json(root / "layout_blueprint.json", blueprint_report) - references = archived_positive + archived_negative - plan, planner_metadata = plan_paper_image( - parsed, - preferences, - mode=planner_mode, - model=planner_model, - reference_images=references, - paper_review=paper_review, - ) - write_text(root / "prompts" / "planning_prompt.txt", planner_metadata.pop("prompt")) - write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) - artifact_names = ["paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"] - for name in artifact_names: - write_json(root / f"{name}.json", plan[name]) plan["style_plan"]["selected_template_id"] = selected_template.get("profile_id") plan["style_plan"]["template_style"] = selected_template.get("style", {}) plan["style_plan"]["template_palette"] = selected_template.get("palette", []) write_json(root / "style_plan.json", plan["style_plan"]) - planning_validation = validate_plan_grounding(plan, parsed) write_json(root / "planning_validation_report.json", planning_validation) if not planning_validation["ok"]: raise ValueError(f"Paper-to-image planning failed scientific grounding validation: {planning_validation['errors']}") diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index e6e884b..85ded54 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -11,7 +11,7 @@ from PIL import Image from rfs.cli import build_parser -from rfs.paper_to_image import run_paper_to_image +from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import review_candidate from rfs.paper_to_image.generator import generate_and_select from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage @@ -127,6 +127,38 @@ def test_inspect_pdf_cli_defaults(self): self.assertEqual(args.deadline, 180) self.assertEqual(args.ocr_engine, "auto") + def test_fast_framework_prompt_writes_contract_without_generation(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paper = root / "paper.md" + paper.write_text(PAPER_TEXT, encoding="utf-8") + out = root / "fast" + + result = run_fast_framework_prompt( + paper=paper, + out=out, + deadline_seconds=180, + planner_mode="heuristic", + ocr_engine="off", + ) + + self.assertTrue(result["ok"]) + self.assertTrue(result["engineering_only"]) + self.assertFalse(result["production_ready"]) + self.assertEqual(result["deadline_seconds"], 180) + for name in ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"]: + self.assertTrue((out / name).exists(), name) + self.assertFalse((out / "selected_image.png").exists()) + overlay = json.loads((out / "overlay_spec.json").read_text(encoding="utf-8")) + self.assertTrue(overlay["labels"]) + + def test_fast_framework_cli_defaults(self): + parser = build_parser() + args = parser.parse_args(["fast-framework-prompt", "--paper", "paper.pdf", "--out", "output/fast"]) + self.assertEqual(args.command, "fast-framework-prompt") + self.assertEqual(args.deadline, 180) + self.assertEqual(args.planner_mode, "vlm") + def test_domain_profiles_detect_method_and_survey(self): method = {"evidence": [{"text": "A neural model is optimized with a training loss and used during inference."}]} survey = {"evidence": [{"text": "This systematic survey presents a taxonomy and review of the landscape."}]} From 981c91ff41e1e7eb9344ccf6d2fae7df3db107b6 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Tue, 21 Jul 2026 17:11:54 +0800 Subject: [PATCH 19/74] Unify paper planning evidence contracts --- rfs/paper_to_image/preparation.py | 3 +- tests/unit/test_paper_contract.py | 85 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_paper_contract.py diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 3c76373..43044c4 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -124,7 +124,8 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d if not isinstance(item, dict): continue evidence = [evidence_by_id.get(value) for value in item.get("evidence_ids", [])] - if evidence and all(float(record.get("confidence", 1.0)) < 0.75 for record in evidence if record): + valid_records = [record for record in evidence if record] + if valid_records and all(float(record.get("confidence", 1.0)) < 0.75 and str(record.get("source") or "").casefold() in {"easyocr", "paddle", "adapter"} for record in valid_records): uncertainties.append(f"{_item_id(item, field, 0)} relies only on low-confidence OCR evidence") spec["uncertainties"] = list(dict.fromkeys(str(item) for item in uncertainties if str(item).strip())) spec.setdefault("forbidden_inventions", []) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py new file mode 100644 index 0000000..41da2cb --- /dev/null +++ b/tests/unit/test_paper_contract.py @@ -0,0 +1,85 @@ +import unittest + +from rfs.paper_to_image.planner import validate_plan_grounding +from rfs.paper_to_image.preparation import build_overlay_spec, normalize_figure_contract + + +class PaperContractTests(unittest.TestCase): + def _parsed(self): + return { + "evidence": [ + {"id": "E0001", "confidence": 1.0}, + {"id": "E0002", "confidence": 0.7, "source": "easyocr"}, + ] + } + + def test_contract_adds_topology_labels_and_separate_flows(self): + plan = { + "paper_summary": { + "research_problem": {"text": "Understand an input.", "evidence_ids": ["E0001"]}, + "central_claim": {"text": "The method refines its output.", "evidence_ids": ["E0001"]}, + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "Refined Output", "evidence_ids": ["E0001"]}], + "training_flow": [{"step": "Train", "evidence_ids": ["E0001"]}], + "inference_flow": [{"step": "Refine", "evidence_ids": ["E0001"]}], + "unknowns": [], + }, + "figure_specification": { + "modules": [{"id": "refiner", "name": "Refiner", "evidence_ids": ["E0001"]}], + "relations": [{"source": "output", "target": "refiner", "type": "feedback", "evidence_ids": ["E0001"]}], + "terminology": {"Refiner": "Refiner"}, + }, + "design_plan": {"reading_order": ["input", "refiner", "output"], "groups": []}, + } + + spec = normalize_figure_contract(plan, self._parsed()) + overlay = build_overlay_spec(plan) + + self.assertEqual(spec["topology"], "feedback") + self.assertEqual(spec["training_flow"][0]["step"], "Train") + self.assertEqual(spec["inference_flow"][0]["step"], "Refine") + self.assertIn("Refiner", spec["required_labels"]) + self.assertEqual(overlay["connectors"][0]["source"], "output") + self.assertTrue(any(item["text"] == "Refined Output" for item in overlay["labels"])) + + def test_grounding_rejects_relation_without_evidence(self): + plan = { + "figure_specification": { + "research_problem": {"text": "Problem", "evidence_ids": ["E0001"]}, + "central_claim": {"text": "Claim", "evidence_ids": ["E0001"]}, + "modules": [ + {"id": "a", "name": "A", "evidence_ids": ["E0001"]}, + {"id": "b", "name": "B", "evidence_ids": ["E0001"]}, + ], + "relations": [{"source": "a", "target": "b", "type": "data_flow", "evidence_ids": []}], + "inputs": [], + "outputs": [], + "innovations": [], + } + } + + result = validate_plan_grounding(plan, self._parsed()) + + self.assertFalse(result["ok"]) + self.assertIn("relation a -> b has no evidence_ids", result["errors"]) + + def test_low_confidence_evidence_is_reported_as_uncertain(self): + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [{"id": "ocr_module", "name": "OCR Module", "evidence_ids": ["E0002"]}], + "relations": [], + "inputs": [], + "outputs": [], + "innovations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, self._parsed()) + + self.assertIn("ocr_module relies only on low-confidence OCR evidence", spec["uncertainties"]) + + +if __name__ == "__main__": + unittest.main() From 9eb88388c0e8ab52f88d5079288480fc60ecf17b Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Wed, 22 Jul 2026 16:25:28 +0800 Subject: [PATCH 20/74] Benchmark fast paper understanding on top-conference papers --- README.md | 12 + benchmarks/README.md | 2 + .../cases/101_vit_linear/case.json | 2 +- .../101_vit_linear/expected_semantics.json | 10 +- .../cases/102_mask_rcnn_branch/case.json | 2 +- .../expected_semantics.json | 10 +- .../cases/103_self_refine_feedback/case.json | 2 +- .../expected_semantics.json | 10 +- .../cases/104_imagebind_multimodal/case.json | 2 +- .../expected_semantics.json | 6 +- .../cases/105_sam_dense_multiframe/case.json | 2 +- .../expected_semantics.json | 6 +- docs/paper-to-image.md | 10 + rfs/cli.py | 14 +- rfs/evaluation/__init__.py | 3 +- rfs/evaluation/benchmarking.py | 135 +++- rfs/paper_to_image/analyzer.py | 53 +- rfs/paper_to_image/planner.py | 60 +- rfs/paper_to_image/preparation.py | 577 +++++++++++++++++- rfs/paper_to_image/review.py | 10 +- skills/research-figure-studio/SKILL.md | 3 + tests/integration/test_paper_to_image.py | 7 + tests/unit/test_benchmarking.py | 38 ++ tests/unit/test_paper_analyzer.py | 20 + tests/unit/test_paper_contract.py | 90 ++- 25 files changed, 978 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 4d79dc2..832ecb4 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,18 @@ For offline engineering validation only, add `--image-asset-mode placeholder --r ## Paper To Image Without PPTX +For the fast “paper to evidence-grounded framework prompt” path, without image generation: + +```powershell +rfs fast-framework-prompt ` + --paper "C:\path\paper.pdf" ` + --out "output\fast_result" ` + --deadline 180 ` + --json +``` + +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. It uses one compact VLM request by default, falls back to evidence-driven topology rules, and caches successful contracts by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. `rfs inspect-pdf` runs only the local extraction and quality checks. + Use `paper-to-image` when the required endpoint is a generated raster framework figure rather than an editable PowerPoint file. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts diff --git a/benchmarks/README.md b/benchmarks/README.md index 1d44455..c29b74e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -22,6 +22,7 @@ A full-slide copy of the reference image is a hard failure even if pixel similar rfs benchmark list --root benchmarks --json rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipeline --json rfs benchmark fetch --case benchmarks/paper-to-image/cases/101_vit_linear --json +rfs benchmark fast --case benchmarks/paper-to-image/cases/101_vit_linear --out output/benchmarks/vit_fast --planner-mode heuristic --json rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json ``` @@ -37,5 +38,6 @@ Real-paper cases commit `source.json` and human-authored Ground Truth, while `be ## Evaluation tiers - Offline contract tier: placeholder assets, deterministic validation, CI-safe. +- Fast planning tier: entity recall, relation recall, forbidden content, evidence grounding, deadline, and cache behavior without image generation. - Production quality tier: real VLM/image models, frozen judge, repeated seeds, and human calibration. - Human audit tier: blinded pairwise ratings for aesthetics, clarity, and information density. diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/case.json b/benchmarks/paper-to-image/cases/101_vit_linear/case.json index 3ac8ca6..e12472a 100644 --- a/benchmarks/paper-to-image/cases/101_vit_linear/case.json +++ b/benchmarks/paper-to-image/cases/101_vit_linear/case.json @@ -11,5 +11,5 @@ "positive_references": [], "negative_references": [], "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, - "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} } diff --git a/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json index dcad857..3b65938 100644 --- a/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/101_vit_linear/expected_semantics.json @@ -1,14 +1,14 @@ { "summary": "Human-authored semantic contract for the Vision Transformer architecture.", "entities": [ - {"id": "input_image", "label": "Input Image", "role": "input", "required": true}, - {"id": "image_patches", "label": "Image Patches", "role": "module", "required": true}, - {"id": "linear_projection", "label": "Linear Projection of Flattened Patches", "role": "module", "required": true}, + {"id": "input_image", "label": "Input Image", "aliases": ["2D Image", "Image"], "role": "input", "required": true}, + {"id": "image_patches", "label": "Image Patches", "aliases": ["Patches"], "role": "module", "required": true}, + {"id": "linear_projection", "label": "Linear Projection of Flattened Patches", "aliases": ["Linear Projection"], "role": "module", "required": true}, {"id": "position_embedding", "label": "Position Embedding", "role": "module", "required": true}, - {"id": "class_token", "label": "Class Token", "role": "module", "required": true}, + {"id": "class_token", "label": "Class Token", "aliases": ["Class Embedding", "[class] Embedding"], "role": "module", "required": true}, {"id": "transformer_encoder", "label": "Transformer Encoder", "role": "module", "required": true}, {"id": "mlp_head", "label": "MLP Head", "role": "module", "required": true}, - {"id": "class_prediction", "label": "Class Prediction", "role": "output", "required": true} + {"id": "class_prediction", "label": "Class Prediction", "aliases": ["Image Classification Predictions", "Classification Predictions", "Class Label"], "role": "output", "required": true} ], "relations": [ {"source": "input_image", "target": "image_patches", "type": "data_flow", "required": true}, diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json index 771007a..67ba6cb 100644 --- a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/case.json @@ -10,5 +10,5 @@ "preferences": "../../preferences/top_conference.json", "positive_references": [], "negative_references": [], "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, - "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} } diff --git a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json index c2187d0..390a603 100644 --- a/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/102_mask_rcnn_branch/expected_semantics.json @@ -1,12 +1,12 @@ { "summary": "Human-authored semantic contract for Mask R-CNN.", "entities": [ - {"id": "input_image", "label": "Input Image", "role": "input", "required": true}, - {"id": "backbone", "label": "Backbone", "role": "module", "required": true}, - {"id": "region_proposals", "label": "Region Proposals", "role": "module", "required": true}, + {"id": "input_image", "label": "Input Image", "aliases": ["Image"], "role": "input", "required": true}, + {"id": "backbone", "label": "Backbone", "aliases": ["Backbone Architecture"], "role": "module", "required": true}, + {"id": "region_proposals", "label": "Region Proposals", "aliases": ["Region Proposal Network (RPN)", "RPN", "Proposals"], "role": "module", "required": true}, {"id": "roi_align", "label": "RoIAlign", "role": "module", "required": true}, - {"id": "classification_head", "label": "Classification", "role": "output", "required": true}, - {"id": "box_head", "label": "Bounding-box Regression", "role": "output", "required": true}, + {"id": "classification_head", "label": "Classification", "aliases": ["Class Label", "Classification Head"], "role": "output", "required": true}, + {"id": "box_head", "label": "Bounding-box Regression", "aliases": ["Bounding Box", "Box Branch", "Box Regression"], "role": "output", "required": true}, {"id": "mask_head", "label": "Mask Branch", "role": "output", "required": true} ], "relations": [ diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json index 9c3d8e0..e75cef8 100644 --- a/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/case.json @@ -4,5 +4,5 @@ "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", "positive_references": [], "negative_references": [], "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, - "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} } diff --git a/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json index 8a10482..104b0bd 100644 --- a/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/103_self_refine_feedback/expected_semantics.json @@ -1,12 +1,12 @@ { "summary": "Human-authored semantic contract for Self-Refine.", "entities": [ - {"id": "task_input", "label": "Input", "role": "input", "required": true}, - {"id": "initial_generator", "label": "Generate", "role": "module", "required": true}, + {"id": "task_input", "label": "Input", "aliases": ["Task Prompt", "Task Instruction"], "role": "input", "required": true}, + {"id": "initial_generator", "label": "Generate", "aliases": ["INIT", "Generator"], "role": "module", "required": true}, {"id": "initial_output", "label": "Initial Output", "role": "module", "required": true}, - {"id": "feedback_module", "label": "Feedback", "role": "module", "required": true}, - {"id": "feedback_text", "label": "Feedback", "role": "control", "required": true}, - {"id": "refinement_module", "label": "Refine", "role": "module", "required": true}, + {"id": "feedback_module", "label": "Feedback", "aliases": ["FEEDBACK", "Feedback Provider"], "role": "module", "required": true}, + {"id": "feedback_text", "label": "Feedback", "aliases": ["Self-Feedback", "Self-Feedback (NL)"], "role": "control", "required": true}, + {"id": "refinement_module", "label": "Refine", "aliases": ["ITERATE / REFINE", "Refiner"], "role": "module", "required": true}, {"id": "refined_output", "label": "Refined Output", "role": "output", "required": true} ], "relations": [ diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json index 9531de9..aa2f8cd 100644 --- a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/case.json @@ -4,5 +4,5 @@ "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", "positive_references": [], "negative_references": [], "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, - "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} } diff --git a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json index 93cd3c5..49bbfa9 100644 --- a/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/104_imagebind_multimodal/expected_semantics.json @@ -7,9 +7,9 @@ {"id": "depth", "label": "Depth", "role": "input", "required": true}, {"id": "thermal", "label": "Thermal", "role": "input", "required": true}, {"id": "imu", "label": "IMU", "role": "input", "required": true}, - {"id": "modality_encoders", "label": "Modality Encoders", "role": "module", "required": true}, - {"id": "joint_embedding", "label": "Joint Embedding Space", "role": "module", "required": true}, - {"id": "emergent_alignment", "label": "Emergent Cross-modal Alignment", "role": "output", "required": true} + {"id": "modality_encoders", "label": "Modality Encoders", "aliases": ["Transformer-based Encoders for Each Modality", "Encoders for Each Modality"], "role": "module", "required": true}, + {"id": "joint_embedding", "label": "Joint Embedding Space", "aliases": ["Single Shared Joint Embedding Space", "Single Shared Representation Space", "Joint Embedding"], "role": "module", "required": true}, + {"id": "emergent_alignment", "label": "Emergent Cross-modal Alignment", "aliases": ["Emergent Alignment", "Binding Agent", "Binding Property"], "role": "output", "required": true} ], "relations": [ {"source": "image", "target": "modality_encoders", "type": "encoding", "required": true}, diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json index dbe7ad7..21ca329 100644 --- a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/case.json @@ -4,5 +4,5 @@ "paper": "inputs/paper.pdf", "source": "source.json", "expected_semantics": "expected_semantics.json", "preferences": "../../preferences/top_conference.json", "positive_references": [], "negative_references": [], "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, - "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "hallucination_count": 0, "forbidden_content_count": 0} + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} } diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json index 394caaa..c881bb8 100644 --- a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json @@ -3,9 +3,9 @@ "entities": [ {"id": "promptable_task", "label": "Promptable Segmentation Task", "role": "panel", "required": true}, {"id": "image", "label": "Image", "role": "input", "required": true}, - {"id": "prompt", "label": "Prompt", "role": "input", "required": true}, - {"id": "valid_mask", "label": "Valid Segmentation Mask", "role": "output", "required": true}, - {"id": "sam_model", "label": "Segment Anything Model", "role": "panel", "required": true}, + {"id": "prompt", "label": "Prompt", "aliases": ["Sparse Prompts", "Dense Prompts", "Segmentation Prompt"], "role": "input", "required": true}, + {"id": "valid_mask", "label": "Valid Segmentation Mask", "aliases": ["Valid Masks", "Valid Mask"], "role": "output", "required": true}, + {"id": "sam_model", "label": "Segment Anything Model", "aliases": ["Segmentation Model (SAM)", "SAM"], "role": "panel", "required": true}, {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, {"id": "prompt_encoder", "label": "Prompt Encoder", "role": "module", "required": true}, {"id": "mask_decoder", "label": "Mask Decoder", "role": "module", "required": true}, diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index be7ef90..bd546b7 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -2,6 +2,16 @@ `rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a content-free architecture blueprint, reference-conditioned Image2 edit generation, and four production quality gates. +## Three-Minute Fast Contract + +```powershell +rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --json +``` + +The fast path stops before image generation. It uses structured block-level PDF extraction, complete Figure 1/overview captions, stable page/content-hash evidence IDs, one compact VLM contract call, deterministic evidence rules, and a local SHA-256 contract cache. A cached paper normally completes in seconds. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. + +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, and parser agreement without model calls. + ## Production Command ```powershell diff --git a/rfs/cli.py b/rfs/cli.py index 6cb5b60..905b6f7 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -11,7 +11,7 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable -from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case +from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, score_benchmark_case, validate_benchmark_case from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro @@ -68,6 +68,7 @@ def _doctor() -> dict: "pdftoppm": {"available": bool(shutil.which("pdftoppm")), "path": shutil.which("pdftoppm")}, "easyocr": {"available": importlib.util.find_spec("easyocr") is not None}, "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, + "fast_contract_cache": {"available": True, "path": str(Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")))}, } ok = all(item["available"] for item in deps.values()) return { @@ -90,6 +91,7 @@ def _doctor() -> dict: "Use rfs presentations-qa as an optional inspection pass; RFS remains the authoritative PPTX compiler.", "Use rfs coevolve-image to run whole-image Creator Agent and Online/Frozen Judge refinement before PPTX conversion.", "Use rfs paper-to-image for evidence-grounded paper summarization, whole-image prompt compilation, candidate generation, and selected_image.png without PPTX.", + "Use rfs fast-framework-prompt for the one-call, cached paper-to-contract fast path; RFS_FAST_FRAMEWORK_MODEL defaults to gemini-2.5-flash.", "Production paper-to-image uses a content-free template blueprint and requires an Image2 edit endpoint; placeholder output is engineering-only.", ], } @@ -284,13 +286,17 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") - benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "run", "score"]) + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "run", "score"]) benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") + benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") + benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") + benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") + benchmark.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for benchmark fast.") benchmark.add_argument("--json", action="store_true", help="Emit JSON.") validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") @@ -544,6 +550,10 @@ def main(argv: list[str] | None = None) -> int: if not args.case or not args.out: parser.error("benchmark run requires --case and --out") result = run_benchmark_case(args.case, args.out) + elif args.benchmark_action == "fast": + if not args.case or not args.out: + parser.error("benchmark fast requires --case and --out") + result = run_fast_benchmark_case(args.case, args.out, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine) else: if not args.case or not args.run_dir: parser.error("benchmark score requires --case and --run") diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index 7e86ec0..b903475 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -1,13 +1,14 @@ """Deterministic and model-assisted quality evaluation APIs.""" from .rebuild_visual import run_rebuild_visual_quality_check -from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, score_benchmark_case, validate_benchmark_case +from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, score_benchmark_case, validate_benchmark_case __all__ = [ "list_benchmark_cases", "fetch_benchmark_case", "run_rebuild_visual_quality_check", "run_benchmark_case", + "run_fast_benchmark_case", "score_benchmark_case", "validate_benchmark_case", ] diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index b38cf54..2e7c199 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -67,50 +67,80 @@ def _score_planning_contract(expected: dict, specification: dict) -> dict[str, A if not specification: return {"available": False} actual_entities = [] - for field in ("inputs", "modules", "outputs", "innovations"): + for field in ("inputs", "modules", "outputs", "innovations", "must_show"): for index, item in enumerate(specification.get(field, []) if isinstance(specification.get(field), list) else []): if not isinstance(item, dict): continue label = next((str(item.get(key) or "").strip() for key in ("visible_label", "name", "text", "statement", "label", "id") if str(item.get(key) or "").strip()), "") - actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label)}) + actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label), "field": field}) + expanded_relations = set() + for index, item in enumerate(specification.get("relations", []) if isinstance(specification.get("relations"), list) else []): + if not isinstance(item, dict): + continue + source = str(item.get("source") or item.get("source_id") or "") + target = str(item.get("target") or item.get("target_id") or "") + if source and target: + expanded_relations.add((source, target)) + label = str(item.get("label") or "").strip() + if source and target and label: + label_id = f"relation_label_{index}" + actual_entities.append({"id": label_id, "label": label, "normalized": _normalized_text(label), "field": "relation_label"}) + expanded_relations.add((source, label_id)) + expanded_relations.add((label_id, target)) expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] mapping: dict[str, str] = {} matched = [] used_actual_ids: set[str] = set() for item in expected_entities: - candidates = [str(item.get("label") or "")] + [str(value) for value in item.get("aliases", []) if str(value).strip()] - normalized = [_normalized_text(value) for value in candidates if _normalized_text(value)] - match = next( - ( - actual - for actual in actual_entities - if actual["id"] not in used_actual_ids - and actual["normalized"] - and any( - value == actual["normalized"] - or value in actual["normalized"] - or actual["normalized"] in value - for value in normalized - ) - ), - None, - ) + primary = _normalized_text(item.get("label")) + aliases = [_normalized_text(value) for value in item.get("aliases", []) if _normalized_text(value)] + def match_score(actual: dict[str, str]) -> int: + value = actual["normalized"] + if not value or actual["id"] in used_actual_ids: + return 0 + bonus = 2 if actual.get("field") in {"inputs", "modules", "outputs", "innovations"} else 1 if actual.get("field") == "relation_label" else 0 + if primary and primary == value: + return 5 + bonus + if any(alias == value for alias in aliases): + return 4 + bonus + if any(alias in value or value in alias for alias in aliases): + return 3 + bonus + if primary and (primary in value or value in primary): + return 1 + bonus + return 0 + ranked = sorted(((match_score(actual), actual) for actual in actual_entities), key=lambda pair: pair[0], reverse=True) + match = ranked[0][1] if ranked and ranked[0][0] > 0 else None if match: mapping[str(item.get("id"))] = match["id"] used_actual_ids.add(match["id"]) matched.append(str(item.get("id"))) expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] - actual_relations = { - (str(item.get("source") or item.get("source_id") or ""), str(item.get("target") or item.get("target_id") or "")) - for item in specification.get("relations", []) - if isinstance(item, dict) - } + actual_relations = expanded_relations + adjacency: dict[str, set[str]] = {} + for source, target in actual_relations: + adjacency.setdefault(source, set()).add(target) + def connected(source: str, target: str, max_hops: int = 2) -> bool: + if (source, target) in actual_relations: + return True + frontier = {source} + visited = {source} + for _ in range(max_hops): + frontier = {neighbor for node in frontier for neighbor in adjacency.get(node, set()) if neighbor not in visited} + if target in frontier: + return True + visited.update(frontier) + return False relation_matches = 0 + matched_relations = [] + missing_relations = [] for relation in expected_relations: source = mapping.get(str(relation.get("source"))) target = mapping.get(str(relation.get("target"))) - if source and target and (source, target) in actual_relations: + if source and target and connected(source, target): relation_matches += 1 + matched_relations.append(f"{relation.get('source')}->{relation.get('target')}") + else: + missing_relations.append(f"{relation.get('source')}->{relation.get('target')}") actual_text = " ".join(item["label"] for item in actual_entities).casefold() forbidden = [label for label in expected.get("forbidden_labels", []) if str(label).casefold() in actual_text] return { @@ -118,7 +148,10 @@ def _score_planning_contract(expected: dict, specification: dict) -> dict[str, A "entity_recall": _clamp(len(matched) / max(1, len(expected_entities))), "relation_recall": _clamp(relation_matches / max(1, len(expected_relations))), "matched_entity_ids": matched, + "entity_mapping": mapping, "missing_entity_ids": [str(item.get("id")) for item in expected_entities if str(item.get("id")) not in matched], + "matched_relations": matched_relations, + "missing_relations": missing_relations, "forbidden_labels_found": forbidden, } @@ -305,9 +338,11 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, hard_failures.append("required relations missing or incorrect") if exact_label_rate < 1.0: hard_failures.append("required labels missing or misspelled") - if planning_contract.get("available") and float(planning_contract.get("entity_recall", 0.0)) < 1.0: + plan_entity_min = float(case.get("thresholds", {}).get("plan_entity_recall", 1.0)) + plan_relation_min = float(case.get("thresholds", {}).get("plan_relation_recall", 1.0)) + if planning_contract.get("available") and float(planning_contract.get("entity_recall", 0.0)) < plan_entity_min: hard_failures.append("paper planning missed required benchmark entities") - if planning_contract.get("available") and float(planning_contract.get("relation_recall", 0.0)) < 1.0: + if planning_contract.get("available") and float(planning_contract.get("relation_recall", 0.0)) < plan_relation_min: hard_failures.append("paper planning missed required benchmark relations") if planning_contract.get("forbidden_labels_found"): hard_failures.append("paper planning introduced forbidden benchmark labels") @@ -616,3 +651,49 @@ def run_benchmark_case(case_dir: str | Path, out: str | Path) -> dict[str, Any]: } write_json(target / "benchmark_run.json", result) return result + + +def run_fast_benchmark_case( + case_dir: str | Path, + out: str | Path, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "off", +) -> dict[str, Any]: + case_root = Path(case_dir).resolve() + case = _load(case_root / "case.json") + if str(case.get("suite") or "") != "paper-to-image": + return {"summary": "Fast framework benchmark only supports paper-to-image cases.", "ok": False, "case_dir": str(case_root)} + paper_path = case_root / str(case.get("paper") or "paper.md") + if not paper_path.exists(): + fetched = fetch_benchmark_case(case_root) + if not fetched.get("ok"): + return {**fetched, "passed": False} + validation = validate_benchmark_case(case_root) + if not validation.get("ok"): + return {**validation, "passed": False} + from ..paper_to_image import run_fast_framework_prompt + + preferences = case_root / str(case.get("preferences")) if case.get("preferences") else None + run = run_fast_framework_prompt( + paper=paper_path, + out=out, + deadline_seconds=deadline_seconds, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine, + preferences_path=preferences if preferences and preferences.exists() else None, + aspect_ratio=str(case.get("run_config", {}).get("aspect_ratio") or "16:9"), + ) + score = score_benchmark_case(case_root, out) + result = { + "summary": "Fast paper framework benchmark completed.", + "ok": bool(run.get("ok")), + "case_id": case.get("case_id"), + "run": run, + "benchmark": score, + "passed_planning_thresholds": not any("plan_" in value for value in score.get("threshold_failures", [])), + } + write_json(Path(out) / "fast_benchmark_result.json", result) + return result diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index f79f904..c4090cc 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import json import re import shutil import subprocess @@ -32,6 +33,7 @@ def _clean(text: str) -> str: value = re.sub(r"\bANIMAGE\b", "AN IMAGE", value) value = re.sub(r"\(\s*([A-Z])\s+([A-Z])\s+([A-Z])\s*\)", r"(\1\2\3)", value) value = re.sub(r"[ \t]+", " ", value) + value = re.sub(r"(?<=\w)-\s+(?=\w)", "", value) value = re.sub(r"\n{3,}", "\n\n", value) return value.strip() @@ -98,7 +100,7 @@ def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[lis def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: records = [] payload = page.get_text("dict") - for block in payload.get("blocks", []): + for parent_index, block in enumerate(payload.get("blocks", []), 1): if int(block.get("type", 0)) != 0: continue for line in block.get("lines", []): @@ -114,7 +116,7 @@ def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: round(max(float(value[2]) for value in bbox_values), 3), round(max(float(value[3]) for value in bbox_values), 3), ] - records.append({"bbox": bbox, "text": text, "source": "pymupdf", "confidence": 1.0}) + records.append({"bbox": bbox, "text": text, "source": "pymupdf", "confidence": 1.0, "parent_block": parent_index}) return records @@ -389,9 +391,18 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) tables = [] formulas = [] for page in pages: - for block in page.get("blocks", []): + page_blocks = page.get("blocks", []) + for block_index, block in enumerate(page_blocks): record = {"page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "caption": block.get("text", "")} if block.get("kind") == "caption": + parent = block.get("parent_block") + continuation = [] + for following in page_blocks[block_index + 1:]: + if parent is None or following.get("parent_block") != parent or re.match(r"^(figure|fig\.|table)\s*\d", str(following.get("text") or ""), re.IGNORECASE): + break + continuation.append(str(following.get("text") or "")) + if continuation: + record["caption"] = _clean(record["caption"] + " " + " ".join(continuation)) figures.append({"id": f"figure_{len(figures) + 1:03d}", **record}) elif block.get("kind") == "table": tables.append({"id": f"table_{len(tables) + 1:03d}", **record}) @@ -409,10 +420,32 @@ def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any] } -def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]: +def _evidence_id(page: int, text: str, bbox: Any, kind: str) -> str: + payload = json.dumps({"page": page, "text": _clean(text), "bbox": bbox, "kind": kind}, ensure_ascii=False, sort_keys=True).encode("utf-8") + return f"E_P{int(page):03d}_{hashlib.sha1(payload).hexdigest()[:12]}" + + +def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], document_index: dict[str, Any], max_chars: int) -> list[dict[str, Any]]: section_positions = sorted(sections, key=lambda item: (item["page"], item.get("block_id", ""))) evidence = [] used = 0 + for figure in document_index.get("figures", []): + text = _clean(figure.get("caption", "")) + if not text or used + len(text) > max_chars: + continue + evidence.append({ + "id": _evidence_id(int(figure.get("page") or 0), text, figure.get("bbox"), "caption"), + "page": figure.get("page"), + "block_id": figure.get("block_id"), + "bbox": figure.get("bbox"), + "section_hint": "Figure Captions", + "kind": "caption", + "source": "pymupdf", + "confidence": 1.0, + "text": text, + "char_count": len(text), + }) + used += len(text) for page in pages: for block in page.get("blocks", []): text = _clean(block.get("text", "")) @@ -421,7 +454,7 @@ def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], text = text[: max(0, max_chars - used)] section = next((item["title"] for item in reversed(section_positions) if item["page"] <= page["page"]), None) evidence.append({ - "id": f"E{len(evidence) + 1:04d}", + "id": _evidence_id(int(page["page"]), text, block.get("bbox"), str(block.get("kind") or "paragraph")), "page": page["page"], "block_id": block.get("id"), "bbox": block.get("bbox"), @@ -433,6 +466,8 @@ def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], "char_count": len(text), }) used += len(text) + for index, item in enumerate(evidence, 1): + item["legacy_id"] = f"E{index:04d}" return evidence @@ -494,7 +529,7 @@ def parse_paper( details = {"warnings": [], "ocr_candidate_pages": [], "ocr_pages": [], "poppler_available": False, "elapsed_seconds": 0.0} sections = _heading_candidates(pages) document_index = _document_index(pages, sections) - evidence = _build_evidence(pages, sections, max_chars) + evidence = _build_evidence(pages, sections, document_index, max_chars) all_text = "\n\n".join(page.get("text", "") for page in pages) if not evidence: raise ValueError("No readable paper text was extracted") @@ -528,7 +563,11 @@ def evidence_excerpt(parsed: dict, max_chars: int = 58000) -> str: prioritized = sorted( parsed.get("evidence", []), key=lambda item: ( - 0 if any(term in str(item.get("section_hint") or "").casefold() for term in ("abstract", "method", "approach", "architecture", "framework", "conclusion")) else 1, + 0 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else + 1 if item.get("kind") == "caption" and re.search(r"\b(overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else + 2 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) else + 3 if item.get("kind") == "caption" else + 4 if any(term in str(item.get("section_hint") or "").casefold() for term in ("abstract", "introduction", "method", "approach", "architecture", "framework", "conclusion")) else 5, int(item.get("page") or 0), str(item.get("block_id") or ""), ), diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index b86a3d6..08defc9 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -35,7 +35,7 @@ def merge_preferences(raw: dict | None, aspect_ratio: str | None = None, languag return merged -def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = None) -> str: +def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = None, evidence_max_chars: int = 58000) -> str: return f""" # Summary @@ -45,6 +45,15 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = Every important factual item must include one or more evidence_ids copied exactly from the supplied evidence labels. Mark uncertain fields as unknown. Use the paper's exact terminology. +Contract completeness rules: +- Treat Figure 1 and captions containing "overview", "framework", "architecture", or "pipeline" as high-priority evidence for the intended paper overview figure. +- Identify every major contribution pillar from the abstract and introduction. For system, dataset, or foundation-model papers, do not omit a data engine, dataset construction process, training loop, or deployment stage when it is a central contribution. +- Represent each separately named scientific component as a separate entity. Do not merge conditioning signals, special tokens, encoders, heads, inputs, or outputs into composite nodes when the paper names them independently. +- Give every input and output a stable id and include the boundary relations from input to the first processing entity and from the final processing entity to the output. +- Every relation endpoint must be an id declared in inputs, modules, outputs, or innovations. +- Include feedback and control edges explicitly instead of describing them only in prose. +- Before returning, check that every item in must_show is represented by a distinct entity or an explicit relation label. + Return this schema: {{ "summary": "Paper-to-image planning result.", @@ -72,8 +81,8 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = "must_show": [{{"text": "...", "evidence_ids": ["E0001"]}}], "modules": [{{"id": "...", "name": "...", "role": "...", "evidence_ids": ["E0001"]}}], "relations": [{{"source": "module_id", "target": "module_id", "type": "data_flow|control_flow|training_only|inference_only|comparison|feedback", "label": "", "evidence_ids": ["E0001"]}}], - "inputs": [], - "outputs": [], + "inputs": [{{"id": "stable_input_id", "name": "exact paper term", "evidence_ids": ["E0001"]}}], + "outputs": [{{"id": "stable_output_id", "name": "exact paper term", "evidence_ids": ["E0001"]}}], "innovations": [], "feedback_loops": [], "training_flow": [], @@ -126,7 +135,7 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = {json.dumps(paper_review or {}, ensure_ascii=False, indent=2)} Paper evidence: -{evidence_excerpt(parsed)} +{evidence_excerpt(parsed, max_chars=evidence_max_chars)} """.strip() @@ -137,18 +146,40 @@ def _first_sentence(text: str) -> str: def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = None) -> dict: if paper_review and paper_review.get("modules"): - review_modules = paper_review.get("modules", [])[:10] + review_modules = [item for item in (list(paper_review.get("modules", [])) + list(paper_review.get("research_objects", [])) + list(paper_review.get("concepts", []))) if item.get("evidence_ids")][:16] modules = [{"id": str(item.get("id")), "name": str(item.get("visible_label") or item.get("statement") or item.get("id")), "role": str(item.get("visual_role") or "module"), "evidence_ids": list(item.get("evidence_ids", []))} for item in review_modules] - relations = [{"source": str(item.get("source_id") or item.get("source") or ""), "target": str(item.get("target_id") or item.get("target") or ""), "type": str(item.get("relation_type") or item.get("type") or "data_flow"), "label": str(item.get("statement") or "")[:48], "evidence_ids": list(item.get("evidence_ids", []))} for item in paper_review.get("relations", []) if (item.get("source_id") or item.get("source")) and (item.get("target_id") or item.get("target"))] + relations = [{"source": str(item.get("source_id") or item.get("source") or ""), "target": str(item.get("target_id") or item.get("target") or ""), "type": str(item.get("relation_type") or item.get("type") or "data_flow"), "label": str(item.get("statement") or "")[:48], "evidence_ids": list(item.get("evidence_ids", []))} for item in paper_review.get("relations", []) if (item.get("source_id") or item.get("source")) and (item.get("target_id") or item.get("target")) and item.get("evidence_ids")] terminology = {str(item.get("statement")): str(item.get("visible_label") or item.get("statement")) for item in paper_review.get("terminology", []) if str(item.get("statement") or "").strip()} title = str(paper_review.get("paper_identity", {}).get("title") or parsed.get("source_name")) research_questions = paper_review.get("research_questions", []) claims = paper_review.get("central_claims", []) figure_goal = str((claims or research_questions or [{"statement": "Explain the paper method faithfully."}])[0].get("statement")) - innovations = paper_review.get("innovations", []) + innovations = [item for item in paper_review.get("innovations", []) if item.get("evidence_ids")] forbidden = [str(item.get("statement")) for item in paper_review.get("forbidden_inventions", [])] - inputs = paper_review.get("inputs", []) - outputs = paper_review.get("outputs", []) + inputs = [item for item in paper_review.get("inputs", []) if item.get("evidence_ids")] + outputs = [item for item in paper_review.get("outputs", []) if item.get("evidence_ids")] + known_modalities = ["image", "text", "audio", "depth", "thermal", "imu", "video"] + expanded_inputs = [] + for item in inputs: + statement = str(item.get("visible_label") or item.get("name") or item.get("statement") or "") + present = [term for term in known_modalities if re.search(rf"\b{term}(?:s)?\b", statement, re.IGNORECASE)] + if len(present) >= 3: + for term in present: + expanded_inputs.append({"id": f"input_{term}", "name": term.upper() if term == "imu" else term.title(), "role": "input modality", "evidence_ids": list(item.get("evidence_ids", []))}) + else: + expanded_inputs.append(item) + inputs = expanded_inputs + if modules: + relation_pairs = {(item["source"], item["target"]) for item in relations} + for item in inputs: + input_id = str(item.get("id") or "") + if input_id and (input_id, modules[0]["id"]) not in relation_pairs: + relations.append({"source": input_id, "target": modules[0]["id"], "type": "data_flow", "label": "encoding", "evidence_ids": list(item.get("evidence_ids", []))}) + if outputs and innovations: + output_id = str(outputs[0].get("id") or "") + innovation_id = str(innovations[0].get("id") or "") + if output_id and innovation_id and (output_id, innovation_id) not in relation_pairs: + relations.append({"source": output_id, "target": innovation_id, "type": "enables", "label": "emergent alignment", "evidence_ids": list(innovations[0].get("evidence_ids", []))}) return { "summary": "Paper-review-grounded fallback planning result.", "paper_summary": {"summary": "Structured paper summary derived from paper_review.json.", "title": title, "paper_type": paper_review.get("paper_identity", {}).get("paper_type", "unknown"), "research_problem": (research_questions or [{}])[0], "central_claim": (claims or [{}])[0], "inputs": inputs, "outputs": outputs, "core_modules": modules, "innovations": innovations, "training_flow": paper_review.get("workflows", {}).get("training", []), "inference_flow": paper_review.get("workflows", {}).get("inference", []), "terminology": terminology, "unknowns": paper_review.get("unknowns", [])}, @@ -241,6 +272,11 @@ def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = } +def build_review_grounded_plan(parsed: dict, preferences: dict, paper_review: dict) -> dict: + """Compile a deterministic figure plan from an evidence-validated paper review.""" + return normalize_plan(_heuristic_plan(parsed, preferences, paper_review=paper_review), preferences) + + def _ensure_summary(value: Any, fallback: str) -> dict: data = value if isinstance(value, dict) else {} data.setdefault("summary", fallback) @@ -260,8 +296,8 @@ def normalize_plan(raw: dict, preferences: dict) -> dict: return result -def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1) -> tuple[dict, dict]: - prompt = _planner_prompt(parsed, preferences, paper_review=paper_review) +def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1, evidence_max_chars: int = 58000) -> tuple[dict, dict]: + prompt = _planner_prompt(parsed, preferences, paper_review=paper_review, evidence_max_chars=evidence_max_chars) metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) @@ -337,6 +373,8 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = - Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. - Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. - Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. +- Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. +- Include explicit input-boundary and output-boundary connectors. """.strip() diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 43044c4..93b6454 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -1,14 +1,17 @@ from __future__ import annotations +import hashlib import json +import os +import re import shutil import time from pathlib import Path from typing import Any, Callable -from ..utils import ensure_dir, write_json, write_text +from ..utils import ensure_dir, read_json, write_json, write_text from .analyzer import paper_markdown, parse_paper -from .planner import compile_image_prompt, merge_preferences, plan_paper_image, validate_plan_grounding +from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_paper_image, validate_plan_grounding from .review import build_paper_review, detect_domain_profile, validate_review_coverage @@ -63,6 +66,46 @@ def _collect_evidence_ids(value: Any) -> set[str]: return found +def _review_evidence_map(value: Any) -> dict[str, list[str]]: + mapping: dict[str, list[str]] = {} + if isinstance(value, dict): + item_id = str(value.get("id") or "").strip() + evidence_ids = [str(item) for item in value.get("evidence_ids", []) if str(item).startswith("E")] + if item_id and evidence_ids: + mapping[item_id] = evidence_ids + for child in value.values(): + mapping.update(_review_evidence_map(child)) + elif isinstance(value, list): + for child in value: + mapping.update(_review_evidence_map(child)) + return mapping + + +def expand_plan_evidence(plan: dict[str, Any], paper_review: dict[str, Any], parsed: dict[str, Any]) -> None: + mapping = _review_evidence_map(paper_review) + valid = {item["id"] for item in parsed.get("evidence", [])} + legacy = {item.get("legacy_id"): item["id"] for item in parsed.get("evidence", []) if item.get("legacy_id")} + def expand(value: Any) -> None: + if isinstance(value, dict): + if isinstance(value.get("evidence_ids"), list): + expanded = [] + for item in value["evidence_ids"]: + key = str(item) + if key in valid: + expanded.append(key) + elif key in legacy: + expanded.append(legacy[key]) + else: + expanded.extend(mapping.get(key, [])) + value["evidence_ids"] = list(dict.fromkeys(expanded)) + for child in value.values(): + expand(child) + elif isinstance(value, list): + for child in value: + expand(child) + expand(plan) + + def _item_id(item: dict[str, Any], field: str, index: int) -> str: return str(item.get("id") or item.get("name") or item.get("visible_label") or item.get("text") or item.get("statement") or f"{field}_{index}").strip() @@ -71,6 +114,15 @@ def _item_label(item: dict[str, Any]) -> str: return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() +def _normalized_label(value: str) -> str: + return "".join(char for char in value.casefold() if char.isalnum()) + + +def _stable_id(prefix: str, value: str, index: int) -> str: + slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", value.casefold()).split() if part)[:48] + return f"{prefix}_{slug or index}" + + def _infer_topology(spec: dict[str, Any]) -> str: relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] if any("feedback" in str(item.get("type") or "").casefold() for item in relations): @@ -86,12 +138,184 @@ def _infer_topology(spec: dict[str, Any]) -> str: outgoing[source] = outgoing.get(source, 0) + 1 if any(value > 1 for value in outgoing.values()): return "branch" - total = sum(len(spec.get(field, []) if isinstance(spec.get(field), list) else []) for field in ("inputs", "modules", "outputs", "innovations")) - if total >= 12: + overview_panels = [item for item in spec.get("modules", []) if isinstance(item, dict) and str(item.get("role") or "") == "overview_panel"] + if len(overview_panels) >= 3: return "dense_multiframe" return "linear" if relations else "unknown" +def _complete_from_overview_caption(spec: dict[str, Any], parsed: dict[str, Any]) -> None: + figures = parsed.get("document_index", {}).get("figures", []) + figure = next((item for item in figures if re.match(r"^(figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE)), None) + figure = figure or next((item for item in figures if re.search(r"\b(overview|framework|architecture|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE)), None) + figure = figure or next((item for item in figures if re.match(r"^(figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE)), None) + if not figure: + return + caption = str(figure.get("caption") or "") + caption_evidence = next((item["id"] for item in parsed.get("evidence", []) if item.get("kind") == "caption" and item.get("page") == figure.get("page") and str(item.get("text") or "").startswith(str(caption)[:60])), None) + evidence_ids = [caption_evidence] if caption_evidence else [] + match = re.search(r"(?:interconnected\s+)?components\s*:\s*(.+)", caption, re.IGNORECASE) + component_ids: dict[str, str] = {} + if match: + raw_components = re.split(r",\s+(?=(?:and\s+)?(?:a|an)\s+)", match.group(1)) + existing = {_normalized_label(_item_label(item)) for field in ("modules", "inputs", "outputs", "innovations") for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) if isinstance(item, dict)} + for index, raw in enumerate(raw_components): + phrase = re.sub(r"^(?:and\s+)?(?:a|an)\s+", "", raw.strip(), flags=re.IGNORECASE) + phrase = re.split(r"\s+(?:that|which|for|by|to)\s+", phrase, maxsplit=1, flags=re.IGNORECASE)[0].strip(" .") + if not 3 <= len(phrase) <= 80: + continue + label = phrase[0].upper() + phrase[1:] + normalized = _normalized_label(label) + existing_match = next((item for item in spec.get("modules", []) if isinstance(item, dict) and normalized and (normalized in _normalized_label(_item_label(item)) or _normalized_label(_item_label(item)) in normalized)), None) + if existing_match: + component_ids[normalized] = str(existing_match.get("id")) + continue + if normalized not in existing: + item_id = _stable_id("overview", label, index) + spec.setdefault("modules", []).append({"id": item_id, "name": label, "role": "overview_panel", "evidence_ids": evidence_ids}) + existing.add(normalized) + component_ids[normalized] = item_id + dataset_match = re.search(r"\b([A-Z]{2,}[A-Z0-9-]*\d[A-Z0-9-]*)\b", caption) if re.search(r"\bdataset\b|data engine|collecting", caption, re.IGNORECASE) else None + dataset_item = None + if dataset_match: + dataset_label = dataset_match.group(1) + dataset_item = next((item for item in spec.get("outputs", []) if isinstance(item, dict) and dataset_label.casefold() in _item_label(item).casefold()), None) + if not dataset_item: + dataset_item = {"id": _stable_id("dataset", dataset_label, 0), "name": dataset_label, "role": "dataset", "evidence_ids": evidence_ids} + spec.setdefault("outputs", []).append(dataset_item) + model_item = next((item for item in spec.get("modules", []) if isinstance(item, dict) and ("segmentation model" in _item_label(item).casefold() or "segment anything model" in _item_label(item).casefold())), None) + model_item = model_item or next((item for item in spec.get("modules", []) if isinstance(item, dict) and ("model" in _item_label(item).casefold() or "sam" in _item_label(item).casefold())), None) + engine_item = next((item for item in spec.get("modules", []) if isinstance(item, dict) and "data engine" in _item_label(item).casefold()), None) + relations = spec.setdefault("relations", []) + pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + if model_item and engine_item and (str(model_item.get("id")), str(engine_item.get("id"))) not in pairs and re.search(r"powers?\s+data\s+annotation|data\s+engine", caption, re.IGNORECASE): + relations.append({"source": str(model_item.get("id")), "target": str(engine_item.get("id")), "type": "annotation_support", "label": "", "evidence_ids": evidence_ids}) + pairs.add((str(model_item.get("id")), str(engine_item.get("id")))) + evidence = parsed.get("evidence", []) + for index, item in enumerate(evidence): + if not re.search(r"has\s+(?:three|3)\s+stages", str(item.get("text") or ""), re.IGNORECASE): + continue + combined = " ".join(str(value.get("text") or "") for value in evidence[index:index + 3]) + stage_match = re.search(r"([A-Za-z]+(?:-[A-Za-z]+)?)\s*,\s*([A-Za-z]+(?:-[A-Za-z]+)?)\s*,\s*(?:and\s+)?([A-Za-z]+(?:\s+[A-Za-z]+)?)\s*\.", combined) + if not stage_match: + continue + stage_evidence = [value["id"] for value in evidence[index:index + 3]] + stage_items = [] + for stage_index, value in enumerate(stage_match.groups()): + label = value.strip().title().replace("-", "-") + existing_stage = next((module for module in spec.get("modules", []) if isinstance(module, dict) and _normalized_label(label) == _normalized_label(_item_label(module))), None) + if existing_stage: + stage_items.append(existing_stage) + else: + stage = {"id": _stable_id("stage", label, stage_index), "name": label, "role": "data_engine_stage", "evidence_ids": stage_evidence} + spec.setdefault("modules", []).append(stage) + stage_items.append(stage) + for source, target in zip(stage_items, stage_items[1:]): + pair = (str(source.get("id")), str(target.get("id"))) + if pair not in pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "stage_transition", "label": "", "evidence_ids": stage_evidence}) + pairs.add(pair) + if dataset_item and stage_items: + pair = (str(stage_items[-1].get("id")), str(dataset_item.get("id"))) + if pair not in pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "data_generation", "label": "", "evidence_ids": stage_evidence + evidence_ids}) + break + + def find_evidence(*terms: str) -> list[str]: + found = next((item for item in evidence if all(term.casefold() in str(item.get("text") or "").casefold() for term in terms)), None) + return [found["id"]] if found else evidence_ids + + def ensure_entity(field: str, label: str, role: str, item_evidence: list[str]) -> dict[str, Any]: + normalized = _normalized_label(label) + existing = next((item for item in spec.get(field, []) if isinstance(item, dict) and normalized and normalized == _normalized_label(_item_label(item))), None) + if existing: + return existing + item = {"id": _stable_id(field.rstrip("s"), label, len(spec.get(field, []))), "name": label, "role": role, "evidence_ids": item_evidence} + spec.setdefault(field, []).append(item) + return item + + def ensure_relation(source: dict[str, Any], target: dict[str, Any], relation_type: str, relation_evidence: list[str]) -> None: + pair = (str(source.get("id")), str(target.get("id"))) + if pair not in pairs: + relations.append({"source": pair[0], "target": pair[1], "type": relation_type, "label": "", "evidence_ids": relation_evidence}) + pairs.add(pair) + + low_caption = caption.casefold() + if "patch" in low_caption and "transformer encoder" in low_caption: + image = ensure_entity("inputs", "Input Image", "input", evidence_ids) + patches = ensure_entity("modules", "Image Patches", "module", evidence_ids) + projection = ensure_entity("modules", "Linear Projection of Flattened Patches", "module", evidence_ids) + position = ensure_entity("modules", "Position Embedding", "conditioning", evidence_ids) + token = ensure_entity("modules", "Class Token", "conditioning", evidence_ids) + encoder = ensure_entity("modules", "Transformer Encoder", "module", evidence_ids) + mlp_evidence = find_evidence("mlp", "head") + mlp = ensure_entity("modules", "MLP Head", "module", mlp_evidence) + prediction = ensure_entity("outputs", "Class Prediction", "output", mlp_evidence) + for source, target, relation_type in [ + (image, patches, "data_flow"), (patches, projection, "data_flow"), (projection, encoder, "data_flow"), + (position, encoder, "conditioning"), (token, encoder, "conditioning"), (encoder, mlp, "data_flow"), (mlp, prediction, "data_flow"), + ]: + ensure_relation(source, target, relation_type, list(dict.fromkeys(source.get("evidence_ids", []) + target.get("evidence_ids", [])))) + + modality_names = [term for term in ("image", "text", "audio", "depth", "thermal", "imu") if re.search(rf"\b{term}(?:s)?\b", low_caption)] + if len(modality_names) >= 4 and ("common embedding" in low_caption or "joint embedding" in low_caption): + modality_inputs = [ensure_entity("inputs", term.upper() if term == "imu" else term.title(), "input modality", evidence_ids) for term in modality_names] + encoders = ensure_entity("modules", "Modality Encoders", "module group", find_evidence("encoder")) + joint = ensure_entity("modules", "Joint Embedding Space", "shared representation", evidence_ids) + emergent = ensure_entity("outputs", "Emergent Cross-modal Alignment", "output", evidence_ids) + for item in modality_inputs: + ensure_relation(item, encoders, "encoding", list(dict.fromkeys(item.get("evidence_ids", []) + encoders.get("evidence_ids", [])))) + ensure_relation(encoders, joint, "alignment", list(dict.fromkeys(encoders.get("evidence_ids", []) + joint.get("evidence_ids", [])))) + ensure_relation(joint, emergent, "enables", evidence_ids) + + if "feedback" in low_caption and ("refine" in low_caption or "refines" in low_caption) and ("generating an output" in low_caption or "initial output" in low_caption): + task = ensure_entity("inputs", "Input", "input", evidence_ids) + generate = ensure_entity("modules", "Generate", "generator", evidence_ids) + initial = ensure_entity("modules", "Initial Output", "artifact", evidence_ids) + feedback = ensure_entity("modules", "Feedback", "feedback provider", evidence_ids) + feedback_text = ensure_entity("modules", "Self-Feedback", "feedback artifact", evidence_ids) + refine = ensure_entity("modules", "Refine", "refiner", evidence_ids) + refined = ensure_entity("outputs", "Refined Output", "output", evidence_ids) + for source, target, relation_type in [ + (task, generate, "data_flow"), (generate, initial, "data_flow"), (initial, feedback, "evaluation"), + (feedback, feedback_text, "feedback"), (initial, refine, "revision_input"), (feedback_text, refine, "feedback"), + (refine, refined, "data_flow"), (refined, feedback, "feedback_loop"), + ]: + ensure_relation(source, target, relation_type, evidence_ids) + + if re.search(r"mask\s*-?r-?cnn", low_caption, re.IGNORECASE) and "framework" in low_caption: + input_image = ensure_entity("inputs", "Input Image", "input", find_evidence("image")) + backbone = ensure_entity("modules", "Backbone", "feature extractor", find_evidence("backbone")) + proposals = ensure_entity("modules", "Region Proposals", "proposal artifact", find_evidence("region proposal")) + roi = ensure_entity("modules", "RoIAlign", "alignment", find_evidence("roialign")) + classification = ensure_entity("outputs", "Classification", "output head", find_evidence("class")) + box = ensure_entity("outputs", "Bounding-box Regression", "output head", find_evidence("bounding box")) + mask = ensure_entity("outputs", "Mask Branch", "output head", find_evidence("mask branch")) + for source, target, relation_type in [ + (input_image, backbone, "data_flow"), (backbone, proposals, "data_flow"), (proposals, roi, "data_flow"), + (backbone, roi, "feature_flow"), (roi, classification, "branch"), (roi, box, "branch"), (roi, mask, "branch"), + ]: + ensure_relation(source, target, relation_type, list(dict.fromkeys(source.get("evidence_ids", []) + target.get("evidence_ids", [])))) + + sam_overview = next((item for item in figures if re.search(r"segment anything model.*overview", str(item.get("caption") or ""), re.IGNORECASE)), None) + if sam_overview: + sam_caption = str(sam_overview.get("caption") or "") + sam_evidence = next((item["id"] for item in evidence if item.get("kind") == "caption" and item.get("page") == sam_overview.get("page") and str(item.get("text") or "").startswith(sam_caption[:60])), None) + sam_ids = [sam_evidence] if sam_evidence else evidence_ids + image = ensure_entity("inputs", "Image", "input", sam_ids) + prompt = ensure_entity("inputs", "Prompt", "input", sam_ids) + valid_mask = ensure_entity("outputs", "Valid Segmentation Mask", "output", sam_ids) + image_encoder = ensure_entity("modules", "Image Encoder", "module", find_evidence("image encoder")) + prompt_encoder = ensure_entity("modules", "Prompt Encoder", "module", find_evidence("prompt encoder")) + mask_decoder = ensure_entity("modules", "Mask Decoder", "module", find_evidence("mask decoder")) + for source, target, relation_type in [ + (image, image_encoder, "data_flow"), (prompt, prompt_encoder, "data_flow"), + (image_encoder, mask_decoder, "feature_flow"), (prompt_encoder, mask_decoder, "conditioning"), + (mask_decoder, valid_mask, "data_flow"), + ]: + ensure_relation(source, target, relation_type, list(dict.fromkeys(source.get("evidence_ids", []) + target.get("evidence_ids", [])))) + + def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} @@ -101,10 +325,149 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d spec["inputs"] = summary.get("inputs") if isinstance(summary.get("inputs"), list) else [] if not isinstance(spec.get("outputs"), list) or not spec.get("outputs"): spec["outputs"] = summary.get("outputs") if isinstance(summary.get("outputs"), list) else [] + for field in ("inputs", "outputs", "innovations"): + normalized_items = [] + for index, raw in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): + if not isinstance(raw, dict): + continue + item = dict(raw) + item.setdefault("id", _item_id(item, field, index)) + normalized_items.append(item) + spec[field] = normalized_items + _complete_from_overview_caption(spec, parsed) + modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] + encoder_modules = [item for item in modules if isinstance(item, dict) and "encoder" in _item_label(item).casefold() and "modality encoder" not in _item_label(item).casefold()] + if len(encoder_modules) >= 3: + group_id = "modality_encoders_group" + evidence_ids = list(dict.fromkeys(value for item in encoder_modules for value in item.get("evidence_ids", []))) + if not any(str(item.get("id")) == group_id for item in modules if isinstance(item, dict)): + modules.append({"id": group_id, "name": "Modality Encoders", "role": "group", "evidence_ids": evidence_ids}) + relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] + encoder_ids = {str(item.get("id")) for item in encoder_modules} + endpoints = modules + list(spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []) + list(spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []) + joint_items = [item for item in endpoints if isinstance(item, dict) and "joint" in _item_label(item).casefold() and ("embedding" in _item_label(item).casefold() or "representation" in _item_label(item).casefold())] + existing = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + for relation in list(relations): + if not isinstance(relation, dict) or str(relation.get("target")) not in encoder_ids: + continue + pair = (str(relation.get("source")), group_id) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "encoding", "label": "", "evidence_ids": list(relation.get("evidence_ids", []))}) + existing.add(pair) + modality_terms = ("image", "text", "audio", "depth", "thermal", "imu", "video", "modalit") + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + if not isinstance(item, dict) or not any(term in _item_label(item).casefold() for term in modality_terms): + continue + pair = (str(item.get("id")), group_id) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "encoding", "label": "", "evidence_ids": list(item.get("evidence_ids", []))}) + existing.add(pair) + for joint in joint_items: + if any(str(item.get("source")) in encoder_ids and str(item.get("target")) == str(joint.get("id")) for item in relations if isinstance(item, dict)): + pair = (group_id, str(joint.get("id"))) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "alignment", "label": "", "evidence_ids": evidence_ids}) + existing.add(pair) + emergent_items = [ + item for item in endpoints + if isinstance(item, dict) + and (("emergent" in _item_label(item).casefold() and "align" in _item_label(item).casefold()) or "binding" in _item_label(item).casefold()) + ] + if not emergent_items: + emergent_relation = next((item for item in relations if isinstance(item, dict) and "emergent" in str(item.get("label") or "").casefold() and "align" in str(item.get("label") or "").casefold()), None) + if emergent_relation: + emergent = {"id": "emergent_alignment_concept", "statement": "Emergent Alignment", "role": "innovation", "evidence_ids": list(emergent_relation.get("evidence_ids", []))} + spec.setdefault("innovations", []).append(emergent) + emergent_items.append(emergent) + for joint in joint_items: + for emergent in emergent_items: + pair = (str(joint.get("id")), str(emergent.get("id"))) + if pair not in existing: + relations.append({"source": pair[0], "target": pair[1], "type": "enables", "label": "emergent alignment", "evidence_ids": list(emergent.get("evidence_ids", []))}) + existing.add(pair) + spec["relations"] = relations + spec["modules"] = modules + modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] + relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] + existing_labels = [_normalized_label(_item_label(item)) for field in ("inputs", "modules", "outputs", "innovations") for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) if isinstance(item, dict)] + component_terms = ("token", "embedding", "encoder", "decoder", "head", "branch", "prompt", "proposal", "mask", "engine", "dataset") + for index, item in enumerate(spec.get("must_show", []) if isinstance(spec.get("must_show"), list) else []): + if not isinstance(item, dict): + continue + text = _item_label(item) + normalized = _normalized_label(text) + if not text or len(text) > 80 or not any(term in text.casefold() for term in component_terms): + continue + if any(normalized and (normalized in label or label in normalized) for label in existing_labels if label): + continue + module = {"id": _stable_id("required", text, index), "name": text, "role": "required_component", "evidence_ids": list(item.get("evidence_ids", []))} + modules.append(module) + existing_labels.append(normalized) + spec["modules"] = modules + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + transformer_target = next((item for item in modules if "transformer encoder" in _item_label(item).casefold()), None) + for item in modules: + label = _item_label(item).casefold() + normalized_component = _normalized_label(label) + if transformer_target and str(item.get("id")) != str(transformer_target.get("id")) and any(term in normalized_component for term in ("classtoken", "classificationtoken", "positionembedding")): + pair = (str(item.get("id")), str(transformer_target.get("id"))) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "conditioning", "label": "", "evidence_ids": list(item.get("evidence_ids", []))}) + existing_pairs.add(pair) + outgoing = {str(item.get("source")) for item in relations if isinstance(item, dict)} + incoming = {str(item.get("target")) for item in relations if isinstance(item, dict)} + def choose_input_target(label: str) -> dict[str, Any] | None: + preferences = [] + low = label.casefold() + if "prompt" in low: + preferences = ["prompt encoder", "encoder", "decoder"] + elif "image" in low or "video" in low: + preferences = ["patch", "image encoder", "backbone", "encoder"] + elif any(term in low for term in ("text", "audio", "depth", "thermal", "imu")): + preferences = [f"{next((term for term in ('text', 'audio', 'depth', 'thermal', 'imu') if term in low), '')} encoder", "modality encoders", "encoder"] + for preference in preferences: + match = next((module for module in modules if preference and preference in _item_label(module).casefold()), None) + if match: + return match + return modules[0] if modules else None + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + source = str(item.get("id")) + if not source: + continue + target = choose_input_target(_item_label(item)) + if target: + pair = (source, str(target.get("id"))) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "data_flow", "label": "", "evidence_ids": list(dict.fromkeys(list(item.get("evidence_ids", [])) + list(target.get("evidence_ids", []))))}) + existing_pairs.add(pair) + outgoing.add(source) + incoming.add(pair[1]) + module_sources = {str(item.get("source")) for item in relations if isinstance(item, dict)} + sink_modules = [item for item in modules if str(item.get("id")) not in module_sources] + for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []: + target_id = str(item.get("id")) + if not target_id: + continue + output_label = _item_label(item).casefold() + source = next((module for module in reversed(modules) if any(term in _item_label(module).casefold() for term in ("head", "decoder", "refine", "classifier", "predict"))), None) + if "mask" in output_label: + source = next((module for module in modules if "mask decoder" in _item_label(module).casefold() or "mask branch" in _item_label(module).casefold()), source) + elif "box" in output_label or "bounding" in output_label or "class" in output_label or "category" in output_label: + source = next((module for module in modules if "box branch" in _item_label(module).casefold() or "classification head" in _item_label(module).casefold()), source) + source = source or (sink_modules[-1] if sink_modules else modules[-1] if modules else None) + if source: + pair = (str(source.get("id")), target_id) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "data_flow", "label": "", "evidence_ids": list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(item.get("evidence_ids", []))))}) + existing_pairs.add(pair) + incoming.add(target_id) + spec["relations"] = relations spec.setdefault("training_flow", summary.get("training_flow") if isinstance(summary.get("training_flow"), list) else []) spec.setdefault("inference_flow", summary.get("inference_flow") if isinstance(summary.get("inference_flow"), list) else []) spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) - spec.setdefault("topology", _infer_topology(spec)) + allowed_topologies = {"linear", "branch", "feedback", "multimodal", "dense_multiframe", "unknown"} + if str(spec.get("topology") or "") not in allowed_topologies: + spec["topology"] = _infer_topology(spec) labels = [] terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} labels.extend(str(value).strip() for value in terminology.values() if str(value).strip()) @@ -161,6 +524,115 @@ def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: } +def merge_review_grounded_contract(primary: dict[str, Any], fallback: dict[str, Any]) -> dict[str, Any]: + primary_spec = primary.get("figure_specification") if isinstance(primary.get("figure_specification"), dict) else {} + fallback_spec = fallback.get("figure_specification") if isinstance(fallback.get("figure_specification"), dict) else {} + endpoint_remap: dict[str, str] = {} + for field in ("inputs", "modules", "outputs", "innovations", "must_show", "relations"): + combined = list(primary_spec.get(field, []) if isinstance(primary_spec.get(field), list) else []) + if field == "relations": + keys = {(str(item.get("source")), str(item.get("target")), str(item.get("type"))) for item in combined if isinstance(item, dict)} + for item in fallback_spec.get(field, []) if isinstance(fallback_spec.get(field), list) else []: + if not isinstance(item, dict): + continue + merged_relation = dict(item) + merged_relation["source"] = endpoint_remap.get(str(item.get("source")), str(item.get("source"))) + merged_relation["target"] = endpoint_remap.get(str(item.get("target")), str(item.get("target"))) + key = (str(merged_relation.get("source")), str(merged_relation.get("target")), str(merged_relation.get("type"))) + if key and key not in keys: + combined.append(merged_relation) + keys.add(key) + else: + by_id = {_item_id(item, field, index).casefold(): _item_id(item, field, index) for index, item in enumerate(combined) if isinstance(item, dict)} + by_label = {_item_label(item).casefold(): _item_id(item, field, index) for index, item in enumerate(combined) if isinstance(item, dict) and _item_label(item)} + for index, item in enumerate(fallback_spec.get(field, []) if isinstance(fallback_spec.get(field), list) else []): + if not isinstance(item, dict): + continue + raw_item_id = _item_id(item, field, index) + item_id = raw_item_id.casefold() + label = _item_label(item).casefold() + if item_id in by_id: + endpoint_remap[raw_item_id] = by_id[item_id] + elif label and label in by_label: + endpoint_remap[raw_item_id] = by_label[label] + else: + combined.append(item) + endpoint_remap[raw_item_id] = raw_item_id + by_id[item_id] = raw_item_id + if label: + by_label[label] = raw_item_id + primary_spec[field] = combined + terminology = dict(fallback_spec.get("terminology") if isinstance(fallback_spec.get("terminology"), dict) else {}) + terminology.update(primary_spec.get("terminology") if isinstance(primary_spec.get("terminology"), dict) else {}) + primary_spec["terminology"] = terminology + primary_spec["forbidden_inventions"] = list(dict.fromkeys( + list(primary_spec.get("forbidden_inventions", []) if isinstance(primary_spec.get("forbidden_inventions"), list) else []) + + list(fallback_spec.get("forbidden_inventions", []) if isinstance(fallback_spec.get("forbidden_inventions"), list) else []) + )) + primary["figure_specification"] = primary_spec + primary_design = primary.get("design_plan") if isinstance(primary.get("design_plan"), dict) else {} + fallback_order = fallback.get("design_plan", {}).get("reading_order", []) if isinstance(fallback.get("design_plan"), dict) else [] + primary_design["reading_order"] = list(dict.fromkeys(list(primary_design.get("reading_order", [])) + list(fallback_order))) + primary["design_plan"] = primary_design + return primary + + +def _paper_review_from_plan(plan: dict[str, Any], selected_domain: dict[str, Any]) -> dict[str, Any]: + summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} + spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + def fact(item: Any, item_id: str) -> dict[str, Any]: + if not isinstance(item, dict): + return {"id": item_id, "statement": str(item or "unknown"), "evidence_ids": [], "status": "unknown"} + return { + "id": str(item.get("id") or item_id), + "statement": str(item.get("statement") or item.get("text") or item.get("name") or "unknown"), + "visible_label": str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or ""), + "evidence_ids": list(item.get("evidence_ids", [])), + "status": str(item.get("status") or "required"), + "importance": str(item.get("importance") or "high"), + "confidence": float(item.get("confidence", 1.0) or 0.0), + "must_appear_in_figure": True, + "visual_role": str(item.get("role") or "module"), + } + return { + "summary": "Compact paper review derived from the fast VLM figure contract.", + "schema_version": "2.0-fast", + "paper_identity": {"title": summary.get("title"), "paper_type": summary.get("paper_type", "unknown")}, + "domain_profile": selected_domain.get("id"), + "research_questions": [fact(spec.get("research_problem") or summary.get("research_problem"), "research_problem")], + "central_claims": [fact(spec.get("central_claim") or summary.get("central_claim"), "central_claim")], + "inputs": [fact(item, f"input_{index}") for index, item in enumerate(spec.get("inputs", [])) if isinstance(item, dict)], + "outputs": [fact(item, f"output_{index}") for index, item in enumerate(spec.get("outputs", [])) if isinstance(item, dict)], + "research_objects": [], + "concepts": [], + "modules": [fact(item, f"module_{index}") for index, item in enumerate(spec.get("modules", [])) if isinstance(item, dict)], + "relations": [ + {**fact(item, f"relation_{index}"), "source_id": item.get("source"), "target_id": item.get("target"), "relation_type": item.get("type", "data_flow")} + for index, item in enumerate(spec.get("relations", [])) if isinstance(item, dict) + ], + "contributions": [], + "innovations": [fact(item, f"innovation_{index}") for index, item in enumerate(spec.get("innovations", [])) if isinstance(item, dict)], + "workflows": { + "training": [fact(item, f"training_{index}") for index, item in enumerate(spec.get("training_flow", []))], + "inference": [fact(item, f"inference_{index}") for index, item in enumerate(spec.get("inference_flow", []))], + }, + "experiments": {}, + "assumptions": [], + "limitations": [], + "results": [], + "terminology": [], + "forbidden_inventions": [{"id": f"forbidden_{index}", "statement": str(value), "evidence_ids": []} for index, value in enumerate(spec.get("forbidden_inventions", []))], + "unknowns": list(spec.get("uncertainties", [])), + } + + +def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: + signature = json.dumps({"version": 3, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + variant = hashlib.sha256(signature).hexdigest()[:16] + root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) + return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" + + def prepare_paper_figure_contract( paper: str | Path, out: str | Path, @@ -176,6 +648,7 @@ def prepare_paper_figure_contract( language: str | None = None, domain_profile: str = "auto", ocr_adapter: Callable | None = None, + fast_mode: bool = False, ) -> dict[str, Any]: started = time.monotonic() root = ensure_dir(out).resolve() @@ -225,36 +698,78 @@ def prepare_paper_figure_contract( selected_domain = detect_domain_profile(parsed, explicit=domain_profile) write_json(root / "domain_profile.json", selected_domain) - remaining = max(10, int(deadline_at - time.monotonic() - 15)) - effective_mode = planner_mode if remaining >= 30 else "heuristic" - paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=effective_mode, model=planner_model, timeout_seconds=min(75, remaining), retries=0) - review_prompt = review_metadata.pop("prompt") + if fast_mode: + remaining = max(10, int(deadline_at - time.monotonic() - 15)) + effective_mode = planner_mode if remaining >= 25 else "heuristic" + cache_path = _fast_cache_path(parsed, str(planner_model or ""), preferences) + if cache_path.exists() and effective_mode == "vlm": + plan = read_json(cache_path) + planner_metadata = {"requested_mode": "vlm", "mode": "vlm", "model": planner_model, "warning": None, "prompt": "", "cached": True} + else: + plan, planner_metadata = plan_paper_image( + parsed, + preferences, + mode=effective_mode, + model=planner_model, + reference_images=archived_positive + archived_negative, + paper_review=None, + timeout_seconds=min(45, remaining), + retries=1, + evidence_max_chars=36000, + ) + if planner_metadata.get("mode") == "vlm": + paper_review = _paper_review_from_plan(plan, selected_domain) + review_metadata = {"summary": "Paper review derived from the successful fast VLM plan.", "requested_mode": planner_mode, "mode": "derived_from_vlm_plan", "model": planner_metadata.get("model"), "warning": None, "prompt": ""} + else: + remaining = max(10, int(deadline_at - time.monotonic() - 15)) + paper_review, review_metadata = build_paper_review( + parsed, + selected_domain, + mode=effective_mode, + model=planner_model, + timeout_seconds=min(35, remaining), + retries=1, + evidence_max_chars=36000, + ) + if review_metadata.get("mode") == "vlm": + plan = build_review_grounded_plan(parsed, preferences, paper_review) + planner_metadata = {**planner_metadata, "mode": "review_grounded_vlm", "model": review_metadata.get("model"), "warning": planner_metadata.get("warning")} + else: + remaining = max(10, int(deadline_at - time.monotonic() - 15)) + effective_mode = planner_mode if remaining >= 30 else "heuristic" + paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=effective_mode, model=planner_model, timeout_seconds=min(35, remaining), retries=1) + remaining = max(10, int(deadline_at - time.monotonic() - 15)) + planning_mode = effective_mode if remaining >= 25 else "heuristic" + plan, planner_metadata = plan_paper_image( + parsed, + preferences, + mode=planning_mode, + model=planner_model, + reference_images=archived_positive + archived_negative, + paper_review=paper_review, + timeout_seconds=min(35, remaining), + retries=1, + ) + fallback_plan = build_review_grounded_plan(parsed, preferences, paper_review) + if review_metadata.get("mode") == "vlm": + plan = merge_review_grounded_contract(plan, fallback_plan) + review_prompt = review_metadata.pop("prompt", "") write_text(root / "prompts" / "paper_review_prompt.txt", review_prompt) write_json(root / "paper_review_metadata.json", review_metadata) write_json(root / "paper_review.json", paper_review) coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=False) write_json(root / "review_coverage_report.json", coverage) - - remaining = max(10, int(deadline_at - time.monotonic() - 15)) - planning_mode = effective_mode if remaining >= 25 else "heuristic" - plan, planner_metadata = plan_paper_image( - parsed, - preferences, - mode=planning_mode, - model=planner_model, - reference_images=archived_positive + archived_negative, - paper_review=paper_review, - timeout_seconds=min(70, remaining), - retries=0, - ) planning_prompt = planner_metadata.pop("prompt") write_text(root / "prompts" / "planning_prompt.txt", planning_prompt) write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) + expand_plan_evidence(plan, paper_review, parsed) normalize_figure_contract(plan, parsed) for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): write_json(root / f"{name}.json", plan[name]) planning_validation = validate_plan_grounding(plan, parsed) write_json(root / "planning_validation_report.json", planning_validation) + if fast_mode and planning_validation.get("ok") and planner_metadata.get("mode") in {"vlm", "review_grounded_vlm"}: + write_json(cache_path, plan) overlay = build_overlay_spec(plan) write_json(root / "overlay_spec.json", overlay) final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1) @@ -268,13 +783,11 @@ def prepare_paper_figure_contract( elapsed = round(time.monotonic() - started, 3) deadline_reached = time.monotonic() >= deadline_at - production_ready = bool( - review_metadata.get("mode") == "vlm" - and planner_metadata.get("mode") == "vlm" - and planning_validation.get("ok") - and not deadline_reached - ) - status = "complete" if production_ready else "completed_with_warnings" + production_ready = bool((review_metadata.get("mode") == "vlm" or planner_metadata.get("mode") == "vlm") and planning_validation.get("ok") and not deadline_reached) + has_warning = bool(planner_metadata.get("warning") or review_metadata.get("warning") or parsed["extraction_report"].get("status") == "warning") + status = "complete" if production_ready and not has_warning else "completed_with_warnings" + contract_source = "cache" if planner_metadata.get("cached") else "vlm" if planner_metadata.get("mode") == "vlm" else "vlm_review_deterministic_compile" if planner_metadata.get("mode") == "review_grounded_vlm" else "deterministic_evidence_rules" + extraction_seconds = float(parsed["extraction_report"].get("elapsed_seconds") or 0.0) result = { "summary": "Fast paper-to-framework contract preparation completed.", "ok": bool(planning_validation.get("ok")), @@ -289,11 +802,14 @@ def prepare_paper_figure_contract( "elapsed_seconds": elapsed, "planner_mode": planner_metadata.get("mode"), "paper_review_mode": review_metadata.get("mode"), + "contract_source": contract_source, + "cache_hit": bool(planner_metadata.get("cached")), "planner_warning": planner_metadata.get("warning"), "review_warning": review_metadata.get("warning"), "topology": plan["figure_specification"].get("topology"), "module_count": len(plan["figure_specification"].get("modules", [])), "relation_count": len(plan["figure_specification"].get("relations", [])), + "stage_timings": {"document_extraction_seconds": extraction_seconds, "semantic_compilation_seconds": round(max(0.0, elapsed - extraction_seconds), 3), "total_seconds": elapsed}, "uncertainties": plan["figure_specification"].get("uncertainties", []), "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], } @@ -315,6 +831,9 @@ def prepare_paper_figure_contract( def run_fast_framework_prompt(**kwargs: Any) -> dict[str, Any]: + if not kwargs.get("planner_model"): + kwargs["planner_model"] = os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" + kwargs["fast_mode"] = True prepared = prepare_paper_figure_contract(**kwargs) internal = {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"} return {key: value for key, value in prepared.items() if key not in internal} diff --git a/rfs/paper_to_image/review.py b/rfs/paper_to_image/review.py index 3200ffc..225d56a 100644 --- a/rfs/paper_to_image/review.py +++ b/rfs/paper_to_image/review.py @@ -68,7 +68,7 @@ def detect_domain_profile(parsed: dict, explicit: str = "auto") -> dict: return {"summary": profile["summary"], "id": selected, "selection": "automatic", "scores": scores, **profile} -def _review_prompt(parsed: dict, domain_profile: dict) -> str: +def _review_prompt(parsed: dict, domain_profile: dict, evidence_max_chars: int = 65000) -> str: return f""" # Summary @@ -143,7 +143,7 @@ def _review_prompt(parsed: dict, domain_profile: dict) -> str: - Short visible labels should normally be at most 32 characters. Paper evidence: -{evidence_excerpt(parsed, max_chars=65000)} +{evidence_excerpt(parsed, max_chars=evidence_max_chars)} """.strip() @@ -235,7 +235,9 @@ def normalize_review(raw: dict, domain_profile: dict) -> dict: def _expand_evidence(review: dict, parsed: dict) -> None: evidence = {item["id"]: item for item in parsed.get("evidence", [])} + aliases = {item.get("legacy_id"): item["id"] for item in parsed.get("evidence", []) if item.get("legacy_id")} def expand(item: dict) -> None: + item["evidence_ids"] = list(dict.fromkeys(aliases.get(str(value), str(value)) for value in item.get("evidence_ids", []))) item["evidence_refs"] = [{"evidence_id": value, "page": evidence[value].get("page"), "section": evidence[value].get("section_hint"), "quote": evidence[value].get("text", "")[:600]} for value in item.get("evidence_ids", []) if value in evidence] for field in ["research_questions", "central_claims", "inputs", "outputs", "research_objects", "concepts", "modules", "relations", "contributions", "innovations", "assumptions", "limitations", "results", "terminology", "forbidden_inventions", "unknowns"]: for item in review.get(field, []): @@ -326,8 +328,8 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s } -def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1) -> tuple[dict, dict]: - prompt = _review_prompt(parsed, domain_profile) +def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1, evidence_max_chars: int = 65000) -> tuple[dict, dict]: + prompt = _review_prompt(parsed, domain_profile, evidence_max_chars=evidence_max_chars) metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_REVIEW_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", explicit_model=model) diff --git a/skills/research-figure-studio/SKILL.md b/skills/research-figure-studio/SKILL.md index 35cee63..96df4bc 100644 --- a/skills/research-figure-studio/SKILL.md +++ b/skills/research-figure-studio/SKILL.md @@ -86,8 +86,11 @@ Use the two independent suites to locate failures before changing the workflow: ```powershell rfs benchmark list --root benchmarks --json rfs benchmark fetch --case --json +rfs benchmark fast --case --out --deadline 180 --json rfs benchmark run --case --out --json rfs benchmark score --case --run --json ``` `paper-to-image` scientific and terminology errors are hard failures. `image-to-ppt` must reject full-slide reference images even when raster similarity is high. + +For prompt-only work, run `rfs fast-framework-prompt --paper --out --deadline 180 --json`. Inspect `extraction_report.json` first, then `planning_validation_report.json`; treat `engineering_only: true` as a non-production result even when deterministic topology coverage is high. diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 85ded54..08b9810 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -159,6 +159,13 @@ def test_fast_framework_cli_defaults(self): self.assertEqual(args.deadline, 180) self.assertEqual(args.planner_mode, "vlm") + def test_benchmark_fast_cli_defaults(self): + parser = build_parser() + args = parser.parse_args(["benchmark", "fast", "--case", "benchmarks/case", "--out", "output/fast"]) + self.assertEqual(args.benchmark_action, "fast") + self.assertEqual(args.deadline, 180) + self.assertEqual(args.planner_mode, "vlm") + def test_domain_profiles_detect_method_and_survey(self): method = {"evidence": [{"text": "A neural model is optimized with a training loss and used during inference."}]} survey = {"evidence": [{"text": "This systematic survey presents a taxonomy and review of the landscape."}]} diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 8825e4b..ffec8cf 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -8,7 +8,9 @@ from pptx.util import Inches from rfs.evaluation.benchmarking import ( + _score_planning_contract, list_benchmark_cases, + run_fast_benchmark_case, score_benchmark_case, validate_benchmark_case, ) @@ -18,6 +20,33 @@ class BenchmarkingTests(unittest.TestCase): + def test_planning_contract_treats_relation_labels_as_intermediate_artifacts(self): + expected = { + "entities": [ + {"id": "generator", "label": "Generate"}, + {"id": "draft", "label": "Initial Output"}, + {"id": "critic", "label": "Feedback"}, + ], + "relations": [ + {"source": "generator", "target": "draft"}, + {"source": "draft", "target": "critic"}, + ], + } + specification = { + "modules": [ + {"id": "init", "name": "Generate"}, + {"id": "feedback", "name": "Feedback"}, + ], + "relations": [ + {"source": "init", "target": "feedback", "label": "Initial Output"}, + ], + } + + result = _score_planning_contract(expected, specification) + + self.assertEqual(result["entity_recall"], 1.0) + self.assertEqual(result["relation_recall"], 1.0) + def test_bundled_cases_validate_and_list(self): p2i = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" i2p = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" @@ -34,6 +63,15 @@ def test_real_paper_case_is_valid_with_source_metadata(self): self.assertTrue(validation["ok"]) self.assertTrue((case_dir / "source.json").exists()) + def test_fast_benchmark_command_runs_without_image_generation(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + result = run_fast_benchmark_case(case_dir, tmp, planner_mode="heuristic", ocr_engine="off") + + self.assertTrue(result["ok"]) + self.assertTrue((Path(tmp) / "fast_benchmark_result.json").exists()) + self.assertFalse((Path(tmp) / "selected_image.png").exists()) + def test_paper_to_image_score_uses_hard_scientific_gates(self): case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index d7fe9c9..804bcbe 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -35,6 +35,17 @@ def draw(page): self.assertIn("Introduction", parsed["headings"]) self.assertTrue(all(item.get("block_id") for item in parsed["evidence"])) + def test_multiline_figure_caption_is_combined_into_priority_evidence(self): + def draw(page): + page.insert_textbox((50, 80, 550, 150), "Figure 1: Three interconnected components: a promptable task,\na model, and a data engine for collecting a large dataset.", fontsize=10) + page.insert_textbox((50, 180, 550, 250), "1 Introduction\nA sufficiently long introduction paragraph for quality validation.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + + caption = parsed["document_index"]["figures"][0]["caption"] + self.assertIn("data engine", caption) + self.assertTrue(any(item["kind"] == "caption" and "data engine" in item["text"] for item in parsed["evidence"])) + def test_two_column_blocks_are_sorted_left_then_right(self): def draw(page): page.insert_textbox((330, 100, 560, 180), "RIGHT COLUMN SECOND with enough words to form a complete paragraph.", fontsize=10) @@ -86,6 +97,15 @@ def test_inspection_reuses_matching_document_cache(self): self.assertTrue(first["document_model"]) self.assertEqual(second["status"], "cached") + def test_evidence_ids_are_stable_across_repeated_parses(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "1 Method\nA stable block of scientific evidence for repeated parsing.", fontsize=10)) + + first = parse_paper(path) + second = parse_paper(path) + + self.assertEqual([item["id"] for item in first["evidence"]], [item["id"] for item in second["evidence"]]) + self.assertTrue(all(item["id"].startswith("E_P") for item in first["evidence"])) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 41da2cb..8e86c0b 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -1,7 +1,7 @@ import unittest from rfs.paper_to_image.planner import validate_plan_grounding -from rfs.paper_to_image.preparation import build_overlay_spec, normalize_figure_contract +from rfs.paper_to_image.preparation import build_overlay_spec, expand_plan_evidence, normalize_figure_contract class PaperContractTests(unittest.TestCase): @@ -80,6 +80,94 @@ def test_low_confidence_evidence_is_reported_as_uncertain(self): self.assertIn("ocr_module relies only on low-confidence OCR evidence", spec["uncertainties"]) + def test_review_fact_ids_expand_to_page_evidence_and_encoder_group(self): + parsed = {"evidence": [{"id": "E0001", "confidence": 1.0}]} + review = {"modules": [{"id": "review_encoders", "evidence_ids": ["E0001"]}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [ + {"id": "image_encoder", "name": "Image Encoder", "evidence_ids": ["review_encoders"]}, + {"id": "text_encoder", "name": "Text Encoder", "evidence_ids": ["review_encoders"]}, + {"id": "audio_encoder", "name": "Audio Encoder", "evidence_ids": ["review_encoders"]}, + {"id": "joint", "name": "Joint Embedding Space", "evidence_ids": ["E0001"]}, + ], + "inputs": [{"id": "image", "name": "Image", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "image", "target": "image_encoder", "evidence_ids": ["E0001"]}, + {"source": "image_encoder", "target": "joint", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + expand_plan_evidence(plan, review, parsed) + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(plan["figure_specification"]["modules"][0]["evidence_ids"], ["E0001"]) + self.assertTrue(any(item.get("id") == "modality_encoders_group" for item in spec["modules"])) + self.assertTrue(any(item.get("source") == "image" and item.get("target") == "modality_encoders_group" for item in spec["relations"])) + + def test_contract_completion_adds_required_conditioning_and_output_boundary(self): + parsed = {"evidence": [{"id": "E0001", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [ + {"id": "patches", "name": "Image Patches", "evidence_ids": ["E0001"]}, + {"id": "encoder", "name": "Transformer Encoder", "evidence_ids": ["E0001"]}, + {"id": "head", "name": "MLP Head", "evidence_ids": ["E0001"]}, + ], + "inputs": [{"id": "image", "name": "Input Image", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "prediction", "name": "Class Prediction", "evidence_ids": ["E0001"]}], + "relations": [ + {"source": "patches", "target": "encoder", "evidence_ids": ["E0001"]}, + {"source": "encoder", "target": "head", "evidence_ids": ["E0001"]}, + ], + "must_show": [{"text": "Class Token", "evidence_ids": ["E0001"]}], + "innovations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + class_token = next(item for item in spec["modules"] if item["name"] == "Class Token") + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + self.assertIn(("image", "patches"), pairs) + self.assertIn((class_token["id"], "encoder"), pairs) + self.assertIn(("head", "prediction"), pairs) + + def test_overview_caption_and_stage_list_complete_missing_panels(self): + caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." + parsed = { + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}, + {"id": "E0002", "page": 2, "kind": "paragraph", "text": "Our data engine has three stages:", "confidence": 1.0}, + {"id": "E0003", "page": 2, "kind": "paragraph", "text": "assisted-manual, semi-automatic, and fully automatic.", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [], "inputs": [], "outputs": [], "innovations": [], "relations": [], "must_show": [], "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = {_item["name"] for _item in spec["modules"]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn("Promptable segmentation task", labels) + self.assertIn("Segmentation model (SAM)", labels) + self.assertIn("Data engine", labels) + self.assertIn("Assisted-Manual", labels) + self.assertTrue(any(item.get("name") == "SA-1B" for item in spec["outputs"])) + self.assertTrue(any("stage_assisted_manual" in source and "stage_semi_automatic" in target for source, target in pairs)) + if __name__ == "__main__": unittest.main() From f6892ab87a4df4e81a7cc3d933a64b70516020fa Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 01:18:45 +0800 Subject: [PATCH 21/74] Generalize and harden fast paper understanding --- .../cases/106_detr_set_prediction/case.json | 15 + .../expected_semantics.json | 28 ++ .../cases/106_detr_set_prediction/source.json | 12 + .../cases/107_clip_contrastive/case.json | 15 + .../expected_semantics.json | 28 ++ .../cases/107_clip_contrastive/source.json | 12 + .../cases/108_nerf_volume_rendering/case.json | 15 + .../expected_semantics.json | 28 ++ .../108_nerf_volume_rendering/source.json | 12 + pyproject.toml | 1 + rfs/cli.py | 39 ++- rfs/evaluation/__init__.py | 3 +- rfs/evaluation/benchmarking.py | 120 +++++++ rfs/paper_to_image/analyzer.py | 180 ++++++++-- rfs/paper_to_image/contract_completion.py | 312 ++++++++++++++++++ rfs/paper_to_image/critics.py | 4 +- rfs/paper_to_image/document_cache.py | 76 +++++ rfs/paper_to_image/inspection.py | 21 +- rfs/paper_to_image/planner.py | 192 ++++++++++- rfs/paper_to_image/preparation.py | 174 ++++------ rfs/paper_to_image/review.py | 4 +- rfs/reference_text_extractor.py | 35 +- rfs/vlm_client.py | 63 +++- tests/integration/test_paper_to_image.py | 15 +- tests/unit/test_benchmarking.py | 17 + tests/unit/test_paper_analyzer.py | 48 ++- tests/unit/test_paper_contract.py | 72 ++++ tests/unit/test_vlm_client.py | 34 ++ 28 files changed, 1415 insertions(+), 160 deletions(-) create mode 100644 benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json create mode 100644 benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json create mode 100644 benchmarks/paper-to-image/cases/107_clip_contrastive/case.json create mode 100644 benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/107_clip_contrastive/source.json create mode 100644 benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json create mode 100644 benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json create mode 100644 rfs/paper_to_image/contract_completion.py create mode 100644 rfs/paper_to_image/document_cache.py create mode 100644 tests/unit/test_vlm_client.py diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json new file mode 100644 index 0000000..94860f8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ECCV 2020 DETR case for learned queries, encoder-decoder flow, branched outputs, and a training-only matching objective.", + "case_id": "106_detr_set_prediction", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json new file mode 100644 index 0000000..56d332d --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for DETR's end-to-end set prediction architecture.", + "entities": [ + {"id": "input_image", "label": "Input Image", "aliases": ["Image"], "role": "input", "required": true}, + {"id": "cnn_backbone", "label": "CNN Backbone", "aliases": ["Convolutional Backbone", "Backbone"], "role": "module", "required": true}, + {"id": "transformer_encoder", "label": "Transformer Encoder", "aliases": ["Encoder"], "role": "module", "required": true}, + {"id": "object_queries", "label": "Object Queries", "aliases": ["Learned Object Queries"], "role": "input", "required": true}, + {"id": "transformer_decoder", "label": "Transformer Decoder", "aliases": ["Decoder"], "role": "module", "required": true}, + {"id": "prediction_ffn", "label": "Prediction Feed-Forward Network", "aliases": ["Prediction FFN", "Feed Forward Network", "FFN"], "role": "module", "required": true}, + {"id": "class_predictions", "label": "Class Predictions", "aliases": ["Class Labels", "Classes"], "role": "output", "required": true}, + {"id": "box_predictions", "label": "Bounding Box Predictions", "aliases": ["Bounding Boxes", "Box Predictions"], "role": "output", "required": true}, + {"id": "bipartite_matching", "label": "Bipartite Matching", "aliases": ["Hungarian Matching", "Set Prediction Loss"], "role": "training", "required": true} + ], + "relations": [ + {"source": "input_image", "target": "cnn_backbone", "type": "data_flow", "required": true}, + {"source": "cnn_backbone", "target": "transformer_encoder", "type": "data_flow", "required": true}, + {"source": "transformer_encoder", "target": "transformer_decoder", "type": "data_flow", "required": true}, + {"source": "object_queries", "target": "transformer_decoder", "type": "conditioning", "required": true}, + {"source": "transformer_decoder", "target": "prediction_ffn", "type": "data_flow", "required": true}, + {"source": "prediction_ffn", "target": "class_predictions", "type": "branch", "required": true}, + {"source": "prediction_ffn", "target": "box_predictions", "type": "branch", "required": true}, + {"source": "class_predictions", "target": "bipartite_matching", "type": "training_objective", "required": true}, + {"source": "box_predictions", "target": "bipartite_matching", "type": "training_objective", "required": true} + ], + "forbidden_labels": ["Region Proposal Network", "RoIAlign", "Non-Maximum Suppression", "Anchor Boxes", "Mask Branch"], + "expected_reading_order": ["input_image", "cnn_backbone", "transformer_encoder", "object_queries", "transformer_decoder", "prediction_ffn", "class_predictions", "box_predictions"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json b/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json new file mode 100644 index 0000000..e800719 --- /dev/null +++ b/benchmarks/paper-to-image/cases/106_detr_set_prediction/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "End-to-End Object Detection with Transformers", + "venue": "ECCV", + "year": 2020, + "arxiv_id": "2005.12872", + "official_url": "https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/832_ECCV_2020_paper.php", + "paper_urls": ["https://arxiv.org/pdf/2005.12872", "https://export.arxiv.org/pdf/2005.12872"], + "target_figure": "Figure 2", + "verified_caption": "DETR uses a conventional CNN backbone, a transformer encoder-decoder, a fixed set of learned object queries, and parallel class and box predictions; bipartite matching supplies the set prediction training objective.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json new file mode 100644 index 0000000..30a56df --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ICML 2021 CLIP case requiring explicit separation of contrastive training and zero-shot inference.", + "case_id": "107_clip_contrastive", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "multimodal", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json new file mode 100644 index 0000000..d1f43c8 --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for CLIP contrastive pre-training and zero-shot inference.", + "entities": [ + {"id": "training_images", "label": "Images", "aliases": ["Image Batch", "Training Images"], "role": "input", "required": true}, + {"id": "training_text", "label": "Text", "aliases": ["Captions", "Text Batch", "Training Text"], "role": "input", "required": true}, + {"id": "image_encoder", "label": "Image Encoder", "role": "module", "required": true}, + {"id": "text_encoder", "label": "Text Encoder", "role": "module", "required": true}, + {"id": "image_embeddings", "label": "Image Embeddings", "aliases": ["Image Features"], "role": "module", "required": true}, + {"id": "text_embeddings", "label": "Text Embeddings", "aliases": ["Text Features"], "role": "module", "required": true}, + {"id": "contrastive_objective", "label": "Contrastive Learning", "aliases": ["Contrastive Pre-training", "Contrastive Objective", "Image-Text Contrastive Loss", "Correct Pairings"], "role": "training", "required": true}, + {"id": "class_descriptions", "label": "Class Descriptions", "aliases": ["Class Names", "Text Prompts"], "role": "input", "required": true}, + {"id": "zero_shot_classifier", "label": "Zero-Shot Classifier", "aliases": ["Zero-shot Prediction", "Zero-shot Linear Classifier"], "role": "output", "required": true} + ], + "relations": [ + {"source": "training_images", "target": "image_encoder", "type": "encoding", "required": true}, + {"source": "training_text", "target": "text_encoder", "type": "encoding", "required": true}, + {"source": "image_encoder", "target": "image_embeddings", "type": "data_flow", "required": true}, + {"source": "text_encoder", "target": "text_embeddings", "type": "data_flow", "required": true}, + {"source": "image_embeddings", "target": "contrastive_objective", "type": "alignment", "required": true}, + {"source": "text_embeddings", "target": "contrastive_objective", "type": "alignment", "required": true}, + {"source": "class_descriptions", "target": "text_encoder", "type": "encoding", "required": true}, + {"source": "text_embeddings", "target": "zero_shot_classifier", "type": "classification", "required": true}, + {"source": "image_embeddings", "target": "zero_shot_classifier", "type": "classification", "required": true} + ], + "forbidden_labels": ["Bounding Box Regression", "Mask Decoder", "Autoregressive Decoder", "Image Generator", "Region Proposals"], + "expected_reading_order": ["training_images", "image_encoder", "image_embeddings", "contrastive_objective", "training_text", "text_encoder", "text_embeddings", "zero_shot_classifier"], + "topology_requirements": {"primary_flow": "multimodal_alignment", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": true} +} diff --git a/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json b/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json new file mode 100644 index 0000000..10de429 --- /dev/null +++ b/benchmarks/paper-to-image/cases/107_clip_contrastive/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Learning Transferable Visual Models From Natural Language Supervision", + "venue": "ICML", + "year": 2021, + "arxiv_id": "2103.00020", + "official_url": "https://proceedings.mlr.press/v139/radford21a.html", + "paper_urls": ["https://arxiv.org/pdf/2103.00020", "https://export.arxiv.org/pdf/2103.00020"], + "target_figure": "Figure 1", + "verified_caption": "CLIP jointly trains an image encoder and a text encoder to predict correct image-text pairings, then uses text embeddings of class descriptions as a zero-shot classifier at inference time.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json new file mode 100644 index 0000000..e344436 --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/case.json @@ -0,0 +1,15 @@ +{ + "summary": "ECCV 2020 NeRF case for coordinate conditioning, branched field outputs, and differentiable rendering.", + "case_id": "108_nerf_volume_rendering", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json new file mode 100644 index 0000000..c7cb0ca --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/expected_semantics.json @@ -0,0 +1,28 @@ +{ + "summary": "Human-authored semantic contract for NeRF's neural scene representation and differentiable rendering pipeline.", + "entities": [ + {"id": "camera_ray", "label": "Camera Ray", "aliases": ["Camera Rays", "Ray"], "role": "input", "required": true}, + {"id": "sampled_points", "label": "Sampled 3D Points", "aliases": ["5D Coordinates", "Spatial Locations"], "role": "module", "required": true}, + {"id": "viewing_direction", "label": "Viewing Direction", "aliases": ["View Direction"], "role": "input", "required": true}, + {"id": "positional_encoding", "label": "Positional Encoding", "aliases": ["Fourier Features"], "role": "module", "required": true}, + {"id": "nerf_mlp", "label": "Neural Radiance Field", "aliases": ["NeRF", "MLP", "Neural Network"], "role": "module", "required": true}, + {"id": "volume_density", "label": "Volume Density", "aliases": ["Density", "Sigma"], "role": "module", "required": true}, + {"id": "color", "label": "Color", "aliases": ["RGB Color", "Radiance"], "role": "module", "required": true}, + {"id": "volume_rendering", "label": "Volume Rendering", "aliases": ["Differentiable Volume Rendering"], "role": "module", "required": true}, + {"id": "rendered_image", "label": "Rendered Image", "aliases": ["Synthesized Image", "Synthesized View", "Novel View"], "role": "output", "required": true} + ], + "relations": [ + {"source": "camera_ray", "target": "sampled_points", "type": "sampling", "required": true}, + {"source": "sampled_points", "target": "positional_encoding", "type": "encoding", "required": true}, + {"source": "positional_encoding", "target": "nerf_mlp", "type": "data_flow", "required": true}, + {"source": "viewing_direction", "target": "nerf_mlp", "type": "conditioning", "required": true}, + {"source": "nerf_mlp", "target": "volume_density", "type": "prediction", "required": true}, + {"source": "nerf_mlp", "target": "color", "type": "prediction", "required": true}, + {"source": "volume_density", "target": "volume_rendering", "type": "rendering_input", "required": true}, + {"source": "color", "target": "volume_rendering", "type": "rendering_input", "required": true}, + {"source": "volume_rendering", "target": "rendered_image", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Voxel Grid", "Mesh Reconstruction", "Convolutional Decoder", "GAN Discriminator", "Depth Sensor"], + "expected_reading_order": ["camera_ray", "sampled_points", "positional_encoding", "nerf_mlp", "volume_density", "color", "volume_rendering", "rendered_image"], + "topology_requirements": {"primary_flow": "left_to_right", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json new file mode 100644 index 0000000..e3cb3ab --- /dev/null +++ b/benchmarks/paper-to-image/cases/108_nerf_volume_rendering/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis", + "venue": "ECCV", + "year": 2020, + "arxiv_id": "2003.08934", + "official_url": "https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/1473_ECCV_2020_paper.php", + "paper_urls": ["https://arxiv.org/pdf/2003.08934", "https://export.arxiv.org/pdf/2003.08934"], + "target_figure": "Figure 2", + "verified_caption": "NeRF samples 5D coordinates along camera rays, maps position and viewing direction through a neural radiance field to color and volume density, and applies differentiable volume rendering to synthesize an image.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/pyproject.toml b/pyproject.toml index ea2fe29..c64c4e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ [project.optional-dependencies] ocr = [ + "rapidocr-onnxruntime", "paddleocr", "easyocr" ] diff --git a/rfs/cli.py b/rfs/cli.py index 905b6f7..42eaa84 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -11,7 +11,7 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable -from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, score_benchmark_case, validate_benchmark_case +from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro @@ -63,10 +63,22 @@ def _doctor() -> dict: "RFS_IMAGE_EDIT_URL": {"present": env_present("RFS_IMAGE_EDIT_URL"), "value": os.getenv("RFS_IMAGE_EDIT_URL") if env_present("RFS_IMAGE_EDIT_URL") else None}, "MODEL_VLM": {"present": env_present("MODEL_VLM"), "value": os.getenv("MODEL_VLM") if env_present("MODEL_VLM") else None}, } + easyocr_model_dir = Path(os.getenv("EASYOCR_MODULE_PATH", "").strip() or (Path.home() / ".EasyOCR" / "model")) + easyocr_files = {path.name: path.stat().st_size for path in easyocr_model_dir.glob("*.pth")} if easyocr_model_dir.exists() else {} + easyocr_models = { + "model_dir": str(easyocr_model_dir), + "detector_ready": "craft_mlt_25k.pth" in easyocr_files, + "en_ready": "english_g2.pth" in easyocr_files, + "ch_ready": "zh_sim_g2.pth" in easyocr_files, + "en_ch_ready": "craft_mlt_25k.pth" in easyocr_files and "zh_sim_g2.pth" in easyocr_files, + "allow_download": str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"}, + "files": easyocr_files, + } optional_pdf = { "pdftotext": {"available": bool(shutil.which("pdftotext")), "path": shutil.which("pdftotext")}, "pdftoppm": {"available": bool(shutil.which("pdftoppm")), "path": shutil.which("pdftoppm")}, - "easyocr": {"available": importlib.util.find_spec("easyocr") is not None}, + "easyocr": {"available": importlib.util.find_spec("easyocr") is not None, "models": easyocr_models}, + "rapidocr": {"available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, "runtime": "onnxruntime", "thread_policy": "1 intra-op / 1 inter-op"}, "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, "fast_contract_cache": {"available": True, "path": str(Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")))}, } @@ -109,7 +121,7 @@ def build_parser() -> argparse.ArgumentParser: inspect_pdf.add_argument("--paper", required=True, help="Paper PDF or supported document path.") inspect_pdf.add_argument("--out", required=True, help="Output directory for the document model and extraction report.") inspect_pdf.add_argument("--deadline", type=int, default=180, help="Soft processing deadline in seconds. Default: 180.") - inspect_pdf.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "off"], default="auto") + inspect_pdf.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") @@ -123,7 +135,7 @@ def build_parser() -> argparse.ArgumentParser: fast_prompt.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") fast_prompt.add_argument("--aspect-ratio", default="16:9") fast_prompt.add_argument("--language", default="English") - fast_prompt.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "off"], default="auto") + fast_prompt.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") fast_prompt.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") fast_prompt.add_argument("--json", action="store_true", help="Emit JSON.") @@ -151,7 +163,7 @@ def build_parser() -> argparse.ArgumentParser: make.add_argument("--critic-model", help="Optional VLM model for asset review and final critic. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") make.add_argument("--critic-iterations", type=int, default=0, help="VLM layout correction iterations, clamped to 0-3. Default: 0.") make.add_argument("--text-extractor-mode", choices=["heuristic", "ocr"], default="ocr", help="Editable text layer source. Default ocr uses local OCR when available and falls back to heuristic.") - make.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle.") + make.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle.") make.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") make.add_argument("--presentations-qa", action="store_true", help="Run optional Presentations plugin import/render/layout QA after export. This never mutates the PPTX.") make.add_argument("--presentations-workspace", help="Optional workspace for Presentations QA scratch artifacts.") @@ -178,7 +190,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") - paper_image.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") + paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") paper_image.add_argument("--json", action="store_true", help="Emit JSON.") @@ -209,7 +221,7 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") paper_editable.add_argument("--allow-engineering-preview", action="store_true", help="Allow offline/non-production image previews to exercise the editable pipeline. Never enabled by default.") paper_editable.add_argument("--no-export-preview", action="store_true") - paper_editable.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "vlm", "off"], default="auto") + paper_editable.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto") paper_editable.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") paper_editable.add_argument("--json", action="store_true", help="Emit JSON.") @@ -234,7 +246,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") rebuild.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") rebuild.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") - rebuild.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") + rebuild.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") rebuild.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") rebuild.add_argument("--json", action="store_true", help="Emit JSON.") @@ -265,7 +277,7 @@ def build_parser() -> argparse.ArgumentParser: rebuild_pro.add_argument("--repair-rounds", type=int, default=2, help="Preview repair rounds to record/run. V1 records conservative no-mutation repair reports by default.") rebuild_pro.add_argument("--repair-mode", choices=["report", "vlm"], default="report", help="Professional repair mode. report records rounds without mutation; vlm applies controlled DSL patches.") rebuild_pro.add_argument("--benchmark-out", help="Optional specialized rebuild output directory to compare against in professional_gap_report.json.") - rebuild_pro.add_argument("--ocr-engine", choices=["paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") + rebuild_pro.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") rebuild_pro.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") rebuild_pro.add_argument("--json", action="store_true", help="Emit JSON.") @@ -286,17 +298,18 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") - benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "run", "score"]) + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "run", "score"]) benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") + benchmark.add_argument("--case-id", action="append", default=[], help="Optional case id filter for fast-suite. Repeat to select multiple cases.") benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") - benchmark.add_argument("--ocr-engine", choices=["auto", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for benchmark fast.") + benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for benchmark fast.") benchmark.add_argument("--json", action="store_true", help="Emit JSON.") validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") @@ -554,6 +567,10 @@ def main(argv: list[str] | None = None) -> int: if not args.case or not args.out: parser.error("benchmark fast requires --case and --out") result = run_fast_benchmark_case(args.case, args.out, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine) + elif args.benchmark_action == "fast-suite": + if not args.out: + parser.error("benchmark fast-suite requires --out") + result = run_fast_benchmark_suite(args.root, args.out, case_ids=args.case_id, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine) else: if not args.case or not args.run_dir: parser.error("benchmark score requires --case and --run") diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index b903475..b138db0 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -1,7 +1,7 @@ """Deterministic and model-assisted quality evaluation APIs.""" from .rebuild_visual import run_rebuild_visual_quality_check -from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, score_benchmark_case, validate_benchmark_case +from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case __all__ = [ "list_benchmark_cases", @@ -9,6 +9,7 @@ "run_rebuild_visual_quality_check", "run_benchmark_case", "run_fast_benchmark_case", + "run_fast_benchmark_suite", "score_benchmark_case", "validate_benchmark_case", ] diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 2e7c199..81e23aa 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -697,3 +697,123 @@ def run_fast_benchmark_case( } write_json(Path(out) / "fast_benchmark_result.json", result) return result + + +def run_fast_benchmark_suite( + benchmarks_root: str | Path, + out: str | Path, + case_ids: list[str] | None = None, + deadline_seconds: int = 180, + planner_mode: str = "vlm", + planner_model: str | None = None, + ocr_engine: str = "off", +) -> dict[str, Any]: + root = Path(benchmarks_root).resolve() + target = ensure_dir(out).resolve() + selected = set(case_ids or []) + listing = list_benchmark_cases(root, suite="paper-to-image") + cases = [item for item in listing.get("cases", []) if item.get("valid") and (not selected or str(item.get("case_id")) in selected)] + results = [] + for item in cases: + case_id = str(item.get("case_id")) + case_out = target / case_id + try: + result = run_fast_benchmark_case( + item["case_dir"], + case_out, + deadline_seconds=deadline_seconds, + planner_mode=planner_mode, + planner_model=planner_model, + ocr_engine=ocr_engine, + ) + except Exception as exc: + result = {"summary": "Fast benchmark case raised an exception.", "ok": False, "case_id": case_id, "error": str(exc)} + run = result.get("run", {}) if isinstance(result.get("run"), dict) else {} + benchmark = result.get("benchmark", {}) if isinstance(result.get("benchmark"), dict) else {} + metrics = benchmark.get("metrics", {}) if isinstance(benchmark.get("metrics"), dict) else {} + provider = run.get("provider", {}) if isinstance(run.get("provider"), dict) else {} + results.append({ + "case_id": case_id, + "ok": bool(result.get("ok")), + "production_ready": bool(run.get("production_ready")), + "passed_planning_thresholds": bool(result.get("passed_planning_thresholds")), + "contract_source": run.get("contract_source"), + "cache_hit": bool(run.get("cache_hit")), + "document_cache_hit": bool(run.get("document_cache_hit")), + "elapsed_seconds": run.get("elapsed_seconds"), + "stage_timings": run.get("stage_timings", {}), + "provider": provider, + "plan_entity_recall": metrics.get("plan_entity_recall"), + "plan_relation_recall": metrics.get("plan_relation_recall"), + "plan_forbidden_content_count": metrics.get("plan_forbidden_content_count"), + "threshold_failures": benchmark.get("threshold_failures", []), + "error": result.get("error"), + }) + + def numbers(field: str) -> list[float]: + values = [] + for item in results: + value = item.get(field) + if isinstance(value, (int, float)): + values.append(float(value)) + return values + + def timing(name: str) -> list[float]: + return [float(item.get("stage_timings", {}).get(name)) for item in results if isinstance(item.get("stage_timings", {}).get(name), (int, float))] + + def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * fraction) - 1)) + return round(ordered[index], 4) + + provider_attempts = sum(int(item.get("provider", {}).get("attempts") or 0) for item in results) + provider_retries = sum(int(item.get("provider", {}).get("retries_used") or 0) for item in results) + provider_calls = [item for item in results if item.get("provider", {}).get("attempts")] + failure_categories: dict[str, int] = {} + for item in provider_calls: + for category in item.get("provider", {}).get("failure_categories", []) or []: + failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1 + total_times = numbers("elapsed_seconds") + entity_scores = numbers("plan_entity_recall") + relation_scores = numbers("plan_relation_recall") + aggregate = { + "case_count": len(results), + "successful_run_count": sum(bool(item.get("ok")) for item in results), + "production_ready_count": sum(bool(item.get("production_ready")) for item in results), + "planning_threshold_pass_count": sum(bool(item.get("passed_planning_thresholds")) for item in results), + "cache_hit_count": sum(bool(item.get("cache_hit")) for item in results), + "cache_hit_rate": round(sum(bool(item.get("cache_hit")) for item in results) / max(1, len(results)), 4), + "document_cache_hit_count": sum(bool(item.get("document_cache_hit")) for item in results), + "document_cache_hit_rate": round(sum(bool(item.get("document_cache_hit")) for item in results) / max(1, len(results)), 4), + "mean_plan_entity_recall": _mean(entity_scores), + "mean_plan_relation_recall": _mean(relation_scores), + "forbidden_content_total": sum(int(item.get("plan_forbidden_content_count") or 0) for item in results), + "mean_total_seconds": _mean(total_times), + "p95_total_seconds": percentile(total_times, 0.95), + "mean_document_preparation_seconds": _mean(timing("document_preparation_seconds")), + "mean_document_extraction_seconds": _mean(timing("document_extraction_seconds")), + "mean_semantic_compilation_seconds": _mean(timing("semantic_compilation_seconds")), + "provider_call_count": len(provider_calls), + "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), + "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, + "provider_attempts": provider_attempts, + "provider_retries_used": provider_retries, + "provider_failure_categories": failure_categories, + } + report = { + "summary": "Fast paper framework benchmark suite completed.", + "ok": bool(results) and all(item.get("ok") for item in results), + "benchmarks_root": str(root), + "out_dir": str(target), + "planner_mode": planner_mode, + "planner_model": planner_model, + "deadline_seconds": deadline_seconds, + "ocr_engine": ocr_engine, + "selected_case_ids": [item.get("case_id") for item in cases], + "aggregate": aggregate, + "cases": results, + } + write_json(target / "fast_suite_report.json", report) + return report diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index c4090cc..e2ae43a 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -13,7 +13,7 @@ from typing import Any, Callable -MOJIBAKE_MARKERS = ("�", "Ã", "Â", "鈭", "鈥", "尾", "卤", "绗", "鍥", "琛") +MOJIBAKE_MARKERS = ("\ufffd", "Ã", "Â", "â€", "锟", "鈥", "銆", "鏅") SECTION_ALIASES = { "abstract": ("abstract",), "introduction": ("introduction", "background"), @@ -43,11 +43,13 @@ def _normalized_compare_text(text: str) -> str: def _replacement_rate(text: str) -> float: - return round(text.count("�") / max(1, len(text)), 6) + value = str(text or "") + return round(value.count("\ufffd") / max(1, len(value)), 6) def _mojibake_rate(text: str) -> float: - return round(sum(text.count(marker) for marker in MOJIBAKE_MARKERS) / max(1, len(text)), 6) + value = str(text or "") + return round(sum(value.count(marker) for marker in MOJIBAKE_MARKERS) / max(1, len(value)), 6) def _block_kind(text: str, width_ratio: float = 0.0) -> str: @@ -183,15 +185,21 @@ def _pdf_metadata(path: Path) -> dict[str, Any]: def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None) -> tuple[list[dict[str, Any]], str]: if adapter: return list(adapter(image_path, lang) or []), "adapter" - from ..reference_text_extractor import run_easyocr, run_paddle_ocr + from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr requested = engine if requested == "auto": - requested = "easyocr" + requested = "rapidocr" try: - import easyocr # noqa: F401 + import rapidocr_onnxruntime # noqa: F401 except Exception: - requested = "paddle" + requested = "easyocr" + try: + import easyocr # noqa: F401 + except Exception: + requested = "paddle" + if requested == "rapidocr": + return run_rapidocr(image_path, lang), "rapidocr" if requested == "easyocr": return run_easyocr(image_path, lang), "easyocr" if requested == "paddle": @@ -199,27 +207,139 @@ def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | N return [], "off" +def _group_ocr_words(records: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: + words = [] + for record in records: + quad = record.get("quad") or [] + if len(quad) < 4 or not _clean(record.get("text", "")): + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + words.append({ + "bbox": [min(xs), min(ys), max(xs), max(ys)], + "text": _clean(record.get("text", "")), + "confidence": float(record.get("confidence") or 0.0), + }) + words.sort(key=lambda item: ((item["bbox"][1] + item["bbox"][3]) / 2.0, item["bbox"][0])) + lines: list[dict[str, Any]] = [] + for word in words: + bbox = word["bbox"] + center_y = (bbox[1] + bbox[3]) / 2.0 + height = max(1.0, bbox[3] - bbox[1]) + line = next(( + item for item in reversed(lines[-12:]) + if abs(center_y - item["center_y"]) <= max(height, item["mean_height"]) * 0.62 + and bbox[0] >= item["bbox"][0] - image_width * 0.08 + ), None) + if line is None: + lines.append({"words": [word], "bbox": list(bbox), "center_y": center_y, "mean_height": height}) + continue + line["words"].append(word) + line["bbox"] = [min(line["bbox"][0], bbox[0]), min(line["bbox"][1], bbox[1]), max(line["bbox"][2], bbox[2]), max(line["bbox"][3], bbox[3])] + line["center_y"] = sum((item["bbox"][1] + item["bbox"][3]) / 2.0 for item in line["words"]) / len(line["words"]) + line["mean_height"] = sum(max(1.0, item["bbox"][3] - item["bbox"][1]) for item in line["words"]) / len(line["words"]) + + grouped = [] + for line in lines: + line["words"].sort(key=lambda item: item["bbox"][0]) + grouped.append({ + "bbox": line["bbox"], + "text": _clean(" ".join(item["text"] for item in line["words"])), + "confidence": round(sum(item["confidence"] for item in line["words"]) / max(1, len(line["words"])), 4), + }) + grouped.sort(key=lambda item: (item["bbox"][1], item["bbox"][0])) + heights = sorted(max(1.0, item["bbox"][3] - item["bbox"][1]) for item in grouped) + median_height = heights[len(heights) // 2] if heights else 12.0 + parent = 0 + previous_by_column: dict[int, dict[str, Any]] = {} + for item in grouped: + center_x = (item["bbox"][0] + item["bbox"][2]) / 2.0 + width = item["bbox"][2] - item["bbox"][0] + column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 + previous = previous_by_column.get(column) + gap = item["bbox"][1] - previous["bbox"][3] if previous else float("inf") + starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|figure|fig\.|table)\b", item["text"], re.IGNORECASE)) + if previous is None or gap > median_height * 1.8 or starts_new: + parent += 1 + item["parent_block"] = parent + previous_by_column[column] = item + return grouped + + +def _assign_ocr_parent_blocks(lines: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: + ordered = sorted(lines, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) + heights = sorted(max(1.0, float(item["bbox"][3]) - float(item["bbox"][1])) for item in ordered) + median_height = heights[len(heights) // 2] if heights else 12.0 + parent = 0 + previous_by_column: dict[int, dict[str, Any]] = {} + last_item: dict[str, Any] | None = None + for item in ordered: + center_x = (float(item["bbox"][0]) + float(item["bbox"][2])) / 2.0 + width = float(item["bbox"][2]) - float(item["bbox"][0]) + column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 + previous = previous_by_column.get(column) + if last_item and last_item.get("_caption_group"): + last_gap = float(item["bbox"][1]) - float(last_item["bbox"][3]) + last_height = max( + median_height, + float(item["bbox"][3]) - float(item["bbox"][1]), + float(last_item["bbox"][3]) - float(last_item["bbox"][1]), + ) + if last_gap <= last_height * 1.15: + previous = last_item + gap = float(item["bbox"][1]) - float(previous["bbox"][3]) if previous else float("inf") + local_height = max( + median_height, + float(item["bbox"][3]) - float(item["bbox"][1]), + (float(previous["bbox"][3]) - float(previous["bbox"][1])) if previous else 0.0, + ) + text = str(item.get("text") or "") + starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|figure|fig\.|table)\b", text, re.IGNORECASE)) + starts_caption = bool(re.match(r"^(?:figure|fig\.|table)\b", text, re.IGNORECASE)) + continuing_caption = bool(previous and previous.get("_caption_group") and gap <= local_height * 1.15 and not starts_new) + paragraph_break = bool(previous and not continuing_caption and str(previous.get("text") or "").rstrip().endswith((".", ":")) and gap > local_height * 0.45) + if previous is None or gap > local_height * 1.15 or starts_new or paragraph_break: + parent += 1 + item["parent_block"] = parent + item["_caption_group"] = starts_caption or continuing_caption + previous_by_column[column] = item + last_item = item + return ordered + + def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None) -> tuple[list[dict[str, Any]], str, str | None]: try: with tempfile.TemporaryDirectory() as temp: target = Path(temp) / f"page_{page_number:03d}.png" - pixmap = page.get_pixmap(dpi=200, alpha=False) + dpi = 72 if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 + pixmap = page.get_pixmap(dpi=dpi, alpha=False) pixmap.save(str(target)) records, used_engine = _ocr_records(target, engine, lang, adapter) + if used_engine != "rapidocr": + records = _group_ocr_words(records, float(pixmap.width)) + else: + normalized_records = [] + for record in records: + quad = record.get("quad") or [] + if len(quad) < 4: + continue + xs = [float(point[0]) for point in quad] + ys = [float(point[1]) for point in quad] + normalized_records.append({"bbox": [min(xs), min(ys), max(xs), max(ys)], "text": _clean(record.get("text", "")), "confidence": float(record.get("confidence") or 0.0)}) + records = _assign_ocr_parent_blocks(normalized_records, float(pixmap.width)) blocks = [] scale_x = float(page.rect.width) / max(1, pixmap.width) scale_y = float(page.rect.height) / max(1, pixmap.height) for record in records: - quad = record.get("quad") or [] - if not quad: + raw_bbox = record.get("bbox") + if not raw_bbox: continue - xs = [float(point[0]) for point in quad] - ys = [float(point[1]) for point in quad] blocks.append({ - "bbox": [min(xs) * scale_x, min(ys) * scale_y, max(xs) * scale_x, max(ys) * scale_y], + "bbox": [float(raw_bbox[0]) * scale_x, float(raw_bbox[1]) * scale_y, float(raw_bbox[2]) * scale_x, float(raw_bbox[3]) * scale_y], "text": _clean(record.get("text", "")), "source": used_engine, "confidence": round(float(record.get("confidence") or 0.0), 4), + "parent_block": record.get("parent_block"), }) return blocks, used_engine, None except Exception as exc: @@ -288,14 +408,22 @@ def _read_pdf_pages( pass ocr_pages: list[int] = [] + ocr_page_durations: list[dict[str, Any]] = [] if ocr_engine != "off": prioritized = ocr_candidates[:max_ocr_pages] for page_number in prioritized: - if deadline_at is not None and time.monotonic() >= deadline_at - 20: - warnings.append("OCR stopped to preserve deadline validation budget") - break + if deadline_at is not None: + remaining = deadline_at - time.monotonic() + completed_times = [float(item["elapsed_seconds"]) for item in ocr_page_durations if isinstance(item.get("elapsed_seconds"), (int, float))] + estimated = (sum(completed_times) / len(completed_times) * 1.25) if completed_times else 45.0 + if remaining < estimated + 20: + warnings.append("OCR stopped to preserve deadline validation budget") + break page = document[page_number - 1] + ocr_started = time.monotonic() blocks, used_engine, error = _ocr_page(page, page_number, ocr_engine, ocr_lang, ocr_adapter) + ocr_elapsed = round(time.monotonic() - ocr_started, 3) + ocr_page_durations.append({"page": page_number, "engine": used_engine, "elapsed_seconds": ocr_elapsed, "success": not bool(error)}) if error: warnings.append(f"page {page_number} OCR unavailable: {error}") continue @@ -323,6 +451,7 @@ def _read_pdf_pages( "metadata": metadata, "ocr_candidate_pages": ocr_candidates, "ocr_pages": ocr_pages, + "ocr_page_durations": ocr_page_durations, "warnings": warnings, "poppler_available": bool(poppler_text), "elapsed_seconds": elapsed, @@ -475,15 +604,25 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st readable = [page for page in pages if len(_normalized_compare_text(page.get("text", ""))) >= 80] readable_ratio = round(len(readable) / max(1, len(pages)), 4) ocr_count = sum(1 for page in pages if page.get("used_ocr")) - pdf_type = "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" + candidate_count = len(details.get("ocr_candidate_pages", [])) + pdf_type = "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count or (pages and candidate_count == len(pages)) else "born_digital" coverage = _section_coverage(pages, index.get("sections", [])) warnings = list(details.get("warnings", [])) + ocr_blocks = [block for page in pages if page.get("used_ocr") for block in page.get("blocks", []) if isinstance(block, dict)] + ocr_confidences = [float(block.get("confidence") or 0.0) for block in ocr_blocks] + mean_ocr_confidence = round(sum(ocr_confidences) / max(1, len(ocr_confidences)), 4) if ocr_blocks else None + short_ocr_blocks = [block for block in ocr_blocks if len(_normalized_compare_text(str(block.get("text") or ""))) < 3] + short_ocr_block_ratio = round(len(short_ocr_blocks) / max(1, len(ocr_blocks)), 4) if ocr_blocks else None if readable_ratio < 0.6: warnings.append("fewer than 60% of pages contain reliable text") + if mean_ocr_confidence is not None and mean_ocr_confidence < 0.5: + warnings.append("mean OCR confidence is below 0.50") + if short_ocr_block_ratio is not None and short_ocr_block_ratio > 0.3: + warnings.append("OCR output is highly fragmented") if not coverage.get("abstract") or not coverage.get("method"): warnings.append("abstract or method-like section was not confidently located") agreements = [page["poppler_agreement"] for page in pages if isinstance(page.get("poppler_agreement"), (int, float))] - report_status = "fail" if readable_ratio < 0.6 else "warning" if warnings else "pass" + report_status = "fail" if readable_ratio < 0.6 or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) else "warning" if warnings else "pass" return { "summary": "PDF extraction quality and fallback report.", "status": report_status, @@ -495,6 +634,9 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "empty_or_low_text_pages": [page["page"] for page in pages if page not in readable], "ocr_candidate_pages": details.get("ocr_candidate_pages", []), "ocr_pages": details.get("ocr_pages", []), + "ocr_page_durations": details.get("ocr_page_durations", []), + "mean_ocr_confidence": mean_ocr_confidence, + "short_ocr_block_ratio": short_ocr_block_ratio, "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), "mojibake_rate": round(sum(page.get("mojibake_rate", 0.0) for page in pages) / max(1, len(pages)), 6), "cross_extractor_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, @@ -531,8 +673,6 @@ def parse_paper( document_index = _document_index(pages, sections) evidence = _build_evidence(pages, sections, document_index, max_chars) all_text = "\n\n".join(page.get("text", "") for page in pages) - if not evidence: - raise ValueError("No readable paper text was extracted") extraction_report = _extraction_report(source, pages, document_index, details) return { "summary": "Paper parsed into a structured, page-aware evidence model.", diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py new file mode 100644 index 0000000..43b20d8 --- /dev/null +++ b/rfs/paper_to_image/contract_completion.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + + +def _normalized(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + + +def _label(item: dict[str, Any]) -> str: + return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() + + +def _stable_id(prefix: str, label: str) -> str: + slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", label.casefold()).split() if part)[:48] + return f"{prefix}_{slug or 'item'}" + + +@dataclass(frozen=True) +class ConceptRule: + key: str + label: str + field: str + role: str + pattern: str + aliases: tuple[str, ...] = () + requires_overview: bool = False + + +CONCEPT_RULES = ( + ConceptRule("input_image", "Input Image", "inputs", "input", r"\binput images?\b|\bsplit an image\b|\braw image\b|\btest images?\b|\bimages? are shown\b", ("image", "2d image")), + ConceptRule("images", "Images", "inputs", "input modality", r"\b(?:batch of\s+)?\(?images?\s*[,)]", ("training images", "image batch"), True), + ConceptRule("image", "Image", "inputs", "input modality", r"\bimages?\s*\+\s*text\b|\bimage encoder\b", ("images", "input image"), True), + ConceptRule("text", "Text", "inputs", "input modality", r"\b(?:batch of\s+)?\(?image\s*,\s*text\)?|\bimages?\s*\+\s*text\b|\btext training examples?\b", ("captions", "training text", "text batch")), + ConceptRule("audio", "Audio", "inputs", "input modality", r"\baudio\b", (), True), + ConceptRule("depth", "Depth", "inputs", "input modality", r"\bdepth\b", (), True), + ConceptRule("thermal", "Thermal", "inputs", "input modality", r"\bthermal\b", (), True), + ConceptRule("imu", "IMU", "inputs", "input modality", r"\bimu\b", (), True), + ConceptRule("prompt", "Prompt", "inputs", "input", r"\b(?:input|segmentation|point|box|mask) prompts?\b|\bprompt encoder\b", ("sparse prompts", "dense prompts", "segmentation prompt"), True), + ConceptRule("task_input", "Input", "inputs", "input", r"\bgiven an input\b|\btask (?:prompt|instruction)\b", ("task prompt", "task instruction"), True), + ConceptRule("camera_ray", "Camera Rays", "inputs", "input", r"\bcamera rays?\b", ("camera ray", "ray", "rays")), + ConceptRule("viewing_direction", "Viewing Direction", "inputs", "conditioning", r"\bview(?:ing)? directions?\b", ("view direction",)), + ConceptRule("object_queries", "Object Queries", "inputs", "conditioning", r"\bobject quer(?:y|ies)\b", ("learned object queries",)), + ConceptRule("class_descriptions", "Class Descriptions", "inputs", "conditioning", r"\b(?:names or descriptions|class names|class descriptions|text prompts)\b", ("class names", "text prompts")), + ConceptRule("cnn_backbone", "CNN Backbone", "modules", "feature extractor", r"\b(?:conventional\s+|common\s+)?cnn backbone\b|\bconvolutional backbone\b", ("backbone", "convolutional backbone")), + ConceptRule("backbone", "Backbone", "modules", "feature extractor", r"\bbackbones?\b", ("backbone architecture",), True), + ConceptRule("image_encoder", "Image Encoder", "modules", "encoder", r"\bimage encoder\b"), + ConceptRule("text_encoder", "Text Encoder", "modules", "encoder", r"\btext encoder\b"), + ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b"), + ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b"), + ConceptRule("position_embedding", "Position Embedding", "modules", "conditioning", r"\bposition embeddings?\b", ("positional embedding",), True), + ConceptRule("positional_encoding", "Positional Encoding", "modules", "conditioning", r"\bposition(?:al)? encod(?:ing|ings)\b", (), True), + ConceptRule("image_patches", "Image Patches", "modules", "intermediate", r"\bimage patches?\b|\bfixed-size patches\b", ("patches",), True), + ConceptRule("linear_projection", "Linear Projection of Flattened Patches", "modules", "projection", r"\blinear projection(?: of flattened patches)?\b|\blinearly embed\b", ("linear projection",), True), + ConceptRule("class_token", "Class Token", "modules", "conditioning", r"\bclass(?:ification)? token\b|\bclass embedding\b", ("classification token", "class embedding"), True), + ConceptRule("mlp_head", "MLP Head", "modules", "prediction head", r"\bmlp head\b|\bclassification head[^.]{0,100}\bmlp\b"), + ConceptRule("sampled_points", "Sampled 3D Points", "modules", "intermediate", r"\bsampl(?:e|ed|ing)\s+(?:the\s+)?(?:5d coordinates|3d points)\b|\b5d coordinates\b", ("5d coordinates", "spatial locations")), + ConceptRule("prediction_ffn", "Feed Forward Network", "modules", "prediction head", r"\bfeed[ -]?forward network\b|\bprediction ffn\b|\bffn\b", ("ffn", "prediction feed-forward network")), + ConceptRule("region_proposals", "Region Proposals", "modules", "proposal artifact", r"\bregion proposals?\b|\bregion proposal network\b|\brpn\b", ("proposals", "region proposal network (rpn)")), + ConceptRule("roi_align", "RoIAlign", "modules", "alignment", r"\broialign\b"), + ConceptRule("generate", "Generate", "modules", "generator", r"\bstarts? by generating\b|\bgenerate(?:s|d| an output)?\b", ("init", "generator"), True), + ConceptRule("initial_output", "Initial Output", "modules", "artifact", r"\binitial output\b|\bpreviously generated output\b", (), True), + ConceptRule("feedback", "Feedback", "modules", "feedback provider", r"\bget feedback\b|\breceive feedback\b|\bfeedback provider\b", ("feedback provider",), True), + ConceptRule("self_feedback", "Self-Feedback", "modules", "feedback artifact", r"\bself[ -]?feedback\b|\bfeedback is passed back\b", ("self-feedback (nl)",), True), + ConceptRule("refine", "Refine", "modules", "refiner", r"\brefines? the previously generated output\b|\biterate\s*/\s*refine\b", ("refiner",), True), + ConceptRule("modality_encoders", "Modality Encoders", "modules", "module group", r"\bmodality encoders?\b|\bencoders for each modality\b", ("transformer-based encoders for each modality",)), + ConceptRule("joint_embedding", "Joint Embedding Space", "modules", "shared representation", r"\b(?:single shared )?(?:joint|common) embedding space\b|\bshared representation space\b", ("joint embedding", "single shared joint embedding space")), + ConceptRule("prompt_encoder", "Prompt Encoder", "modules", "encoder", r"\bprompt encoder\b"), + ConceptRule("mask_decoder", "Mask Decoder", "modules", "decoder", r"\bmask decoder\b"), + ConceptRule("mlp", "MLP", "modules", "neural field", r"\bneural radiance field\b|\bfeeding[^.]{0,100}\bmlp\b|\bmlp to produce\b", ("neural radiance field", "neural network")), + ConceptRule("image_embeddings", "Image Embeddings", "modules", "representation", r"\bimage embeddings?\b|\bimage features?\b", (), True), + ConceptRule("text_embeddings", "Text Embeddings", "modules", "representation", r"\btext embeddings?\b|\btext features?\b"), + ConceptRule("contrastive_objective", "Contrastive Learning", "modules", "training objective", r"\bcontrastive (?:pre-?training|learning|objective|loss)\b|\bcorrect pairings?\b", ("contrastive pre-training", "image-text contrastive loss", "correct pairings")), + ConceptRule("bipartite_matching", "Bipartite Matching", "modules", "training objective", r"\bbipartite matching\b|\bhungarian matching\b", ("hungarian matching", "set prediction loss")), + ConceptRule("volume_density", "Volume Density", "modules", "field output", r"\bvolume density\b", ("density", "sigma")), + ConceptRule("color", "Color", "modules", "field output", r"\b(?:rgb|emitted) colou?r\b|\bradiance\b|\bcolou?r and volume density\b", ("rgb color", "radiance"), True), + ConceptRule("volume_rendering", "Volume Rendering", "modules", "renderer", r"\b(?:differentiable\s+)?volume rendering\b", ("differentiable volume rendering",)), + ConceptRule("class_predictions", "Class Predictions", "outputs", "output", r"\bclass (?:prediction|predictions|labels?)\b|\bpredicts?[^.]{0,80}\bclass\b", ("classes", "class labels")), + ConceptRule("class_prediction", "Class Prediction", "outputs", "output", r"\bimage classification predictions?\b|\bclassification predictions?\b|\bclass label\b", ("image classification predictions", "classification predictions", "class label")), + ConceptRule("box_predictions", "Bounding Box Predictions", "outputs", "output", r"\bbounding box(?:es| predictions?)?\b|\bbox predictions?\b", ("bounding boxes", "box predictions")), + ConceptRule("classification", "Classification", "outputs", "output head", r"\bclassification head\b|\bpredicts? the class label\b", ("class label",)), + ConceptRule("box_regression", "Bounding-box Regression", "outputs", "output head", r"\bbounding[ -]?box regression\b|\bbox regression\b", ("bounding box", "box branch")), + ConceptRule("mask_branch", "Mask Branch", "outputs", "output head", r"\bmask branch\b"), + ConceptRule("refined_output", "Refined Output", "outputs", "output", r"\brefined output\b|\bquality of the output improves\b"), + ConceptRule("emergent_alignment", "Emergent Cross-modal Alignment", "outputs", "output", r"\bemergent (?:cross-modal )?alignments?\b|\bbinding (?:agent|property)\b", ("emergent alignment", "binding agent", "binding property")), + ConceptRule("valid_mask", "Valid Segmentation Mask", "outputs", "output", r"\bvalid masks?\b|\bobject masks?\b", ("valid masks", "valid mask"), True), + ConceptRule("zero_shot_classifier", "Zero-Shot Classifier", "outputs", "inference output", r"\bzero[ -]?shot (?:linear )?classifier\b|\bzero[ -]?shot prediction\b", ("zero-shot linear classifier", "zero-shot prediction")), + ConceptRule("rendered_image", "Rendered Image", "outputs", "output", r"\b(?:rendered|synthesi[sz]ed) (?:images?|views?)\b|\bsynthesi[sz]e images?\b|\bnovel views?\b", ("synthesized image", "synthesized view", "novel view")), +) + + +RELATION_RULES = ( + ("input_image", "image_patches", "data_flow"), + ("image_patches", "linear_projection", "data_flow"), + ("linear_projection", "transformer_encoder", "data_flow"), + ("position_embedding", "transformer_encoder", "conditioning"), + ("class_token", "transformer_encoder", "conditioning"), + ("transformer_encoder", "mlp_head", "data_flow"), + ("mlp_head", "class_prediction", "data_flow"), + ("input_image", "cnn_backbone", "data_flow"), + ("input_image", "backbone", "data_flow"), + ("cnn_backbone", "transformer_encoder", "data_flow"), + ("positional_encoding", "transformer_encoder", "conditioning"), + ("transformer_encoder", "transformer_decoder", "data_flow"), + ("object_queries", "transformer_decoder", "conditioning"), + ("transformer_decoder", "prediction_ffn", "data_flow"), + ("prediction_ffn", "class_predictions", "branch"), + ("prediction_ffn", "box_predictions", "branch"), + ("class_predictions", "bipartite_matching", "training_objective"), + ("box_predictions", "bipartite_matching", "training_objective"), + ("backbone", "region_proposals", "data_flow"), + ("region_proposals", "roi_align", "data_flow"), + ("backbone", "roi_align", "feature_flow"), + ("roi_align", "classification", "branch"), + ("roi_align", "class_predictions", "branch"), + ("roi_align", "box_regression", "branch"), + ("roi_align", "box_predictions", "branch"), + ("roi_align", "mask_branch", "branch"), + ("task_input", "generate", "data_flow"), + ("generate", "initial_output", "data_flow"), + ("initial_output", "feedback", "evaluation"), + ("feedback", "self_feedback", "feedback"), + ("initial_output", "refine", "revision_input"), + ("self_feedback", "refine", "feedback"), + ("refine", "refined_output", "data_flow"), + ("refined_output", "feedback", "feedback_loop"), + ("image", "modality_encoders", "encoding"), + ("text", "modality_encoders", "encoding"), + ("audio", "modality_encoders", "encoding"), + ("depth", "modality_encoders", "encoding"), + ("thermal", "modality_encoders", "encoding"), + ("imu", "modality_encoders", "encoding"), + ("modality_encoders", "joint_embedding", "alignment"), + ("joint_embedding", "emergent_alignment", "enables"), + ("image", "image_encoder", "data_flow"), + ("prompt", "prompt_encoder", "data_flow"), + ("image_encoder", "mask_decoder", "feature_flow"), + ("prompt_encoder", "mask_decoder", "conditioning"), + ("mask_decoder", "valid_mask", "data_flow"), + ("images", "image_encoder", "encoding"), + ("text", "text_encoder", "encoding"), + ("image_encoder", "image_embeddings", "data_flow"), + ("text_encoder", "text_embeddings", "data_flow"), + ("image_embeddings", "contrastive_objective", "alignment"), + ("text_embeddings", "contrastive_objective", "alignment"), + ("class_descriptions", "text_encoder", "encoding"), + ("image_embeddings", "zero_shot_classifier", "classification"), + ("text_embeddings", "zero_shot_classifier", "classification"), + ("camera_ray", "sampled_points", "sampling"), + ("sampled_points", "positional_encoding", "encoding"), + ("positional_encoding", "mlp", "data_flow"), + ("viewing_direction", "mlp", "conditioning"), + ("mlp", "volume_density", "prediction"), + ("mlp", "color", "prediction"), + ("volume_density", "volume_rendering", "rendering_input"), + ("color", "volume_rendering", "rendering_input"), + ("volume_rendering", "rendered_image", "data_flow"), +) + + +def _overview_candidates(parsed: dict[str, Any], limit: int = 2) -> list[dict[str, Any]]: + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|model|system|workflow|directly predicts)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention|distribution)\b", re.IGNORECASE) + ranked = [] + for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): + caption = str(figure.get("caption") or "") + score = (8 if positive.search(caption) else 0) + (4 if re.match(r"^(figure|fig\.)\s*[12]\b", caption, re.IGNORECASE) else 0) + score += 3 if 160 <= len(caption) <= 1600 else 0 + score -= 7 if negative.search(caption) else 0 + ranked.append((score, -index, figure)) + return [item for _, _, item in sorted(ranked, reverse=True)[:limit]] + + +def _evidence_for_figure(parsed: dict[str, Any], figure: dict[str, Any]) -> dict[str, Any] | None: + caption = str(figure.get("caption") or "") + return next( + ( + item + for item in parsed.get("evidence", []) + if item.get("kind") == "caption" + and item.get("page") == figure.get("page") + and str(item.get("text") or "").startswith(caption[:60]) + ), + None, + ) + + +def _relevant_evidence(parsed: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + candidates = _overview_candidates(parsed) + selected = [item for item in (_evidence_for_figure(parsed, figure) for figure in candidates) if item] + selected_ids = {item["id"] for item in selected} + page_count = max(1, int(parsed.get("page_count") or 1)) + early_page_limit = max(8, int(page_count * 0.45)) + useful_sections = ("abstract", "introduction", "method", "approach", "architecture", "framework", "model", "system", "figure captions") + relevant = list(selected) + for item in parsed.get("evidence", []): + if item.get("id") in selected_ids: + continue + section = str(item.get("section_hint") or "").casefold() + if any(term in section for term in ("reference", "acknowledg")): + continue + if int(item.get("page") or page_count + 1) <= early_page_limit or any(term in section for term in useful_sections): + relevant.append(item) + return selected, relevant + + +def _find_matches(rule: ConceptRule, selected: list[dict[str, Any]], relevant: list[dict[str, Any]]) -> list[dict[str, Any]]: + pattern = re.compile(rule.pattern, re.IGNORECASE) + primary = [item for item in selected if pattern.search(str(item.get("text") or ""))] + if primary: + return primary[:2] + if rule.requires_overview: + return [] + secondary = [item for item in relevant if pattern.search(str(item.get("text") or ""))] + return secondary[:2] + + +def _find_existing(spec: dict[str, Any], rule: ConceptRule) -> tuple[str, dict[str, Any]] | None: + accepted = {_normalized(rule.key), _normalized(rule.label), *(_normalized(value) for value in rule.aliases)} + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + value = _normalized(_label(item)) + item_id = _normalized(item.get("id")) + if item_id in accepted or value in accepted: + return field, item + if rule.key != "self_feedback" and field == rule.field and len(value) >= 8 and any(len(candidate) >= 8 and (value in candidate or candidate in value) for candidate in accepted if candidate): + return field, item + return None + + +def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + selected, relevant = _relevant_evidence(parsed) + found: dict[str, dict[str, Any]] = {} + added_entities: list[str] = [] + upgraded_entities: list[str] = [] + adopted_entities: list[str] = [] + for rule in CONCEPT_RULES: + matches = _find_matches(rule, selected, relevant) + if not matches: + continue + evidence_ids = list(dict.fromkeys(str(item.get("id")) for item in matches if item.get("id"))) + existing = _find_existing(spec, rule) + if existing: + _, item = existing + current = _label(item) + if str(item.get("role") or "") == "paper-derived stage requiring VLM verification": + item["role"] = rule.role + adopted_entities.append(str(item.get("id") or rule.key)) + if _normalized(current) != _normalized(rule.label) and (_normalized(current) in _normalized(rule.label) or current.casefold().startswith(("neural network module", "learned "))): + item["name"] = rule.label + upgraded_entities.append(str(item.get("id") or rule.key)) + item["evidence_ids"] = list(dict.fromkeys(list(item.get("evidence_ids", [])) + evidence_ids)) + found[rule.key] = item + continue + item = {"id": _stable_id(rule.field.rstrip("s"), rule.label), "name": rule.label, "role": rule.role, "evidence_ids": evidence_ids} + spec.setdefault(rule.field, []).append(item) + found[rule.key] = item + added_entities.append(item["id"]) + + for rule in CONCEPT_RULES: + if rule.key not in found: + existing = _find_existing(spec, rule) + if existing: + found[rule.key] = existing[1] + + relations = spec.setdefault("relations", []) + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + selected_ids = [str(item.get("id")) for item in selected if item.get("id")] + evidence_by_id = {str(item.get("id")): item for item in relevant if item.get("id")} + added_relations: list[str] = [] + for source_key, target_key, relation_type in RELATION_RULES: + source = found.get(source_key) + target = found.get(target_key) + if not source or not target: + continue + pair = (str(source.get("id")), str(target.get("id"))) + if pair in existing_pairs: + continue + evidence_ids = list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(target.get("evidence_ids", [])))) + if selected_ids and not any(value in selected_ids for value in evidence_ids): + source_text = _label(source).casefold() + target_text = _label(target).casefold() + bridge = next((item for item in relevant if source_text.split()[-1] in str(item.get("text") or "").casefold() and target_text.split()[-1] in str(item.get("text") or "").casefold()), None) + source_pages = [int(evidence_by_id[value].get("page") or 0) for value in source.get("evidence_ids", []) if value in evidence_by_id] + target_pages = [int(evidence_by_id[value].get("page") or 0) for value in target.get("evidence_ids", []) if value in evidence_by_id] + nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 3) + if not bridge and not nearby: + continue + if bridge: + evidence_ids.append(str(bridge.get("id"))) + relations.append({"source": pair[0], "target": pair[1], "type": relation_type, "label": "", "evidence_ids": list(dict.fromkeys(evidence_ids))}) + existing_pairs.add(pair) + added_relations.append(f"{pair[0]}->{pair[1]}") + + overview_terms = [rule.key for rule in CONCEPT_RULES if _find_matches(rule, selected, [])] + covered_terms = [key for key in overview_terms if key in found] + return { + "summary": "Conservative evidence-driven contract completion report.", + "selected_overview_evidence_ids": selected_ids, + "overview_term_count": len(overview_terms), + "covered_overview_term_count": len(covered_terms), + "overview_term_coverage": round(len(covered_terms) / max(1, len(overview_terms)), 4), + "added_entities": added_entities, + "upgraded_entities": upgraded_entities, + "adopted_entities": adopted_entities, + "added_relations": added_relations, + "rules_are_evidence_gated": True, + } diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 95f7e05..1df7600 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -7,7 +7,7 @@ from PIL import Image -from ..reference_text_extractor import run_easyocr, run_paddle_ocr +from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available @@ -39,6 +39,8 @@ def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = No if adapter: records = adapter(path, lang) return records, "adapter", None + if engine == "rapidocr": + return run_rapidocr(path, lang), "rapidocr", None if engine == "easyocr": return run_easyocr(path, lang), "easyocr", None if engine == "paddle": diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py new file mode 100644 index 0000000..a2e947e --- /dev/null +++ b/rfs/paper_to_image/document_cache.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +from ..utils import read_json, write_json + + +DOCUMENT_CACHE_VERSION = 2 + + +def _cache_root() -> Path: + return Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) + + +def document_cache_path( + source: str | Path, + *, + ocr_engine: str, + ocr_lang: str, + max_chars: int, + max_ocr_pages: int, +) -> Path: + path = Path(source).resolve() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + signature = json.dumps({ + "version": DOCUMENT_CACHE_VERSION, + "suffix": path.suffix.lower(), + "ocr_engine": str(ocr_engine), + "ocr_lang": str(ocr_lang), + "max_chars": int(max_chars), + "max_ocr_pages": int(max_ocr_pages), + }, sort_keys=True).encode("utf-8") + variant = hashlib.sha256(signature).hexdigest()[:16] + return _cache_root() / "documents" / digest / variant / "document_model.json" + + +def read_document_cache( + source: str | Path, + *, + ocr_engine: str, + ocr_lang: str, + max_chars: int = 90000, + max_ocr_pages: int = 6, +) -> dict[str, Any] | None: + path = document_cache_path(source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, max_chars=max_chars, max_ocr_pages=max_ocr_pages) + if not path.exists(): + return None + cached = read_json(path) + if not isinstance(cached, dict) or cached.get("extraction_report", {}).get("status") == "fail": + return None + active = Path(source).resolve() + cached["source_path"] = str(active) + cached["source_name"] = active.name + if isinstance(cached.get("extraction_report"), dict): + cached["extraction_report"]["source_path"] = str(active) + return cached + + +def write_document_cache( + source: str | Path, + parsed: dict[str, Any], + *, + ocr_engine: str, + ocr_lang: str, + max_chars: int = 90000, + max_ocr_pages: int = 6, +) -> Path | None: + if parsed.get("extraction_report", {}).get("status") == "fail": + return None + path = document_cache_path(source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, max_chars=max_chars, max_ocr_pages=max_ocr_pages) + write_json(path, parsed) + return path diff --git a/rfs/paper_to_image/inspection.py b/rfs/paper_to_image/inspection.py index 1df10bc..13712ba 100644 --- a/rfs/paper_to_image/inspection.py +++ b/rfs/paper_to_image/inspection.py @@ -8,6 +8,7 @@ from ..utils import ensure_dir, read_json, write_json, write_text from .analyzer import paper_markdown, parse_paper +from .document_cache import read_document_cache, write_document_cache def _source_hash(path: Path) -> str: @@ -52,13 +53,18 @@ def inspect_paper( "elapsed_seconds": elapsed, } deadline = max(30, int(deadline_seconds)) - parsed = parse_paper( - active_source, - deadline_at=time.monotonic() + deadline, - ocr_engine=ocr_engine, - ocr_lang=ocr_lang, - ocr_adapter=ocr_adapter, - ) + parsed = None if ocr_adapter else read_document_cache(active_source, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + document_cache_hit = parsed is not None + if parsed is None: + parsed = parse_paper( + active_source, + deadline_at=time.monotonic() + deadline, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + ocr_adapter=ocr_adapter, + ) + if not ocr_adapter: + write_document_cache(active_source, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) write_json(model_path, parsed) write_json(root / "extraction_report.json", parsed["extraction_report"]) write_json(root / "section_index.json", parsed["document_index"]) @@ -72,6 +78,7 @@ def inspect_paper( "source_sha256": digest, "page_count": parsed["page_count"], "evidence_count": len(parsed["evidence"]), + "document_cache_hit": document_cache_hit, "extraction_report": str(root / "extraction_report.json"), "document_model": str(model_path), "elapsed_seconds": elapsed, diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 08defc9..cb49af1 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -139,11 +139,166 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = """.strip() +def _overview_figure_candidates(parsed: dict, limit: int = 4) -> list[dict[str, Any]]: + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|method|model|system|workflow)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention map|distribution)\b", re.IGNORECASE) + ranked = [] + for index, item in enumerate(parsed.get("document_index", {}).get("figures", [])): + caption = str(item.get("caption") or "").strip() + if not caption: + continue + score = 0 + if positive.search(caption): + score += 8 + if re.match(r"^(figure|fig\.)\s*[12]\b", caption, re.IGNORECASE): + score += 5 + if 180 <= len(caption) <= 1400: + score += 4 + score += min(5, len(re.findall(r"\b(?:uses?|takes?|feeds?|passes?|predicts?|produces?|outputs?|trains?|samples?|renders?|synthesizes?|optimizes?)\b", caption, re.IGNORECASE))) + if negative.search(caption): + score -= 7 + ranked.append((score, -index, item)) + return [item for _, _, item in sorted(ranked, reverse=True)[: max(1, int(limit))]] + + +def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: int = 36000) -> str: + candidates = _overview_figure_candidates(parsed) + return f""" +# Summary + +You are compiling a scientific paper into a directed semantic graph for ONE framework figure. Return JSON only. Focus exclusively on scientific meaning; layout, illustration style, and image-generation wording will be compiled later by code. + +Primary task: +1. Select the most information-rich method/architecture/framework figure caption from the candidates below. Figure 1 is not automatically best; prefer a caption that explicitly names components and directed operations. +2. Extract every separately named input, component, intermediate representation, conditioning signal, training-only objective, inference-only step, and output needed to reproduce that overview. +3. Use exact short paper terms as visible names. Do not replace "image encoder" with a prose description such as "network that extracts image features". +4. Split compound outputs and branches into separate entities. If a component predicts class and box, create separate class and box outputs. If two encoders produce two embeddings, create both embeddings. +5. Preserve direction. Every relation endpoint must be a declared entity id. Include input boundary edges and final output edges. +6. Separate training and inference. Matching, losses, supervision, optimization, and feedback belong in training_flow or training-only entities; test-time classification, decoding, retrieval, rendering, or deployment belong in inference_flow. +7. Every factual item must cite evidence_ids copied exactly from the evidence. Unsupported details must be omitted and recorded under uncertainties. +8. Ignore baseline/comparison methods mentioned only to contrast with the proposed method. Never import terms from references, acknowledgements, bibliography, or result-only figures. + +Return exactly this compact schema: +{{ + "summary": "Fast evidence-grounded semantic plan.", + "paper_summary": {{ + "summary": "Structured paper summary.", + "title": "...", + "paper_type": "method|system|dataset|benchmark|analysis|survey|application|unknown", + "research_problem": {{"text": "...", "evidence_ids": []}}, + "central_claim": {{"text": "...", "evidence_ids": []}}, + "inputs": [], + "outputs": [], + "core_modules": [], + "innovations": [], + "training_flow": [], + "inference_flow": [], + "terminology": {{}}, + "unknowns": [] + }}, + "figure_specification": {{ + "summary": "Scientific contract for the figure.", + "figure_goal": "...", + "research_problem": {{"text": "...", "evidence_ids": []}}, + "central_claim": {{"text": "...", "evidence_ids": []}}, + "storyline": [], + "must_show": [], + "inputs": [{{"id": "...", "name": "exact short term", "role": "input", "evidence_ids": []}}], + "modules": [{{"id": "...", "name": "exact short term", "role": "module|intermediate|training_objective|conditioning", "evidence_ids": []}}], + "outputs": [{{"id": "...", "name": "exact short term", "role": "output", "evidence_ids": []}}], + "relations": [{{"source": "...", "target": "...", "type": "data_flow|encoding|conditioning|branch|alignment|prediction|training_objective|rendering_input|feedback", "label": "", "evidence_ids": []}}], + "innovations": [], + "feedback_loops": [], + "training_flow": [], + "inference_flow": [], + "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", + "required_labels": [], + "terminology": {{}}, + "forbidden_inventions": [], + "uncertainties": [] + }} +}} + +Candidate overview captions: +{json.dumps(candidates, ensure_ascii=False, indent=2)} + +User preferences relevant to wording only: +{json.dumps({key: preferences.get(key) for key in ('language', 'must_show', 'must_not_show')}, ensure_ascii=False, indent=2)} + +Prioritized paper evidence: +{evidence_excerpt(parsed, max_chars=evidence_max_chars)} +""".strip() + + +def _compile_fast_plan(raw: dict, preferences: dict) -> dict: + result = normalize_plan(raw, preferences) + spec = result["figure_specification"] + entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + entity_ids = [str(item.get("id") or "") for item in entities if str(item.get("id") or "")] + labels = [str(item.get("name") or item.get("text") or item.get("statement") or "").strip() for item in entities] + labels = [value for value in labels if value] + topology = str(spec.get("topology") or "unknown") + pattern = {"feedback": "loop", "multimodal": "hub_and_spoke", "dense_multiframe": "stacked", "branch": "two_stage"}.get(topology, "left_to_right") + result["design_plan"] = { + "summary": "Deterministic information narrative compiled from the fast semantic graph.", + "reading_order": entity_ids, + "groups": [], + "innovation_emphasis": [str(item.get("name") or item.get("text") or item.get("statement") or "") for item in spec.get("innovations", []) if isinstance(item, dict)], + "preserve": labels, + "remove": list(spec.get("forbidden_inventions", []) if isinstance(spec.get("forbidden_inventions"), list) else []), + } + result["layout_intent"] = { + "summary": "Deterministic layout intent compiled from semantic topology.", + "pattern": pattern, + "canvas_ratio": preferences.get("aspect_ratio", "16:9"), + "regions": [], + "flow_description": "Render all declared directed relations without reversing arrows.", + "whitespace": "moderate", + } + result["visual_metaphors"] = { + "summary": "Minimal paper-grounded visual objects; exact labels and arrows are added as an editable overlay.", + "items": [{"module_id": str(item.get("id") or ""), "metaphor": str(item.get("name") or "scientific module"), "must_show": [], "avoid_showing": []} for item in entities], + } + result["style_plan"] = { + "summary": "Fast-path style contract.", + "medium": preferences.get("style_description"), + "palette": preferences.get("preferred_palette", []), + "viewpoint": "front-facing academic diagram", + "line_and_shadow": "crisp thin lines with restrained shadows", + "visual_density": preferences.get("visual_density", "high"), + "background": "clean light background", + "text_policy": "background regions only; exact labels and arrows are supplied by overlay_spec.json", + "positive_reference_rules": [], + "negative_reference_rules": preferences.get("avoid", []), + } + return result + + def _first_sentence(text: str) -> str: sentences = [item.strip() for item in re.split(r"(?<=[.!?。!?])\s+", text) if len(item.strip()) > 20] return sentences[0][:500] if sentences else text[:500].strip() +def _plausible_component_heading(value: str) -> bool: + text = re.sub(r"^\d+(?:\.\d+)*\s*", "", str(value or "")).strip() + low = text.casefold() + if not 3 <= len(text) <= 72 or len(text.split()) > 9: + return False + if any(term in low for term in ("http", "www.", "acknowledg", "reference", "appendix", "picture credit", "et al.", "university", "institute")): + return False + if re.search(r"[π∑=·]|\([^)]*\d{4}[^)]*\)|\b(?:fig(?:ure)?|table)\s*\d", text, re.IGNORECASE): + return False + if text.count(",") >= 2 or text.count(":") >= 1: + return False + component_terms = ("architecture", "pipeline", "framework", "module", "encoder", "decoder", "backbone", "parser", "reasoner", "renderer", "retrieval", "generation", "refinement", "training", "inference", "data engine", "model") + return any(term in low for term in component_terms) + + def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = None) -> dict: if paper_review and paper_review.get("modules"): review_modules = [item for item in (list(paper_review.get("modules", [])) + list(paper_review.get("research_objects", [])) + list(paper_review.get("concepts", []))) if item.get("evidence_ids")][:16] @@ -194,7 +349,7 @@ def _heuristic_plan(parsed: dict, preferences: dict, paper_review: dict | None = first = evidence[0] title = next((line.strip() for line in first["text"].splitlines() if 10 <= len(line.strip()) <= 180), parsed["source_name"]) generic = {"abstract", "introduction", "related work", "method", "methods", "experiments", "results", "conclusion", "references"} - names = [item for item in headings if item.lower() not in generic][:6] + names = [item for item in headings if item.lower() not in generic and _plausible_component_heading(item)][:6] if len(names) < 3: names = ["Research Input", "Core Method", "Research Output"] modules = [] @@ -296,13 +451,44 @@ def normalize_plan(raw: dict, preferences: dict) -> dict: return result +def plan_fast_paper_contract( + parsed: dict, + preferences: dict, + mode: str = "vlm", + model: str | None = None, + timeout_seconds: int = 45, + retries: int = 2, + evidence_max_chars: int = 36000, +) -> tuple[dict, dict]: + prompt = _fast_planner_prompt(parsed, preferences, evidence_max_chars=evidence_max_chars) + metadata: dict[str, Any] = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} + if mode == "vlm" and vlm_credentials_available(): + resolved = resolve_vlm_model("RFS_FAST_FRAMEWORK_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) + try: + raw = call_vlm_json( + prompt, + [], + model=resolved, + timeout=max(10, int(timeout_seconds)), + retries=max(0, int(retries)), + call_metadata=metadata["provider"], + ) + metadata["model"] = resolved + return _compile_fast_plan(raw, preferences), metadata + except Exception as exc: + metadata.update({"mode": "heuristic_after_vlm_failure", "warning": str(exc), "model": resolved}) + elif mode == "vlm": + metadata.update({"mode": "heuristic_without_credentials", "warning": "API_BASE and API_KEY/GEMINI_API_KEY are required for VLM planning"}) + return normalize_plan(_heuristic_plan(parsed, preferences), preferences), metadata + + def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1, evidence_max_chars: int = 58000) -> tuple[dict, dict]: prompt = _planner_prompt(parsed, preferences, paper_review=paper_review, evidence_max_chars=evidence_max_chars) - metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} + metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries))) + raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"]) metadata["model"] = resolved return normalize_plan(raw, preferences), metadata except Exception as exc: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 93b6454..bff2ff8 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -11,7 +11,9 @@ from ..utils import ensure_dir, read_json, write_json, write_text from .analyzer import paper_markdown, parse_paper -from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_paper_image, validate_plan_grounding +from .contract_completion import augment_contract_from_evidence +from .document_cache import read_document_cache, write_document_cache +from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_fast_paper_contract, plan_paper_image, validate_plan_grounding from .review import build_paper_review, detect_domain_profile, validate_review_coverage @@ -130,7 +132,9 @@ def _infer_topology(spec: dict[str, Any]) -> str: inputs = [item for item in spec.get("inputs", []) if isinstance(item, dict)] labels = " ".join(_item_label(item).casefold() for item in inputs) modalities = sum(term in labels for term in ("image", "text", "audio", "video", "depth", "thermal", "imu")) - if len(inputs) >= 3 or modalities >= 3: + modules = [item for item in spec.get("modules", []) if isinstance(item, dict)] + encoder_count = sum("encoder" in _item_label(item).casefold() for item in modules) + if modalities >= 3 or (modalities >= 2 and encoder_count >= 2): return "multimodal" outgoing: dict[str, int] = {} for relation in relations: @@ -221,101 +225,6 @@ def _complete_from_overview_caption(spec: dict[str, Any], parsed: dict[str, Any] relations.append({"source": pair[0], "target": pair[1], "type": "data_generation", "label": "", "evidence_ids": stage_evidence + evidence_ids}) break - def find_evidence(*terms: str) -> list[str]: - found = next((item for item in evidence if all(term.casefold() in str(item.get("text") or "").casefold() for term in terms)), None) - return [found["id"]] if found else evidence_ids - - def ensure_entity(field: str, label: str, role: str, item_evidence: list[str]) -> dict[str, Any]: - normalized = _normalized_label(label) - existing = next((item for item in spec.get(field, []) if isinstance(item, dict) and normalized and normalized == _normalized_label(_item_label(item))), None) - if existing: - return existing - item = {"id": _stable_id(field.rstrip("s"), label, len(spec.get(field, []))), "name": label, "role": role, "evidence_ids": item_evidence} - spec.setdefault(field, []).append(item) - return item - - def ensure_relation(source: dict[str, Any], target: dict[str, Any], relation_type: str, relation_evidence: list[str]) -> None: - pair = (str(source.get("id")), str(target.get("id"))) - if pair not in pairs: - relations.append({"source": pair[0], "target": pair[1], "type": relation_type, "label": "", "evidence_ids": relation_evidence}) - pairs.add(pair) - - low_caption = caption.casefold() - if "patch" in low_caption and "transformer encoder" in low_caption: - image = ensure_entity("inputs", "Input Image", "input", evidence_ids) - patches = ensure_entity("modules", "Image Patches", "module", evidence_ids) - projection = ensure_entity("modules", "Linear Projection of Flattened Patches", "module", evidence_ids) - position = ensure_entity("modules", "Position Embedding", "conditioning", evidence_ids) - token = ensure_entity("modules", "Class Token", "conditioning", evidence_ids) - encoder = ensure_entity("modules", "Transformer Encoder", "module", evidence_ids) - mlp_evidence = find_evidence("mlp", "head") - mlp = ensure_entity("modules", "MLP Head", "module", mlp_evidence) - prediction = ensure_entity("outputs", "Class Prediction", "output", mlp_evidence) - for source, target, relation_type in [ - (image, patches, "data_flow"), (patches, projection, "data_flow"), (projection, encoder, "data_flow"), - (position, encoder, "conditioning"), (token, encoder, "conditioning"), (encoder, mlp, "data_flow"), (mlp, prediction, "data_flow"), - ]: - ensure_relation(source, target, relation_type, list(dict.fromkeys(source.get("evidence_ids", []) + target.get("evidence_ids", [])))) - - modality_names = [term for term in ("image", "text", "audio", "depth", "thermal", "imu") if re.search(rf"\b{term}(?:s)?\b", low_caption)] - if len(modality_names) >= 4 and ("common embedding" in low_caption or "joint embedding" in low_caption): - modality_inputs = [ensure_entity("inputs", term.upper() if term == "imu" else term.title(), "input modality", evidence_ids) for term in modality_names] - encoders = ensure_entity("modules", "Modality Encoders", "module group", find_evidence("encoder")) - joint = ensure_entity("modules", "Joint Embedding Space", "shared representation", evidence_ids) - emergent = ensure_entity("outputs", "Emergent Cross-modal Alignment", "output", evidence_ids) - for item in modality_inputs: - ensure_relation(item, encoders, "encoding", list(dict.fromkeys(item.get("evidence_ids", []) + encoders.get("evidence_ids", [])))) - ensure_relation(encoders, joint, "alignment", list(dict.fromkeys(encoders.get("evidence_ids", []) + joint.get("evidence_ids", [])))) - ensure_relation(joint, emergent, "enables", evidence_ids) - - if "feedback" in low_caption and ("refine" in low_caption or "refines" in low_caption) and ("generating an output" in low_caption or "initial output" in low_caption): - task = ensure_entity("inputs", "Input", "input", evidence_ids) - generate = ensure_entity("modules", "Generate", "generator", evidence_ids) - initial = ensure_entity("modules", "Initial Output", "artifact", evidence_ids) - feedback = ensure_entity("modules", "Feedback", "feedback provider", evidence_ids) - feedback_text = ensure_entity("modules", "Self-Feedback", "feedback artifact", evidence_ids) - refine = ensure_entity("modules", "Refine", "refiner", evidence_ids) - refined = ensure_entity("outputs", "Refined Output", "output", evidence_ids) - for source, target, relation_type in [ - (task, generate, "data_flow"), (generate, initial, "data_flow"), (initial, feedback, "evaluation"), - (feedback, feedback_text, "feedback"), (initial, refine, "revision_input"), (feedback_text, refine, "feedback"), - (refine, refined, "data_flow"), (refined, feedback, "feedback_loop"), - ]: - ensure_relation(source, target, relation_type, evidence_ids) - - if re.search(r"mask\s*-?r-?cnn", low_caption, re.IGNORECASE) and "framework" in low_caption: - input_image = ensure_entity("inputs", "Input Image", "input", find_evidence("image")) - backbone = ensure_entity("modules", "Backbone", "feature extractor", find_evidence("backbone")) - proposals = ensure_entity("modules", "Region Proposals", "proposal artifact", find_evidence("region proposal")) - roi = ensure_entity("modules", "RoIAlign", "alignment", find_evidence("roialign")) - classification = ensure_entity("outputs", "Classification", "output head", find_evidence("class")) - box = ensure_entity("outputs", "Bounding-box Regression", "output head", find_evidence("bounding box")) - mask = ensure_entity("outputs", "Mask Branch", "output head", find_evidence("mask branch")) - for source, target, relation_type in [ - (input_image, backbone, "data_flow"), (backbone, proposals, "data_flow"), (proposals, roi, "data_flow"), - (backbone, roi, "feature_flow"), (roi, classification, "branch"), (roi, box, "branch"), (roi, mask, "branch"), - ]: - ensure_relation(source, target, relation_type, list(dict.fromkeys(source.get("evidence_ids", []) + target.get("evidence_ids", [])))) - - sam_overview = next((item for item in figures if re.search(r"segment anything model.*overview", str(item.get("caption") or ""), re.IGNORECASE)), None) - if sam_overview: - sam_caption = str(sam_overview.get("caption") or "") - sam_evidence = next((item["id"] for item in evidence if item.get("kind") == "caption" and item.get("page") == sam_overview.get("page") and str(item.get("text") or "").startswith(sam_caption[:60])), None) - sam_ids = [sam_evidence] if sam_evidence else evidence_ids - image = ensure_entity("inputs", "Image", "input", sam_ids) - prompt = ensure_entity("inputs", "Prompt", "input", sam_ids) - valid_mask = ensure_entity("outputs", "Valid Segmentation Mask", "output", sam_ids) - image_encoder = ensure_entity("modules", "Image Encoder", "module", find_evidence("image encoder")) - prompt_encoder = ensure_entity("modules", "Prompt Encoder", "module", find_evidence("prompt encoder")) - mask_decoder = ensure_entity("modules", "Mask Decoder", "module", find_evidence("mask decoder")) - for source, target, relation_type in [ - (image, image_encoder, "data_flow"), (prompt, prompt_encoder, "data_flow"), - (image_encoder, mask_decoder, "feature_flow"), (prompt_encoder, mask_decoder, "conditioning"), - (mask_decoder, valid_mask, "data_flow"), - ]: - ensure_relation(source, target, relation_type, list(dict.fromkeys(source.get("evidence_ids", []) + target.get("evidence_ids", [])))) - - def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} @@ -334,6 +243,23 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d item.setdefault("id", _item_id(item, field, index)) normalized_items.append(item) spec[field] = normalized_items + completion_report = augment_contract_from_evidence(spec, parsed) + plan["contract_completion_report"] = completion_report + if len(completion_report.get("added_entities", [])) >= 3: + fallback_ids = { + str(item.get("id")) + for item in (spec.get("modules", []) if isinstance(spec.get("modules"), list) else []) + if isinstance(item, dict) and str(item.get("role") or "") == "paper-derived stage requiring VLM verification" + } + if fallback_ids: + spec["modules"] = [item for item in spec.get("modules", []) if not (isinstance(item, dict) and str(item.get("id")) in fallback_ids)] + spec["relations"] = [ + item for item in spec.get("relations", []) if not ( + isinstance(item, dict) + and (str(item.get("source")) in fallback_ids or str(item.get("target")) in fallback_ids) + ) + ] + completion_report["removed_fallback_entities"] = sorted(fallback_ids) _complete_from_overview_caption(spec, parsed) modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] encoder_modules = [item for item in modules if isinstance(item, dict) and "encoder" in _item_label(item).casefold() and "modality encoder" not in _item_label(item).casefold()] @@ -627,7 +553,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 3, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 4, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -674,7 +600,13 @@ def prepare_paper_figure_contract( deadline = max(30, int(deadline_seconds)) deadline_at = started + deadline - parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter) + parsed = None if ocr_adapter else read_document_cache(archived_paper, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + document_cache_hit = parsed is not None + if parsed is None: + parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter) + if not ocr_adapter: + write_document_cache(archived_paper, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + document_preparation_seconds = round(time.monotonic() - started, 3) write_json(root / "document_model.json", parsed) write_json(root / "extraction_report.json", parsed["extraction_report"]) write_json(root / "document_index.json", parsed["document_index"]) @@ -706,15 +638,13 @@ def prepare_paper_figure_contract( plan = read_json(cache_path) planner_metadata = {"requested_mode": "vlm", "mode": "vlm", "model": planner_model, "warning": None, "prompt": "", "cached": True} else: - plan, planner_metadata = plan_paper_image( + plan, planner_metadata = plan_fast_paper_contract( parsed, preferences, mode=effective_mode, model=planner_model, - reference_images=archived_positive + archived_negative, - paper_review=None, - timeout_seconds=min(45, remaining), - retries=1, + timeout_seconds=min(35, remaining), + retries=2, evidence_max_chars=36000, ) if planner_metadata.get("mode") == "vlm": @@ -764,6 +694,8 @@ def prepare_paper_figure_contract( write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) expand_plan_evidence(plan, paper_review, parsed) normalize_figure_contract(plan, parsed) + completion_report = plan.get("contract_completion_report") if isinstance(plan.get("contract_completion_report"), dict) else {} + write_json(root / "contract_completion_report.json", completion_report) for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): write_json(root / f"{name}.json", plan[name]) planning_validation = validate_plan_grounding(plan, parsed) @@ -787,7 +719,22 @@ def prepare_paper_figure_contract( has_warning = bool(planner_metadata.get("warning") or review_metadata.get("warning") or parsed["extraction_report"].get("status") == "warning") status = "complete" if production_ready and not has_warning else "completed_with_warnings" contract_source = "cache" if planner_metadata.get("cached") else "vlm" if planner_metadata.get("mode") == "vlm" else "vlm_review_deterministic_compile" if planner_metadata.get("mode") == "review_grounded_vlm" else "deterministic_evidence_rules" - extraction_seconds = float(parsed["extraction_report"].get("elapsed_seconds") or 0.0) + source_extraction_seconds = float(parsed["extraction_report"].get("elapsed_seconds") or 0.0) + extraction_seconds = 0.0 if document_cache_hit else source_extraction_seconds + planner_provider = planner_metadata.get("provider") if isinstance(planner_metadata.get("provider"), dict) else {} + review_provider = review_metadata.get("provider") if isinstance(review_metadata.get("provider"), dict) else {} + provider_calls = [item for item in (planner_provider, review_provider) if item.get("attempts")] + provider_summary = { + "planner": planner_provider, + "review": review_provider, + "attempts": sum(int(item.get("attempts") or 0) for item in provider_calls), + "retries_used": sum(int(item.get("retries_used") or 0) for item in provider_calls), + "success": any(bool(item.get("success")) for item in provider_calls) if provider_calls else None, + "successful_call_count": sum(bool(item.get("success")) for item in provider_calls), + "call_count": len(provider_calls), + "elapsed_seconds": round(sum(float(item.get("elapsed_seconds") or 0.0) for item in provider_calls), 3), + "failure_categories": list(dict.fromkeys(category for item in provider_calls for category in (item.get("failure_categories") or []))), + } result = { "summary": "Fast paper-to-framework contract preparation completed.", "ok": bool(planning_validation.get("ok")), @@ -804,14 +751,27 @@ def prepare_paper_figure_contract( "paper_review_mode": review_metadata.get("mode"), "contract_source": contract_source, "cache_hit": bool(planner_metadata.get("cached")), + "document_cache_hit": document_cache_hit, + "provider": provider_summary, + "contract_completion": { + "overview_term_coverage": completion_report.get("overview_term_coverage"), + "added_entity_count": len(completion_report.get("added_entities", [])), + "added_relation_count": len(completion_report.get("added_relations", [])), + }, "planner_warning": planner_metadata.get("warning"), "review_warning": review_metadata.get("warning"), "topology": plan["figure_specification"].get("topology"), "module_count": len(plan["figure_specification"].get("modules", [])), "relation_count": len(plan["figure_specification"].get("relations", [])), - "stage_timings": {"document_extraction_seconds": extraction_seconds, "semantic_compilation_seconds": round(max(0.0, elapsed - extraction_seconds), 3), "total_seconds": elapsed}, + "stage_timings": { + "document_preparation_seconds": document_preparation_seconds, + "document_extraction_seconds": extraction_seconds, + "cached_source_extraction_seconds": source_extraction_seconds if document_cache_hit else None, + "semantic_compilation_seconds": round(max(0.0, elapsed - document_preparation_seconds), 3), + "total_seconds": elapsed, + }, "uncertainties": plan["figure_specification"].get("uncertainties", []), - "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], + "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "contract_completion_report.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], } write_json(root / "run_report.json", result) return { diff --git a/rfs/paper_to_image/review.py b/rfs/paper_to_image/review.py index 225d56a..3aef6bb 100644 --- a/rfs/paper_to_image/review.py +++ b/rfs/paper_to_image/review.py @@ -330,11 +330,11 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1, evidence_max_chars: int = 65000) -> tuple[dict, dict]: prompt = _review_prompt(parsed, domain_profile, evidence_max_chars=evidence_max_chars) - metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt} + metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_REVIEW_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries))) + raw = call_vlm_json(prompt, [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"]) metadata["model"] = resolved review = normalize_review(raw, domain_profile) _expand_evidence(review, parsed) diff --git a/rfs/reference_text_extractor.py b/rfs/reference_text_extractor.py index 25b5dbc..48bd7b2 100644 --- a/rfs/reference_text_extractor.py +++ b/rfs/reference_text_extractor.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +import os from pathlib import Path from typing import Any, Callable @@ -8,6 +9,8 @@ OCR_LOW_CONFIDENCE = 0.65 +_EASYOCR_READERS: dict[tuple[str, ...], Any] = {} +_RAPIDOCR_ENGINE: Any | None = None def _round4(value: float) -> float: @@ -160,7 +163,16 @@ def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: import easyocr # type: ignore lang_map = {"en_ch": ["ch_sim", "en"], "ch": ["ch_sim"], "en": ["en"]} - reader = easyocr.Reader(lang_map.get(str(lang), ["ch_sim", "en"]), gpu=False) + languages = tuple(lang_map.get(str(lang), ["ch_sim", "en"])) + reader = _EASYOCR_READERS.get(languages) + if reader is None: + allow_download = str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"} + try: + reader = easyocr.Reader(list(languages), gpu=False, download_enabled=allow_download, verbose=False) + except Exception as exc: + hint = " Set RFS_OCR_ALLOW_DOWNLOAD=1 for an explicit one-time model download." if not allow_download else "" + raise RuntimeError(f"EasyOCR model is not ready for languages {list(languages)}.{hint} Original error: {exc}") from exc + _EASYOCR_READERS[languages] = reader records = [] for quad, text, confidence in reader.readtext(str(image_path)): points = _normalize_quad(quad) @@ -169,6 +181,25 @@ def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: return records +def run_rapidocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: + del lang + global _RAPIDOCR_ENGINE + if _RAPIDOCR_ENGINE is None: + from rapidocr_onnxruntime import RapidOCR # type: ignore + + _RAPIDOCR_ENGINE = RapidOCR(intra_op_num_threads=1, inter_op_num_threads=1, det_limit_side_len=512) + result, _timings = _RAPIDOCR_ENGINE(str(image_path)) + records = [] + for item in result or []: + if not isinstance(item, (list, tuple)) or len(item) < 3: + continue + quad = _normalize_quad(item[0]) + text = str(item[1] or "").strip() + if quad and text: + records.append({"text": text, "confidence": float(item[2] or 0.0), "quad": quad}) + return records + + def _record_to_region(record: dict[str, Any], image: Image.Image, program: dict, index: int, canvas_height_in: float, engine: str) -> dict: width, height = image.size xs = [point[0] for point in record["quad"]] @@ -237,6 +268,8 @@ def extract_reference_text( try: if ocr_adapter: raw_records = ocr_adapter(reference_path, lang) + elif requested_engine == "rapidocr": + raw_records = run_rapidocr(reference_path, lang) elif requested_engine == "easyocr": raw_records = run_easyocr(reference_path, lang) else: diff --git a/rfs/vlm_client.py b/rfs/vlm_client.py index b29f1f0..5c5d792 100644 --- a/rfs/vlm_client.py +++ b/rfs/vlm_client.py @@ -42,7 +42,30 @@ def resolve_vlm_model(*env_names: str, explicit_model: str | None = None) -> str return os.getenv("MODEL_VLM", "").strip() or DEFAULT_VLM_MODEL -def call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None = None, timeout: int = 180, retries: int = 1) -> dict: +def _error_category(exc: Exception) -> str: + name = type(exc).__name__.casefold() + message = str(exc).casefold() + if "timeout" in name or "timeout" in message: + return "timeout" + if "ssl" in name or "ssl" in message or "eof" in message: + return "tls" + if "json" in name or "json" in message: + return "invalid_json" + if "connection" in name or "connection" in message: + return "connection" + if "http" in name or "status" in message: + return "http" + return "provider" + + +def call_vlm_json( + prompt: str, + image_paths: list[str | Path], + model: str | None = None, + timeout: int = 180, + retries: int = 1, + call_metadata: dict[str, Any] | None = None, +) -> dict: api_base = os.getenv("API_BASE", "").rstrip("/") api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") if not api_base or not api_key: @@ -61,8 +84,22 @@ def call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None "messages": [{"role": "user", "content": content}], "temperature": 0.1, } + started = time.monotonic() last_error: Exception | None = None - for attempt in range(max(1, int(retries) + 1)): + failures: list[dict[str, Any]] = [] + attempts = max(1, int(retries) + 1) + if call_metadata is not None: + call_metadata.update({ + "attempts": 0, + "retries_allowed": max(0, int(retries)), + "retries_used": 0, + "success": False, + "elapsed_seconds": 0.0, + "failure_categories": [], + }) + for attempt in range(attempts): + if call_metadata is not None: + call_metadata["attempts"] = attempt + 1 try: response = requests.post( f"{api_base}/chat/completions", @@ -72,9 +109,25 @@ def call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None ) response.raise_for_status() content_text = response.json()["choices"][0]["message"]["content"] - return extract_json(content_text) + result = extract_json(content_text) + if call_metadata is not None: + call_metadata.update({ + "success": True, + "retries_used": attempt, + "elapsed_seconds": round(time.monotonic() - started, 3), + "failure_categories": list(dict.fromkeys(item["category"] for item in failures)), + }) + return result except Exception as exc: last_error = exc - if attempt < max(1, int(retries) + 1) - 1: - time.sleep(1.5) + failures.append({"attempt": attempt + 1, "category": _error_category(exc)}) + if attempt < attempts - 1: + time.sleep(min(2.0, 0.75 * (2 ** attempt))) + if call_metadata is not None: + call_metadata.update({ + "success": False, + "retries_used": max(0, attempts - 1), + "elapsed_seconds": round(time.monotonic() - started, 3), + "failure_categories": list(dict.fromkeys(item["category"] for item in failures)), + }) raise last_error or RuntimeError("VLM call failed") diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 08b9810..e428d46 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -10,7 +10,7 @@ from PIL import Image -from rfs.cli import build_parser +from rfs.cli import _doctor, build_parser from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import review_candidate from rfs.paper_to_image.generator import generate_and_select @@ -127,6 +127,13 @@ def test_inspect_pdf_cli_defaults(self): self.assertEqual(args.deadline, 180) self.assertEqual(args.ocr_engine, "auto") + def test_doctor_reports_easyocr_model_readiness(self): + report = _doctor() + models = report["pdf_tools"]["easyocr"]["models"] + self.assertIn("en_ready", models) + self.assertIn("en_ch_ready", models) + self.assertIn("allow_download", models) + def test_fast_framework_prompt_writes_contract_without_generation(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -166,6 +173,12 @@ def test_benchmark_fast_cli_defaults(self): self.assertEqual(args.deadline, 180) self.assertEqual(args.planner_mode, "vlm") + def test_benchmark_fast_suite_cli_accepts_repeatable_case_ids(self): + parser = build_parser() + args = parser.parse_args(["benchmark", "fast-suite", "--root", "benchmarks", "--out", "output/suite", "--case-id", "101_vit_linear", "--case-id", "106_detr_set_prediction"]) + self.assertEqual(args.benchmark_action, "fast-suite") + self.assertEqual(args.case_id, ["101_vit_linear", "106_detr_set_prediction"]) + def test_domain_profiles_detect_method_and_survey(self): method = {"evidence": [{"text": "A neural model is optimized with a training loss and used during inference."}]} survey = {"evidence": [{"text": "This systematic survey presents a taxonomy and review of the landscape."}]} diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index ffec8cf..6f97e3a 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -11,6 +11,7 @@ _score_planning_contract, list_benchmark_cases, run_fast_benchmark_case, + run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case, ) @@ -72,6 +73,22 @@ def test_fast_benchmark_command_runs_without_image_generation(self): self.assertTrue((Path(tmp) / "fast_benchmark_result.json").exists()) self.assertFalse((Path(tmp) / "selected_image.png").exists()) + def test_fast_suite_aggregates_reliability_and_timing_metrics(self): + with tempfile.TemporaryDirectory() as tmp: + result = run_fast_benchmark_suite( + ROOT / "benchmarks", + tmp, + case_ids=["001_linear_pipeline"], + planner_mode="heuristic", + ocr_engine="off", + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["aggregate"]["case_count"], 1) + self.assertIn("mean_total_seconds", result["aggregate"]) + self.assertIn("provider_success_rate", result["aggregate"]) + self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) + def test_paper_to_image_score_uses_hard_scientific_gates(self): case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 804bcbe..4da876c 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -1,10 +1,11 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch import fitz -from rfs.paper_to_image.analyzer import parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, parse_paper from rfs.paper_to_image.inspection import inspect_paper @@ -88,6 +89,42 @@ def test_low_text_page_uses_local_ocr_adapter(self): self.assertTrue(parsed["pages"][0]["used_ocr"]) self.assertIn("Image Encoder", parsed["pages"][0]["text"]) + def test_empty_ocr_result_returns_failed_document_model_instead_of_raising(self): + path = self._pdf(lambda _page: None) + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=lambda _image, _lang: []) + + self.assertEqual(parsed["extraction_report"]["status"], "fail") + self.assertEqual(parsed["evidence"], []) + self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") + + def test_ocr_lines_are_grouped_into_complete_figure_caption(self): + path = self._pdf(lambda _page: None) + records = [ + {"text": "Figure 1: Overview of the proposed framework.", "confidence": 0.95, "quad": [[40, 100], [560, 100], [560, 125], [40, 125]]}, + {"text": "The image encoder feeds a transformer decoder.", "confidence": 0.94, "quad": [[40, 130], [560, 130], [560, 155], [40, 155]]}, + {"text": "1 Introduction", "confidence": 0.98, "quad": [[40, 210], [250, 210], [250, 240], [40, 240]]}, + {"text": "A sufficiently long scientific paragraph provides reliable evidence for the method.", "confidence": 0.96, "quad": [[40, 250], [560, 250], [560, 275], [40, 275]]}, + ] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=lambda _image, _lang: records) + + self.assertEqual(parsed["document_index"]["figures"][0]["caption"], "Figure 1: Overview of the proposed framework. The image encoder feeds a transformer decoder.") + self.assertGreater(parsed["extraction_report"]["mean_ocr_confidence"], 0.9) + + def test_rapidocr_caption_group_survives_sentence_boundaries(self): + lines = [ + {"bbox": [40, 100, 560, 120], "text": "Fig. 1: Overview of the model.", "confidence": 0.98}, + {"bbox": [40, 124, 560, 144], "text": "The encoder produces features.", "confidence": 0.98}, + {"bbox": [40, 148, 560, 168], "text": "A decoder predicts the output.", "confidence": 0.98}, + {"bbox": [40, 210, 560, 230], "text": "We next describe the training objective.", "confidence": 0.98}, + ] + + grouped = _assign_ocr_parent_blocks(lines, 600) + + self.assertEqual({item["parent_block"] for item in grouped[:3]}, {1}) + self.assertNotEqual(grouped[2]["parent_block"], grouped[3]["parent_block"]) + def test_inspection_reuses_matching_document_cache(self): path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "Abstract Method A sufficiently long scientific paragraph for deterministic PDF inspection.", fontsize=10)) with tempfile.TemporaryDirectory() as temp: @@ -97,6 +134,15 @@ def test_inspection_reuses_matching_document_cache(self): self.assertTrue(first["document_model"]) self.assertEqual(second["status"], "cached") + def test_global_document_cache_reuses_parse_across_output_directories(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 220), "Abstract Method A sufficiently long scientific paragraph for global cache validation. The document contains repeated evidence, page-aware terminology, a complete method description, and enough characters to satisfy the readable-page quality gate.", fontsize=10)) + with tempfile.TemporaryDirectory() as cache, tempfile.TemporaryDirectory() as first_out, tempfile.TemporaryDirectory() as second_out, patch.dict("os.environ", {"RFS_CACHE_DIR": cache}, clear=False): + first = inspect_paper(path, first_out, ocr_engine="off") + second = inspect_paper(path, second_out, ocr_engine="off") + + self.assertFalse(first["document_cache_hit"]) + self.assertTrue(second["document_cache_hit"]) + def test_evidence_ids_are_stable_across_repeated_parses(self): path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "1 Method\nA stable block of scientific evidence for repeated parsing.", fontsize=10)) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 8e86c0b..1ea9100 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -168,6 +168,78 @@ def test_overview_caption_and_stage_list_complete_missing_panels(self): self.assertTrue(any(item.get("name") == "SA-1B" for item in spec["outputs"])) self.assertTrue(any("stage_assisted_manual" in source and "stage_semi_automatic" in target for source, target in pairs)) + def test_generic_completion_recovers_detr_without_bibliography_noise(self): + captions = [ + "Fig. 1: DETR directly predicts the final set of detections by combining a common CNN with a transformer architecture. During training, bipartite matching uniquely assigns predictions with ground truth boxes and class predictions.", + "Fig. 2: DETR uses a conventional CNN backbone for an input image, supplements features with a positional encoding, passes them into a transformer encoder, and uses object queries in a transformer decoder. A feed forward network predicts a class and bounding box.", + ] + parsed = { + "page_count": 12, + "document_index": {"figures": [{"page": index + 1, "caption": text} for index, text in enumerate(captions)]}, + "evidence": [ + {"id": f"E000{index + 1}", "page": index + 1, "kind": "caption", "text": text, "confidence": 1.0} + for index, text in enumerate(captions) + ] + [{"id": "E0099", "page": 11, "kind": "heading", "text": "Learning non-maximum suppression. In: ECCV", "section_hint": "References", "confidence": 1.0}], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + labels = {_item.get("name") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + + self.assertIn("Object Queries", labels) + self.assertIn("Bipartite Matching", labels) + self.assertIn("Bounding Box Predictions", labels) + self.assertNotIn("Non-Maximum Suppression", labels) + self.assertIn((ids["Object Queries"], ids["Transformer Decoder"]), pairs) + self.assertIn((ids["Feed Forward Network"], ids["Bounding Box Predictions"]), pairs) + + def test_generic_completion_keeps_clip_modalities_and_encoders_distinct(self): + caption = "Figure 1. Summary of our approach. CLIP jointly trains an image encoder and a text encoder to predict the correct pairings of a batch of (image, text) training examples. At test time the text encoder synthesizes a zero-shot linear classifier by embedding the names or descriptions of the target classes." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "The image encoder and text encoder maximize similarity of image and text embeddings using a contrastive objective.", "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [{"id": "fallback_contrastive", "name": "Contrastive Pre-training", "role": "paper-derived stage requiring VLM verification", "evidence_ids": ["E0002"]}], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertNotEqual(ids["Text"], ids["Text Encoder"]) + self.assertIn((ids["Text"], ids["Text Encoder"]), pairs) + self.assertIn((ids["Class Descriptions"], ids["Text Encoder"]), pairs) + self.assertIn((ids["Text Encoder"], ids["Text Embeddings"]), pairs) + self.assertTrue(any(item.get("id") == "fallback_contrastive" for item in spec["modules"])) + self.assertEqual(spec["topology"], "multimodal") + + def test_generic_completion_recovers_nerf_branch_without_false_multimodal_topology(self): + caption = "Fig. 2: An overview of our neural radiance field and differentiable rendering procedure. We synthesize images by sampling 5D coordinates along camera rays, feeding locations and viewing direction into an MLP to produce a color and volume density, and using volume rendering to composite these values into an image." + positional = "Fig. 4: Our full model passes input coordinates through a high-frequency positional encoding." + parsed = { + "page_count": 10, + "document_index": {"figures": [{"page": 2, "caption": caption}, {"page": 4, "caption": positional}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Method", "confidence": 1.0}, + {"id": "E0002", "page": 4, "kind": "caption", "text": positional, "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn((ids["Viewing Direction"], ids["MLP"]), pairs) + self.assertIn((ids["MLP"], ids["Color"]), pairs) + self.assertIn((ids["Volume Rendering"], ids["Rendered Image"]), pairs) + self.assertEqual(spec["topology"], "branch") + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_vlm_client.py b/tests/unit/test_vlm_client.py new file mode 100644 index 0000000..bfa75e4 --- /dev/null +++ b/tests/unit/test_vlm_client.py @@ -0,0 +1,34 @@ +import unittest +from unittest.mock import patch + +import requests + +from rfs.vlm_client import call_vlm_json + + +class _Response: + def raise_for_status(self): + return None + + def json(self): + return {"choices": [{"message": {"content": '{"ok": true}'}}]} + + +class VlmClientTests(unittest.TestCase): + @patch("rfs.vlm_client.time.sleep", return_value=None) + @patch("rfs.vlm_client.requests.post", side_effect=[requests.exceptions.SSLError("EOF during TLS"), _Response()]) + def test_retry_metadata_records_recovery_without_secrets(self, _post, _sleep): + metadata = {} + with patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-token"}, clear=False): + result = call_vlm_json("Return JSON", [], model="test-model", timeout=10, retries=1, call_metadata=metadata) + + self.assertTrue(result["ok"]) + self.assertEqual(metadata["attempts"], 2) + self.assertEqual(metadata["retries_used"], 1) + self.assertTrue(metadata["success"]) + self.assertEqual(metadata["failure_categories"], ["tls"]) + self.assertNotIn("secret-token", str(metadata)) + + +if __name__ == "__main__": + unittest.main() From 13094554452c0c33bdda2cf77869230db1a70a62 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 01:20:16 +0800 Subject: [PATCH 22/74] Document fast paper reliability workflow --- README.md | 10 +++++++++- benchmarks/README.md | 5 +++++ docs/paper-to-image.md | 13 +++++++++++-- skills/research-figure-studio/SKILL.md | 13 ++++++++++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 832ecb4..42e61eb 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,15 @@ rfs fast-framework-prompt ` --json ``` -This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. It uses one compact VLM request by default, falls back to evidence-driven topology rules, and caches successful contracts by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. `rfs inspect-pdf` runs only the local extraction and quality checks. +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. + +Use `rfs inspect-pdf` for parser-only diagnostics. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package and model readiness. + +Run a repeatable fast suite with provider, cache, recall, and timing aggregation: + +```powershell +rfs benchmark fast-suite --root benchmarks --out output\benchmarks\fast-suite --planner-mode heuristic --json +``` Use `paper-to-image` when the required endpoint is a generated raster framework figure rather than an editable PowerPoint file. The production route performs a diff --git a/benchmarks/README.md b/benchmarks/README.md index c29b74e..d817611 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -23,6 +23,7 @@ rfs benchmark list --root benchmarks --json rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipeline --json rfs benchmark fetch --case benchmarks/paper-to-image/cases/101_vit_linear --json rfs benchmark fast --case benchmarks/paper-to-image/cases/101_vit_linear --out output/benchmarks/vit_fast --planner-mode heuristic --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast_suite --planner-mode heuristic --json rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json ``` @@ -35,6 +36,10 @@ Generated runs and reports belong under `output/benchmarks/` and are not source Real-paper cases commit `source.json` and human-authored Ground Truth, while `benchmark fetch` downloads the PDF into an ignored local `inputs/` directory. This keeps the public repository reproducible without redistributing publisher files. +The committed unseen-generalization set includes DETR, CLIP, and NeRF in addition to ViT, Mask R-CNN, Self-Refine, ImageBind, and SAM. Local image-only scan regressions remain under ignored `tmp/pdfs/` and `output/pdf/`; do not commit publisher PDFs or generated OCR artifacts. + +`benchmark fast-suite` writes `fast_suite_report.json` with per-case results and aggregate entity/relation recall, forbidden content, cache hit rates, provider success/retry counts, failure categories, and stage timings. + ## Evaluation tiers - Offline contract tier: placeholder assets, deterministic validation, CI-safe. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index bd546b7..5360e61 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -8,9 +8,18 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --json ``` -The fast path stops before image generation. It uses structured block-level PDF extraction, complete Figure 1/overview captions, stable page/content-hash evidence IDs, one compact VLM contract call, deterministic evidence rules, and a local SHA-256 contract cache. A cached paper normally completes in seconds. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. +The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, and parser agreement without model calls. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, and parser agreement without model calls. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. + +Run all or selected paper benchmarks with aggregate reliability metrics: + +```powershell +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast-suite --planner-mode heuristic --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case-id 106_detr_set_prediction --case-id 107_clip_contrastive --json +``` + +`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, and parser/semantic/total timings. ## Production Command diff --git a/skills/research-figure-studio/SKILL.md b/skills/research-figure-studio/SKILL.md index 96df4bc..8ad9cba 100644 --- a/skills/research-figure-studio/SKILL.md +++ b/skills/research-figure-studio/SKILL.md @@ -27,6 +27,16 @@ rfs rebuild-editable --reference --out --asset-policy smart Use `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview` only for explicit offline engineering validation. Never present that result as production-approved. +For paper understanding or prompt-only work, start with: + +```powershell +rfs doctor --json +rfs inspect-pdf --paper --out --json +rfs fast-framework-prompt --paper --out --deadline 180 --json +``` + +`inspect-pdf` and the fast path share a global document cache. For scanned PDFs, `auto` prefers RapidOCR when installed. Do not enable automatic EasyOCR model downloads implicitly; use `RFS_OCR_ALLOW_DOWNLOAD=1` only when the user explicitly wants the one-time download. + ## Required semantic policy - Read and structure the paper before generating the visual reference. @@ -87,10 +97,11 @@ Use the two independent suites to locate failures before changing the workflow: rfs benchmark list --root benchmarks --json rfs benchmark fetch --case --json rfs benchmark fast --case --out --deadline 180 --json +rfs benchmark fast-suite --root benchmarks --out --planner-mode heuristic --json rfs benchmark run --case --out --json rfs benchmark score --case --run --json ``` `paper-to-image` scientific and terminology errors are hard failures. `image-to-ppt` must reject full-slide reference images even when raster similarity is high. -For prompt-only work, run `rfs fast-framework-prompt --paper --out --deadline 180 --json`. Inspect `extraction_report.json` first, then `planning_validation_report.json`; treat `engineering_only: true` as a non-production result even when deterministic topology coverage is high. +For prompt-only work, inspect `extraction_report.json`, `contract_completion_report.json`, and `planning_validation_report.json` in that order. Treat `engineering_only: true` as non-production even when deterministic topology coverage is high. Use `fast-suite` after parser or contract changes and reject regressions in forbidden content, evidence grounding, provider reliability, or stage timings. From e8c289c202bef9559016a56b0471d3227901dfe8 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 02:44:39 +0800 Subject: [PATCH 23/74] Harden fast paper contracts across NLP architectures --- benchmarks/README.md | 2 +- .../109_transformer_encoder_decoder/case.json | 15 ++ .../expected_semantics.json | 30 +++ .../source.json | 12 ++ .../110_bert_pretrain_finetune/case.json | 15 ++ .../expected_semantics.json | 29 +++ .../110_bert_pretrain_finetune/source.json | 12 ++ .../111_rag_retrieval_generation/case.json | 15 ++ .../expected_semantics.json | 24 +++ .../111_rag_retrieval_generation/source.json | 12 ++ docs/paper-to-image.md | 2 + rfs/paper_to_image/contract_completion.py | 192 ++++++++++++++++-- tests/unit/test_paper_contract.py | 121 +++++++++++ 13 files changed, 467 insertions(+), 14 deletions(-) create mode 100644 benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json create mode 100644 benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json create mode 100644 benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json create mode 100644 benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json create mode 100644 benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json create mode 100644 benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json create mode 100644 benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json diff --git a/benchmarks/README.md b/benchmarks/README.md index d817611..31009f7 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -36,7 +36,7 @@ Generated runs and reports belong under `output/benchmarks/` and are not source Real-paper cases commit `source.json` and human-authored Ground Truth, while `benchmark fetch` downloads the PDF into an ignored local `inputs/` directory. This keeps the public repository reproducible without redistributing publisher files. -The committed unseen-generalization set includes DETR, CLIP, and NeRF in addition to ViT, Mask R-CNN, Self-Refine, ImageBind, and SAM. Local image-only scan regressions remain under ignored `tmp/pdfs/` and `output/pdf/`; do not commit publisher PDFs or generated OCR artifacts. +The committed unseen-generalization set spans vision, multimodal learning, and NLP: ViT, Mask R-CNN, Self-Refine, ImageBind, SAM, DETR, CLIP, NeRF, Transformer, BERT, and retrieval-augmented generation. The NLP cases specifically exercise short overview captions, method-text recovery, embedding summation, pre-training/fine-tuning separation, and retrieval-conditioned generation. Local image-only scan regressions remain under ignored `tmp/pdfs/` and `output/pdf/`; do not commit publisher PDFs or generated OCR artifacts. `benchmark fast-suite` writes `fast_suite_report.json` with per-case results and aggregate entity/relation recall, forbidden content, cache hit rates, provider success/retry counts, failure categories, and stage timings. diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json new file mode 100644 index 0000000..22cdb5e --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NeurIPS 2017 Transformer case for dual embedding paths, encoder-decoder conditioning, and output projection.", + "case_id": "109_transformer_encoder_decoder", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json new file mode 100644 index 0000000..2d079df --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/expected_semantics.json @@ -0,0 +1,30 @@ +{ + "summary": "Human-authored semantic contract for the Transformer encoder-decoder architecture.", + "entities": [ + {"id": "source_tokens", "label": "Inputs", "aliases": ["Input Sequence", "Source Tokens"], "role": "input", "required": true}, + {"id": "input_embedding", "label": "Input Embedding", "aliases": ["Input Embeddings"], "role": "module", "required": true}, + {"id": "position_encoding", "label": "Positional Encoding", "aliases": ["Position Encoding"], "role": "module", "required": true}, + {"id": "encoder_stack", "label": "Encoder Stack", "aliases": ["Encoder", "N× Encoder"], "role": "module", "required": true}, + {"id": "target_tokens", "label": "Outputs (shifted right)", "aliases": ["Target Tokens", "Target Sequence"], "role": "input", "required": true}, + {"id": "output_embedding", "label": "Output Embedding", "aliases": ["Output Embeddings"], "role": "module", "required": true}, + {"id": "decoder_stack", "label": "Decoder Stack", "aliases": ["Decoder", "N× Decoder"], "role": "module", "required": true}, + {"id": "linear", "label": "Linear", "aliases": ["Linear Layer"], "role": "module", "required": true}, + {"id": "softmax", "label": "Softmax", "role": "module", "required": true}, + {"id": "output_probabilities", "label": "Output Probabilities", "aliases": ["Next-token Probabilities", "Outputs"], "role": "output", "required": true} + ], + "relations": [ + {"source": "source_tokens", "target": "input_embedding", "type": "embedding", "required": true}, + {"source": "input_embedding", "target": "encoder_stack", "type": "data_flow", "required": true}, + {"source": "position_encoding", "target": "encoder_stack", "type": "conditioning", "required": true}, + {"source": "target_tokens", "target": "output_embedding", "type": "embedding", "required": true}, + {"source": "output_embedding", "target": "decoder_stack", "type": "data_flow", "required": true}, + {"source": "position_encoding", "target": "decoder_stack", "type": "conditioning", "required": true}, + {"source": "encoder_stack", "target": "decoder_stack", "type": "cross_attention", "required": true}, + {"source": "decoder_stack", "target": "linear", "type": "data_flow", "required": true}, + {"source": "linear", "target": "softmax", "type": "data_flow", "required": true}, + {"source": "softmax", "target": "output_probabilities", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Convolutional Backbone", "Recurrent Layer", "LSTM", "Region Proposals"], + "expected_reading_order": ["source_tokens", "input_embedding", "encoder_stack", "target_tokens", "output_embedding", "decoder_stack", "linear", "softmax", "output_probabilities"], + "topology_requirements": {"primary_flow": "encoder_decoder", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json new file mode 100644 index 0000000..be0df1d --- /dev/null +++ b/benchmarks/paper-to-image/cases/109_transformer_encoder_decoder/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Attention Is All You Need", + "venue": "NeurIPS", + "year": 2017, + "arxiv_id": "1706.03762", + "official_url": "https://papers.nips.cc/paper/7181-attention-is-all-you-need", + "paper_urls": ["https://arxiv.org/pdf/1706.03762", "https://export.arxiv.org/pdf/1706.03762"], + "target_figure": "Figure 1", + "verified_caption": "The Transformer model architecture contains encoder and decoder stacks, embeddings with positional encoding, attention and feed-forward sublayers, and a linear-softmax output path.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json new file mode 100644 index 0000000..63d8280 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NAACL 2019 BERT case for embedding summation, pre-training objectives, and downstream fine-tuning.", + "case_id": "110_bert_pretrain_finetune", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "dense_multiframe", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json new file mode 100644 index 0000000..527cf05 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/expected_semantics.json @@ -0,0 +1,29 @@ +{ + "summary": "Human-authored semantic contract for BERT pre-training, input representation, and fine-tuning.", + "entities": [ + {"id": "input_tokens", "label": "Input Tokens", "aliases": ["Token Sequence", "Input Sequence"], "role": "input", "required": true}, + {"id": "token_embeddings", "label": "Token Embeddings", "role": "module", "required": true}, + {"id": "segment_embeddings", "label": "Segment Embeddings", "aliases": ["Segmentation Embeddings"], "role": "module", "required": true}, + {"id": "position_embeddings", "label": "Position Embeddings", "role": "module", "required": true}, + {"id": "input_representation", "label": "Input Representation", "aliases": ["BERT Input Representation"], "role": "module", "required": true}, + {"id": "bert_encoder", "label": "BERT", "aliases": ["BERT Encoder", "Bidirectional Transformer Encoder"], "role": "module", "required": true}, + {"id": "masked_lm", "label": "Masked LM", "aliases": ["Masked Language Model", "Masked Language Modeling"], "role": "training", "required": true}, + {"id": "next_sentence", "label": "Next Sentence Prediction", "aliases": ["NSP"], "role": "training", "required": true}, + {"id": "fine_tuning", "label": "Fine-tuning", "aliases": ["Fine-tune"], "role": "module", "required": true}, + {"id": "downstream_tasks", "label": "Downstream Tasks", "aliases": ["Task-specific Outputs", "Fine-tuning Tasks"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_tokens", "target": "token_embeddings", "type": "embedding", "required": true}, + {"source": "token_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "segment_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "position_embeddings", "target": "input_representation", "type": "sum", "required": true}, + {"source": "input_representation", "target": "bert_encoder", "type": "data_flow", "required": true}, + {"source": "bert_encoder", "target": "masked_lm", "type": "training_objective", "required": true}, + {"source": "bert_encoder", "target": "next_sentence", "type": "training_objective", "required": true}, + {"source": "bert_encoder", "target": "fine_tuning", "type": "initialization", "required": true}, + {"source": "fine_tuning", "target": "downstream_tasks", "type": "data_flow", "required": true} + ], + "forbidden_labels": ["Autoregressive Decoder", "Convolutional Backbone", "Retrieval Index", "Image Encoder"], + "expected_reading_order": ["input_tokens", "token_embeddings", "segment_embeddings", "position_embeddings", "input_representation", "bert_encoder", "fine_tuning", "downstream_tasks"], + "topology_requirements": {"primary_flow": "pretrain_then_finetune", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": true} +} diff --git a/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json new file mode 100644 index 0000000..f200c15 --- /dev/null +++ b/benchmarks/paper-to-image/cases/110_bert_pretrain_finetune/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", + "venue": "NAACL", + "year": 2019, + "arxiv_id": "1810.04805", + "official_url": "https://aclanthology.org/N19-1423/", + "paper_urls": ["https://arxiv.org/pdf/1810.04805", "https://export.arxiv.org/pdf/1810.04805"], + "target_figure": "Figures 1 and 2", + "verified_caption": "BERT uses the same bidirectional Transformer encoder for pre-training and fine-tuning; its input representation sums token, segment, and position embeddings.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json new file mode 100644 index 0000000..89c7667 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/case.json @@ -0,0 +1,15 @@ +{ + "summary": "NeurIPS 2020 RAG case for parametric/non-parametric composition and retrieval-conditioned generation.", + "case_id": "111_rag_retrieval_generation", + "suite": "paper-to-image", + "tier": "real-paper-unseen-generalization", + "topology": "branch", + "paper": "inputs/paper.pdf", + "source": "source.json", + "expected_semantics": "expected_semantics.json", + "preferences": "../../preferences/top_conference.json", + "positive_references": [], + "negative_references": [], + "run_config": {"planner_mode": "heuristic", "image_asset_mode": "placeholder", "image_candidates": 2, "review_mode": "heuristic", "aspect_ratio": "16:9", "ocr_engine": "off"}, + "thresholds": {"entity_recall": 1.0, "relation_recall": 1.0, "exact_label_rate": 1.0, "plan_entity_recall": 0.8, "plan_relation_recall": 0.7, "hallucination_count": 0, "forbidden_content_count": 0, "plan_forbidden_content_count": 0} +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json new file mode 100644 index 0000000..93e8caf --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json @@ -0,0 +1,24 @@ +{ + "summary": "Human-authored semantic contract for retrieval-augmented generation.", + "entities": [ + {"id": "input_query", "label": "Input Query", "aliases": ["Query", "Input x"], "role": "input", "required": true}, + {"id": "query_encoder", "label": "Query Encoder", "role": "module", "required": true}, + {"id": "document_index", "label": "Document Index", "aliases": ["Non-parametric Memory", "Wikipedia Index"], "role": "module", "required": true}, + {"id": "retriever", "label": "Retriever", "aliases": ["Neural Retriever"], "role": "module", "required": true}, + {"id": "top_k_documents", "label": "Top-K Documents", "aliases": ["Retrieved Documents", "Retrieved Passages"], "role": "module", "required": true}, + {"id": "generator", "label": "Generator", "aliases": ["Seq2seq Generator", "Pre-trained Generator"], "role": "module", "required": true}, + {"id": "output_sequence", "label": "Output Sequence", "aliases": ["Generated Output", "Output y"], "role": "output", "required": true} + ], + "relations": [ + {"source": "input_query", "target": "query_encoder", "type": "encoding", "required": true}, + {"source": "query_encoder", "target": "retriever", "type": "retrieval_query", "required": true}, + {"source": "document_index", "target": "retriever", "type": "memory_lookup", "required": true}, + {"source": "retriever", "target": "top_k_documents", "type": "retrieval", "required": true}, + {"source": "input_query", "target": "generator", "type": "generation_input", "required": true}, + {"source": "top_k_documents", "target": "generator", "type": "conditioning", "required": true}, + {"source": "generator", "target": "output_sequence", "type": "generation", "required": true} + ], + "forbidden_labels": ["Image Encoder", "Object Queries", "Mask Decoder", "Convolutional Backbone"], + "expected_reading_order": ["input_query", "query_encoder", "document_index", "retriever", "top_k_documents", "generator", "output_sequence"], + "topology_requirements": {"primary_flow": "retrieval_then_generation", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} +} diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json new file mode 100644 index 0000000..e3435f1 --- /dev/null +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/source.json @@ -0,0 +1,12 @@ +{ + "summary": "Publication metadata and local-fetch sources; the PDF is not redistributed by this repository.", + "paper_title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks", + "venue": "NeurIPS", + "year": 2020, + "arxiv_id": "2005.11401", + "official_url": "https://papers.nips.cc/paper/2020/hash/6b493230205f780e1bc26945df7481e5-Abstract.html", + "paper_urls": ["https://arxiv.org/pdf/2005.11401", "https://export.arxiv.org/pdf/2005.11401"], + "target_figure": "Figure 1", + "verified_caption": "RAG combines a query encoder and non-parametric document index with a pre-trained seq2seq generator; retrieved top-K documents condition output generation.", + "license_note": "Verify the source license before redistributing the PDF or figure image. Local benchmark fetching is for research use." +} diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 5360e61..11e1423 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -17,9 +17,11 @@ Run all or selected paper benchmarks with aggregate reliability metrics: ```powershell rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast-suite --planner-mode heuristic --json rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case-id 106_detr_set_prediction --case-id 107_clip_contrastive --json +rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json ``` `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, and parser/semantic/total timings. +The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. These repairs are paper-name agnostic and are covered by the NLP suite. ## Production Command diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 43b20d8..048dd02 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -27,6 +27,7 @@ class ConceptRule: pattern: str aliases: tuple[str, ...] = () requires_overview: bool = False + context_pattern: str | None = None CONCEPT_RULES = ( @@ -48,10 +49,10 @@ class ConceptRule: ConceptRule("backbone", "Backbone", "modules", "feature extractor", r"\bbackbones?\b", ("backbone architecture",), True), ConceptRule("image_encoder", "Image Encoder", "modules", "encoder", r"\bimage encoder\b"), ConceptRule("text_encoder", "Text Encoder", "modules", "encoder", r"\btext encoder\b"), - ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b"), - ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b"), + ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b", (), True), + ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b", (), True), ConceptRule("position_embedding", "Position Embedding", "modules", "conditioning", r"\bposition embeddings?\b", ("positional embedding",), True), - ConceptRule("positional_encoding", "Positional Encoding", "modules", "conditioning", r"\bposition(?:al)? encod(?:ing|ings)\b", (), True), + ConceptRule("positional_encoding", "Positional Encoding", "modules", "conditioning", r"\bposition(?:al)? encod(?:ing|ings)\b"), ConceptRule("image_patches", "Image Patches", "modules", "intermediate", r"\bimage patches?\b|\bfixed-size patches\b", ("patches",), True), ConceptRule("linear_projection", "Linear Projection of Flattened Patches", "modules", "projection", r"\blinear projection(?: of flattened patches)?\b|\blinearly embed\b", ("linear projection",), True), ConceptRule("class_token", "Class Token", "modules", "conditioning", r"\bclass(?:ification)? token\b|\bclass embedding\b", ("classification token", "class embedding"), True), @@ -88,6 +89,32 @@ class ConceptRule: ConceptRule("valid_mask", "Valid Segmentation Mask", "outputs", "output", r"\bvalid masks?\b|\bobject masks?\b", ("valid masks", "valid mask"), True), ConceptRule("zero_shot_classifier", "Zero-Shot Classifier", "outputs", "inference output", r"\bzero[ -]?shot (?:linear )?classifier\b|\bzero[ -]?shot prediction\b", ("zero-shot linear classifier", "zero-shot prediction")), ConceptRule("rendered_image", "Rendered Image", "outputs", "output", r"\b(?:rendered|synthesi[sz]ed) (?:images?|views?)\b|\bsynthesi[sz]e images?\b|\bnovel views?\b", ("synthesized image", "synthesized view", "novel view")), + ConceptRule("source_tokens", "Inputs", "inputs", "source sequence", r"\bencoder maps an input sequence(?: of symbol representations)?\b|\blearned embeddings to convert the input\s+tokens\b", ("input sequence", "source tokens")), + ConceptRule("input_embedding", "Input Embedding", "modules", "embedding", r"\blearned embeddings to convert the input\s+tokens\b|\binput embeddings?\b", ("input embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), + ConceptRule("encoder_stack", "Encoder Stack", "modules", "encoder", r"\bencoder is composed of a stack\b|\bencoder stack\b", ("encoder", "transformer encoder")), + ConceptRule("target_tokens", "Outputs (shifted right)", "inputs", "target sequence", r"\boutput embeddings? (?:are )?offset by one position\b|\binput\s+tokens and output tokens\b", ("target tokens", "target sequence", "outputs shifted right")), + ConceptRule("output_embedding", "Output Embedding", "modules", "embedding", r"\boutput embeddings?\b|\binput\s+tokens and output tokens\b", ("output embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), + ConceptRule("decoder_stack", "Decoder Stack", "modules", "decoder", r"\bdecoder is also composed of a stack\b|\bdecoder stack\b", ("decoder", "transformer decoder")), + ConceptRule("output_linear", "Linear", "modules", "output projection", r"\blearned linear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("linear layer", "output projection")), + ConceptRule("output_softmax", "Softmax", "modules", "output normalization", r"\blinear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("softmax layer",)), + ConceptRule("output_probabilities", "Output Probabilities", "outputs", "output", r"\bpredicted next-token probabilities\b", ("predicted next-token probabilities", "next-token probabilities", "outputs")), + ConceptRule("input_tokens", "Input Sequence", "inputs", "input sequence", r"\brepresent the input\s+sequence\b|\bfor a given token, its input representation\b", ("input tokens", "token sequence"), context_pattern=r"\btoken embeddings?\b.*\b(?:segment|segmentation) embeddings?\b.*\bposition embeddings?\b"), + ConceptRule("token_embeddings", "Token Embeddings", "modules", "embedding", r"\btoken embeddings?\b", (), context_pattern=r"\b(?:segment|segmentation) embeddings?\b.*\bposition embeddings?\b"), + ConceptRule("segment_embeddings", "Segment Embeddings", "modules", "embedding", r"\b(?:segment|segmentation) embeddings?\b", ("segmentation embeddings",), context_pattern=r"\btoken embeddings?\b.*\bposition embeddings?\b"), + ConceptRule("bert_position_embeddings", "Position Embeddings", "modules", "embedding", r"\bposition embeddings?\b", (), context_pattern=r"\btoken embeddings?\b.*\b(?:segment|segmentation) embeddings?\b"), + ConceptRule("input_representation", "Input Representation", "modules", "representation", r"\binput representation\b|\binput embeddings are the sum of\b", ("bert input representation",), context_pattern=r"\btoken embeddings?\b.*\b(?:segment|segmentation) embeddings?\b.*\bposition embeddings?\b"), + ConceptRule("bidirectional_encoder", "Bidirectional Transformer Encoder", "modules", "encoder", r"\bmulti-layer bidirectional transformer en-?\s*coder\b|\bbidirectional transformer encoder\b", ("bert", "bert encoder")), + ConceptRule("masked_lm", "Masked LM", "modules", "training objective", r"\bmasked (?:language model(?:ing)?|lm)\b", ("masked language model", "masked language modeling", "mlm objective"), context_pattern=r"\bnext sentence prediction\b|\bNSP\b"), + ConceptRule("next_sentence", "Next Sentence Prediction", "modules", "training objective", r"\bnext sentence prediction\b|\bNSP\b", ("nsp",), context_pattern=r"\bmasked (?:language model(?:ing)?|lm)\b"), + ConceptRule("fine_tuning", "Fine-tuning", "modules", "adaptation", r"\bframework:\s*pre-training and fine-tuning\b|\bpre-trained model parameters are used to initialize models for different down-stream tasks\b", ("fine-tune", "fine tuning")), + ConceptRule("downstream_tasks", "Downstream Tasks", "outputs", "task outputs", r"\bmodels for different down-stream tasks\b|\blabeled data from the downstream tasks\b|\bmodel many downstream tasks\b", ("task-specific outputs", "fine-tuning tasks")), + ConceptRule("input_query", "Input Query", "inputs", "query", r"\bfor query x\b|\bgiven a query x\b|\bquery representation produced by a query encoder\b", ("query", "input x")), + ConceptRule("query_encoder", "Query Encoder", "modules", "encoder", r"\bquery encoder\b"), + ConceptRule("document_index", "Document Index", "modules", "non-parametric memory", r"\bdocument index\b|\bdense vector index\b", ("non-parametric memory", "wikipedia index")), + ConceptRule("retriever", "Retriever", "modules", "retriever", r"\b(?:pre-trained |neural )?retriever\b"), + ConceptRule("top_k_documents", "Top-K Documents", "modules", "retrieved context", r"\btop-?k documents\b|\btop k documents are retrieved\b", ("retrieved documents", "retrieved passages")), + ConceptRule("generator_module", "Generator", "modules", "generator", r"\bpre-trained seq2seq model\s*\(\s*generator\s*\)|\bgenerator p|\bgenerator component\b", ("generate", "seq2seq generator", "pre-trained generator")), + ConceptRule("output_sequence", "Output Sequence", "outputs", "generated output", r"\bgenerator produces the output sequence\b|\bgenerating the target sequence y\b|\bfinal prediction y\b", ("prediction y", "final prediction", "generated output", "output y")), ) @@ -157,20 +184,49 @@ class ConceptRule: ("volume_density", "volume_rendering", "rendering_input"), ("color", "volume_rendering", "rendering_input"), ("volume_rendering", "rendered_image", "data_flow"), + ("source_tokens", "input_embedding", "embedding"), + ("input_embedding", "encoder_stack", "data_flow"), + ("positional_encoding", "encoder_stack", "conditioning"), + ("target_tokens", "output_embedding", "embedding"), + ("output_embedding", "decoder_stack", "data_flow"), + ("positional_encoding", "decoder_stack", "conditioning"), + ("encoder_stack", "decoder_stack", "cross_attention"), + ("decoder_stack", "output_linear", "data_flow"), + ("output_linear", "output_softmax", "data_flow"), + ("output_softmax", "output_probabilities", "data_flow"), + ("input_tokens", "token_embeddings", "embedding"), + ("token_embeddings", "input_representation", "sum"), + ("segment_embeddings", "input_representation", "sum"), + ("bert_position_embeddings", "input_representation", "sum"), + ("input_representation", "bidirectional_encoder", "data_flow"), + ("bidirectional_encoder", "masked_lm", "training_objective"), + ("bidirectional_encoder", "next_sentence", "training_objective"), + ("bidirectional_encoder", "fine_tuning", "initialization"), + ("fine_tuning", "downstream_tasks", "data_flow"), + ("input_query", "query_encoder", "encoding"), + ("query_encoder", "retriever", "retrieval_query"), + ("document_index", "retriever", "memory_lookup"), + ("retriever", "top_k_documents", "retrieval"), + ("input_query", "generator_module", "generation_input"), + ("top_k_documents", "generator_module", "conditioning"), + ("generator_module", "output_sequence", "generation"), ) def _overview_candidates(parsed: dict[str, Any], limit: int = 2) -> list[dict[str, Any]]: - positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|model|system|workflow|directly predicts)\b", re.IGNORECASE) - negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention|distribution)\b", re.IGNORECASE) + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|model|system|workflow|representation|encoder|decoder|directly predicts)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|differences?|versus|performance|result|ablation|visualization|qualitative|attention|distribution|interface)\b", re.IGNORECASE) ranked = [] for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): caption = str(figure.get("caption") or "") - score = (8 if positive.search(caption) else 0) + (4 if re.match(r"^(figure|fig\.)\s*[12]\b", caption, re.IGNORECASE) else 0) + positive_match = positive.search(caption) + if not positive_match: + continue + score = 8 + (4 if re.match(r"^(figure|fig\.)\s*[12]\b", caption, re.IGNORECASE) else 0) score += 3 if 160 <= len(caption) <= 1600 else 0 - score -= 7 if negative.search(caption) else 0 + score -= 14 if negative.search(caption) else 0 ranked.append((score, -index, figure)) - return [item for _, _, item in sorted(ranked, reverse=True)[:limit]] + return [item for score, _, item in sorted(ranked, reverse=True) if score > 0][:limit] def _evidence_for_figure(parsed: dict[str, Any], figure: dict[str, Any]) -> dict[str, Any] | None: @@ -206,15 +262,37 @@ def _relevant_evidence(parsed: dict[str, Any]) -> tuple[list[dict[str, Any]], li return selected, relevant +def _window_matches(pattern: re.Pattern[str], items: list[dict[str, Any]], max_window: int = 3) -> list[dict[str, Any]]: + for start in range(len(items)): + page = items[start].get("page") + for size in range(2, max_window + 1): + window = items[start:start + size] + if len(window) != size or any(item.get("page") != page for item in window): + break + if pattern.search(" ".join(str(item.get("text") or "") for item in window)): + return window + return [] + + def _find_matches(rule: ConceptRule, selected: list[dict[str, Any]], relevant: list[dict[str, Any]]) -> list[dict[str, Any]]: pattern = re.compile(rule.pattern, re.IGNORECASE) + if rule.context_pattern: + context = re.compile(rule.context_pattern, re.IGNORECASE | re.DOTALL) + context_text = " ".join(str(item.get("text") or "") for item in relevant) + if not context.search(context_text): + return [] primary = [item for item in selected if pattern.search(str(item.get("text") or ""))] if primary: return primary[:2] + primary_window = _window_matches(pattern, selected) + if primary_window: + return primary_window if rule.requires_overview: return [] secondary = [item for item in relevant if pattern.search(str(item.get("text") or ""))] - return secondary[:2] + if secondary: + return secondary[:2] + return _window_matches(pattern, relevant) def _find_existing(spec: dict[str, Any], rule: ConceptRule) -> tuple[str, dict[str, Any]] | None: @@ -232,8 +310,69 @@ def _find_existing(spec: dict[str, Any], rule: ConceptRule) -> tuple[str, dict[s return None +def _relation_bridge(source: dict[str, Any], target: dict[str, Any], relevant: list[dict[str, Any]]) -> dict[str, Any] | None: + def terms(item: dict[str, Any]) -> set[str]: + values = set() + for word in re.findall(r"[a-z0-9]+", _label(item).casefold()): + if len(word) < 4 or word in {"shifted", "right", "stack", "module", "layer"}: + continue + values.add(word[:-1] if word.endswith("s") and len(word) > 4 else word) + return values + + source_terms = terms(source) + target_terms = terms(target) + distinct_target_terms = target_terms - source_terms or target_terms + if not source_terms or not distinct_target_terms: + return None + return next( + ( + item + for item in relevant + if any(term in _normalized(item.get("text")) for term in source_terms) + and any(term in _normalized(item.get("text")) for term in distinct_target_terms) + ), + None, + ) + + +def _ground_statement(item: object, relevant: list[dict[str, Any]]) -> list[str]: + if not isinstance(item, dict) or item.get("evidence_ids"): + return [] + statement = str(item.get("text") or item.get("statement") or "").strip() + if not statement or statement.casefold() == "unknown": + return [] + stop_words = {"a", "an", "and", "are", "as", "for", "in", "is", "of", "on", "or", "the", "to", "we", "with"} + target_terms = {word for word in re.findall(r"[a-z0-9]+", statement.casefold()) if len(word) >= 4 and word not in stop_words} + if not target_terms: + return [] + best_score = 0.0 + best_window: list[dict[str, Any]] = [] + for start in range(len(relevant)): + page = relevant[start].get("page") + for size in range(1, 4): + window = relevant[start:start + size] + if len(window) != size or any(value.get("page") != page for value in window): + break + text_terms = set(re.findall(r"[a-z0-9]+", " ".join(str(value.get("text") or "") for value in window).casefold())) + score = len(target_terms & text_terms) / len(target_terms) + if score > best_score: + best_score = score + best_window = window + if best_score < 0.6: + item["text"] = "unknown" + item["status"] = "unknown" + return [] + evidence_ids = [str(value.get("id")) for value in best_window if value.get("id")] + item["evidence_ids"] = evidence_ids + return evidence_ids + + def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: selected, relevant = _relevant_evidence(parsed) + grounded_statements = { + field: _ground_statement(spec.get(field), relevant) + for field in ("research_problem", "central_claim") + } found: dict[str, dict[str, Any]] = {} added_entities: list[str] = [] upgraded_entities: list[str] = [] @@ -250,7 +389,9 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) if str(item.get("role") or "") == "paper-derived stage requiring VLM verification": item["role"] = rule.role adopted_entities.append(str(item.get("id") or rule.key)) - if _normalized(current) != _normalized(rule.label) and (_normalized(current) in _normalized(rule.label) or current.casefold().startswith(("neural network module", "learned "))): + accepted_names = {_normalized(rule.label), *(_normalized(value) for value in rule.aliases)} + exact_rule_id = _normalized(item.get("id")) == _normalized(rule.key) + if _normalized(current) != _normalized(rule.label) and (exact_rule_id or _normalized(current) in accepted_names or _normalized(current) in _normalized(rule.label) or current.casefold().startswith(("neural network module", "learned "))): item["name"] = rule.label upgraded_entities.append(str(item.get("id") or rule.key)) item["evidence_ids"] = list(dict.fromkeys(list(item.get("evidence_ids", [])) + evidence_ids)) @@ -272,6 +413,7 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) selected_ids = [str(item.get("id")) for item in selected if item.get("id")] evidence_by_id = {str(item.get("id")): item for item in relevant if item.get("id")} added_relations: list[str] = [] + repaired_relations: list[str] = [] for source_key, target_key, relation_type in RELATION_RULES: source = found.get(source_key) target = found.get(target_key) @@ -282,9 +424,7 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) continue evidence_ids = list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(target.get("evidence_ids", [])))) if selected_ids and not any(value in selected_ids for value in evidence_ids): - source_text = _label(source).casefold() - target_text = _label(target).casefold() - bridge = next((item for item in relevant if source_text.split()[-1] in str(item.get("text") or "").casefold() and target_text.split()[-1] in str(item.get("text") or "").casefold()), None) + bridge = _relation_bridge(source, target, relevant) source_pages = [int(evidence_by_id[value].get("page") or 0) for value in source.get("evidence_ids", []) if value in evidence_by_id] target_pages = [int(evidence_by_id[value].get("page") or 0) for value in target.get("evidence_ids", []) if value in evidence_by_id] nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 3) @@ -296,6 +436,30 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) existing_pairs.add(pair) added_relations.append(f"{pair[0]}->{pair[1]}") + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in spec.get(field, []) if isinstance(item, dict) and item.get("id") + } + for relation in relations: + if not isinstance(relation, dict) or relation.get("evidence_ids"): + continue + source = endpoints.get(str(relation.get("source"))) + target = endpoints.get(str(relation.get("target"))) + if not source or not target: + continue + candidate_ids = list(dict.fromkeys(list(source.get("evidence_ids", [])) + list(target.get("evidence_ids", [])))) + source_pages = [int(evidence_by_id[value].get("page") or 0) for value in source.get("evidence_ids", []) if value in evidence_by_id] + target_pages = [int(evidence_by_id[value].get("page") or 0) for value in target.get("evidence_ids", []) if value in evidence_by_id] + nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 3) + bridge = _relation_bridge(source, target, relevant) + if not candidate_ids or not (any(value in selected_ids for value in candidate_ids) or nearby or bridge): + continue + if bridge and bridge.get("id"): + candidate_ids.append(str(bridge["id"])) + relation["evidence_ids"] = list(dict.fromkeys(candidate_ids)) + repaired_relations.append(f"{relation.get('source')}->{relation.get('target')}") + overview_terms = [rule.key for rule in CONCEPT_RULES if _find_matches(rule, selected, [])] covered_terms = [key for key in overview_terms if key in found] return { @@ -308,5 +472,7 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) "upgraded_entities": upgraded_entities, "adopted_entities": adopted_entities, "added_relations": added_relations, + "repaired_relations": repaired_relations, + "grounded_statements": {field: ids for field, ids in grounded_statements.items() if ids}, "rules_are_evidence_gated": True, } diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 1ea9100..4c4718f 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -240,6 +240,127 @@ def test_generic_completion_recovers_nerf_branch_without_false_multimodal_topolo self.assertIn((ids["Volume Rendering"], ids["Rendered Image"]), pairs) self.assertEqual(spec["topology"], "branch") + def test_generic_completion_recovers_encoder_decoder_from_method_lines(self): + caption = "Figure 1: The model architecture." + lines = [ + "Here, the encoder maps an input sequence of symbol representations to continuous representations.", + "The encoder is composed of a stack of identical layers.", + "The decoder is also composed of a stack of identical layers and attends over the output of the encoder stack.", + "We use learned embeddings to convert the input", + "tokens and output tokens to vectors. We also use the usual learned linear transfor-", + "mation and softmax function to convert the decoder output to predicted next-token probabilities.", + "The output embeddings are offset by one position.", + "We add positional encodings to the input embeddings at the bottoms of the encoder and decoder stacks.", + ] + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "confidence": 1.0}] + + [{"id": f"E00{index + 2:02d}", "page": 3 + index // 3, "kind": "paragraph", "text": text, "section_hint": "Method", "confidence": 1.0} for index, text in enumerate(lines)], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Inputs", "Input Embedding", "Encoder Stack", "Outputs (shifted right)", "Output Embedding", "Decoder Stack", "Linear", "Softmax", "Output Probabilities"): + self.assertIn(label, ids) + self.assertIn((ids["Encoder Stack"], ids["Decoder Stack"]), pairs) + self.assertIn((ids["Linear"], ids["Softmax"]), pairs) + + def test_generic_completion_recovers_embedding_pretraining_and_finetuning(self): + captions = [ + "Figure 1: Overall pre-training and fine-tuning procedures. The same pre-trained model parameters are used to initialize models for different down-stream tasks.", + "Figure 2: Input representation. The input embeddings are the sum of the token embeddings, the segmentation embeddings and the position embeddings.", + ] + method = [ + "The model architecture is a multi-layer bidirectional Transformer encoder.", + "Task #1: Masked LM", + "Task #2: Next Sentence Prediction (NSP)", + "We represent the input sequence and fine-tune using labeled data from the downstream tasks.", + ] + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": index + 2, "caption": text} for index, text in enumerate(captions)]}, + "evidence": [{"id": f"E000{index + 1}", "page": index + 2, "kind": "caption", "text": text, "section_hint": "Figure Captions", "confidence": 1.0} for index, text in enumerate(captions)] + + [{"id": f"E001{index}", "page": 4, "kind": "paragraph", "text": text, "section_hint": "Method", "confidence": 1.0} for index, text in enumerate(method)], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Sequence", "Token Embeddings", "Segment Embeddings", "Position Embeddings", "Input Representation", "Bidirectional Transformer Encoder", "Masked LM", "Next Sentence Prediction", "Fine-tuning", "Downstream Tasks"): + self.assertIn(label, ids) + self.assertIn((ids["Token Embeddings"], ids["Input Representation"]), pairs) + self.assertIn((ids["Bidirectional Transformer Encoder"], ids["Fine-tuning"]), pairs) + + def test_generic_completion_recovers_retrieval_conditioned_generation(self): + caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." + result_caption = "Figure 2: Posterior for each generated token with five retrieved documents." + parsed = { + "page_count": 6, + "document_index": {"figures": [{"page": 2, "caption": caption}, {"page": 5, "caption": result_caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 5, "kind": "caption", "text": result_caption, "section_hint": "Figure Captions", "confidence": 1.0}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Query", "Query Encoder", "Document Index", "Retriever", "Top-K Documents", "Generator", "Output Sequence"): + self.assertIn(label, ids) + self.assertNotIn("Generate", ids) + self.assertIn((ids["Document Index"], ids["Retriever"]), pairs) + self.assertIn((ids["Top-K Documents"], ids["Generator"]), pairs) + + def test_generic_completion_repairs_vlm_ids_labels_and_relation_grounding(self): + caption = "Figure 1: Pre-training and fine-tuning procedures." + parsed = { + "page_count": 5, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "The input embeddings are the sum of the token embeddings, segmentation embeddings and position embeddings.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0003", "page": 3, "kind": "paragraph", "text": "The architecture is a multi-layer bidirectional Transformer encoder.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0004", "page": 4, "kind": "paragraph", "text": "We train with a masked language model and next sentence prediction.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0005", "page": 4, "kind": "paragraph", "text": "We represent the input sequence for downstream tasks.", "section_hint": "Method", "confidence": 1.0}, + {"id": "E0006", "page": 1, "kind": "paragraph", "text": "A major limitation is that standard language models are", "section_hint": "Introduction", "confidence": 1.0}, + {"id": "E0007", "page": 1, "kind": "paragraph", "text": "unidirectional.", "section_hint": "Introduction", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "input_tokens", "name": "input example", "evidence_ids": ["E0005"]}], + "modules": [ + {"id": "embedding_layer", "name": "input embeddings", "evidence_ids": ["E0002"]}, + {"id": "transformer_encoder", "name": "Bidirectional Transformer Encoder", "evidence_ids": ["E0003"]}, + {"id": "mlm_head", "name": "MLM objective", "evidence_ids": []}, + ], + "research_problem": {"text": "Standard language models are unidirectional.", "evidence_ids": []}, + "outputs": [], "innovations": [], "must_show": [], "terminology": {}, + "relations": [ + {"source": "embedding_layer", "target": "transformer_encoder", "type": "data_flow", "evidence_ids": []}, + {"source": "transformer_encoder", "target": "mlm_head", "type": "prediction", "evidence_ids": []}, + ], + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(next(item for item in spec["inputs"] if item["id"] == "input_tokens")["name"], "Input Sequence") + self.assertEqual(next(item for item in spec["modules"] if item["id"] == "mlm_head")["name"], "Masked LM") + self.assertTrue(next(item for item in spec["modules"] if item["id"] == "mlm_head")["evidence_ids"]) + self.assertTrue(all(item.get("evidence_ids") for item in spec["relations"] if item.get("source") in {"embedding_layer", "transformer_encoder"} and item.get("target") in {"transformer_encoder", "mlm_head"})) + self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0006", "E0007"]) + if __name__ == "__main__": unittest.main() From a3b1a9622fe582fb9257e5ae947201bcc58310dd Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 03:12:45 +0800 Subject: [PATCH 24/74] Improve OCR and long-paper evidence scheduling --- README.md | 2 +- docs/paper-to-image.md | 6 +- rfs/evaluation/benchmarking.py | 11 ++ rfs/paper_to_image/analyzer.py | 208 +++++++++++++++++++++++++-- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 14 +- tests/unit/test_benchmarking.py | 2 + tests/unit/test_paper_analyzer.py | 73 +++++++++- 8 files changed, 302 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 42e61eb..ee5ceb8 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 11e1423..c1c732e 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -10,7 +10,7 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --j The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, and parser agreement without model calls. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. Run all or selected paper benchmarks with aggregate reliability metrics: @@ -20,9 +20,11 @@ rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json ``` -`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, and parser/semantic/total timings. +`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, and OCR candidate/scheduled/completed totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. These repairs are paper-name agnostic and are covered by the NLP suite. +When a paper exceeds the evidence character budget, the extractor reserves representative evidence from every page, then prioritizes topology-defining statements and Abstract, Conclusion, Method, Introduction, and Experiments content. This prevents long introductions or appendices from silently excluding later conclusions and framework definitions. + ## Production Command ```powershell diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 81e23aa..25ced8d 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -732,6 +732,7 @@ def run_fast_benchmark_suite( benchmark = result.get("benchmark", {}) if isinstance(result.get("benchmark"), dict) else {} metrics = benchmark.get("metrics", {}) if isinstance(benchmark.get("metrics"), dict) else {} provider = run.get("provider", {}) if isinstance(run.get("provider"), dict) else {} + extraction_quality = run.get("extraction_quality", {}) if isinstance(run.get("extraction_quality"), dict) else {} results.append({ "case_id": case_id, "ok": bool(result.get("ok")), @@ -742,6 +743,7 @@ def run_fast_benchmark_suite( "document_cache_hit": bool(run.get("document_cache_hit")), "elapsed_seconds": run.get("elapsed_seconds"), "stage_timings": run.get("stage_timings", {}), + "extraction_quality": extraction_quality, "provider": provider, "plan_entity_recall": metrics.get("plan_entity_recall"), "plan_relation_recall": metrics.get("plan_relation_recall"), @@ -761,6 +763,9 @@ def numbers(field: str) -> list[float]: def timing(name: str) -> list[float]: return [float(item.get("stage_timings", {}).get(name)) for item in results if isinstance(item.get("stage_timings", {}).get(name), (int, float))] + def extraction_metric(name: str) -> list[float]: + return [float(item.get("extraction_quality", {}).get(name)) for item in results if isinstance(item.get("extraction_quality", {}).get(name), (int, float))] + def percentile(values: list[float], fraction: float) -> float: if not values: return 0.0 @@ -795,6 +800,12 @@ def percentile(values: list[float], fraction: float) -> float: "mean_document_preparation_seconds": _mean(timing("document_preparation_seconds")), "mean_document_extraction_seconds": _mean(timing("document_extraction_seconds")), "mean_semantic_compilation_seconds": _mean(timing("semantic_compilation_seconds")), + "mean_readable_page_ratio": _mean(extraction_metric("readable_page_ratio")), + "mean_evidence_page_coverage_ratio": _mean(extraction_metric("evidence_page_coverage_ratio")), + "mean_evidence_char_count": _mean(extraction_metric("evidence_char_count")), + "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), + "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), + "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), "provider_call_count": len(provider_calls), "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index e2ae43a..1ec6416 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -346,6 +346,81 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call return [], engine, str(exc) +def _prioritize_ocr_candidates( + pages: list[dict[str, Any]], + candidates: list[int], + max_pages: int, +) -> tuple[list[int], list[dict[str, Any]]]: + if max_pages <= 0 or not candidates: + return [], [] + page_count = max(1, len(pages)) + signals = { + "overview_figure": re.compile(r"\b(?:figure|fig\.)\s*[12]\b.*\b(?:overview|framework|architecture|pipeline|system|model)\b", re.IGNORECASE | re.DOTALL), + "method": re.compile(r"\b(?:methods?|methodology|approach|architecture|system overview|framework)\b", re.IGNORECASE), + "abstract": re.compile(r"\babstract\b", re.IGNORECASE), + "conclusion": re.compile(r"\b(?:conclusions?|discussion)\b", re.IGNORECASE), + "caption": re.compile(r"\b(?:figure|fig\.|table)\s*\d", re.IGNORECASE), + } + scored: list[tuple[int, int, list[str]]] = [] + for page_number in candidates: + page = pages[page_number - 1] if 0 < page_number <= len(pages) else {} + text = str(page.get("text") or "") + reasons: list[str] = [] + score = 0 + for name, weight in (("overview_figure", 14), ("method", 10), ("abstract", 9), ("conclusion", 8), ("caption", 5)): + if signals[name].search(text): + score += weight + reasons.append(name) + if page_number == 1: + score += 7 + reasons.append("first_page") + elif page_number <= 3: + score += 3 + reasons.append("early_page") + scored.append((score, page_number, reasons)) + + selected: list[int] = [] + reason_map: dict[int, list[str]] = {} + for score, page_number, reasons in sorted(scored, key=lambda item: (-item[0], item[1])): + if score <= 0 or len(selected) >= max_pages: + break + selected.append(page_number) + reason_map[page_number] = reasons + + anchors = [ + 1, + min(2, page_count), + max(1, round(page_count * 0.25)), + max(1, round(page_count * 0.50)), + max(1, round(page_count * 0.75)), + max(1, round(page_count * 0.85)), + ] + for anchor in anchors: + if len(selected) >= max_pages: + break + if anchor in selected: + continue + available = [value for value in candidates if value not in selected] + if not available: + break + page_number = min(available, key=lambda value: (abs(value - anchor), value)) + selected.append(page_number) + reason_map.setdefault(page_number, []).append(f"coverage_anchor_{anchor}") + + for page_number in candidates: + if len(selected) >= max_pages: + break + if page_number not in selected: + selected.append(page_number) + reason_map.setdefault(page_number, []).append("remaining_candidate") + + details = [ + {"page": page_number, "rank": rank, "reasons": reason_map.get(page_number, [])} + for rank, page_number in enumerate(selected, 1) + ] + return selected, details + + def _read_pdf_pages( path: Path, max_chars: int, @@ -402,15 +477,17 @@ def _read_pdf_pages( "used_ocr": False, }) used_chars += len(page_text) - if used_chars >= max_chars and "max_chars reached; remaining pages retain metadata but not full evidence text" not in warnings: - warnings.append("max_chars reached; remaining pages retain metadata but not full evidence text") + if used_chars >= max_chars and "document text exceeds evidence budget; evidence will be sampled across all pages" not in warnings: + warnings.append("document text exceeds evidence budget; evidence will be sampled across all pages") finally: pass ocr_pages: list[int] = [] ocr_page_durations: list[dict[str, Any]] = [] + prioritized: list[int] = [] + ocr_priority: list[dict[str, Any]] = [] if ocr_engine != "off": - prioritized = ocr_candidates[:max_ocr_pages] + prioritized, ocr_priority = _prioritize_ocr_candidates(pages, ocr_candidates, max_ocr_pages) for page_number in prioritized: if deadline_at is not None: remaining = deadline_at - time.monotonic() @@ -450,6 +527,8 @@ def _read_pdf_pages( return pages, { "metadata": metadata, "ocr_candidate_pages": ocr_candidates, + "ocr_priority_pages": prioritized, + "ocr_priority": ocr_priority, "ocr_pages": ocr_pages, "ocr_page_durations": ocr_page_durations, "warnings": warnings, @@ -556,7 +635,7 @@ def _evidence_id(page: int, text: str, bbox: Any, kind: str) -> str: def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], document_index: dict[str, Any], max_chars: int) -> list[dict[str, Any]]: section_positions = sorted(sections, key=lambda item: (item["page"], item.get("block_id", ""))) - evidence = [] + evidence: list[dict[str, Any]] = [] used = 0 for figure in document_index.get("figures", []): text = _clean(figure.get("caption", "")) @@ -575,17 +654,26 @@ def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], "char_count": len(text), }) used += len(text) + + block_candidates: list[dict[str, Any]] = [] for page in pages: for block in page.get("blocks", []): text = _clean(block.get("text", "")) - if not text or used >= max_chars: + if not text or block.get("kind") == "caption": continue - text = text[: max(0, max_chars - used)] - section = next((item["title"] for item in reversed(section_positions) if item["page"] <= page["page"]), None) - evidence.append({ + block_id = str(block.get("id") or "") + section = next( + ( + item["title"] + for item in reversed(section_positions) + if (int(item["page"]), str(item.get("block_id") or "")) <= (int(page["page"]), block_id) + ), + None, + ) + block_candidates.append({ "id": _evidence_id(int(page["page"]), text, block.get("bbox"), str(block.get("kind") or "paragraph")), "page": page["page"], - "block_id": block.get("id"), + "block_id": block_id, "bbox": block.get("bbox"), "section_hint": section, "kind": block.get("kind"), @@ -594,7 +682,101 @@ def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], "text": text, "char_count": len(text), }) - used += len(text) + + selected_ids: set[str] = set() + page_groups: dict[int, list[dict[str, Any]]] = {} + for item in block_candidates: + page_groups.setdefault(int(item["page"]), []).append(item) + + def add(item: dict[str, Any], max_length: int | None = None) -> bool: + nonlocal used + item_id = str(item["id"]) + if item_id in selected_ids or used >= max_chars: + return False + remaining = max_chars - used + if remaining <= 0: + return False + value = dict(item) + if max_length is not None and len(value["text"]) > max_length: + value["text"] = value["text"][:max_length].rstrip() + value["char_count"] = len(value["text"]) + value["id"] = _evidence_id(int(value["page"]), value["text"], value.get("bbox"), str(value.get("kind") or "paragraph")) + if len(value["text"]) > remaining: + if remaining < 40: + return False + value["text"] = value["text"][:remaining].rstrip() + value["char_count"] = len(value["text"]) + value["id"] = _evidence_id(int(value["page"]), value["text"], value.get("bbox"), str(value.get("kind") or "paragraph")) + item_id = str(value["id"]) + evidence.append(value) + selected_ids.add(str(item["id"])) + selected_ids.add(item_id) + used += len(value["text"]) + return True + + # Reserve a representative block for every page before spending the budget + # on long early sections. This keeps conclusions and appendices discoverable. + representative_quota = max(80, min(600, max(1, max_chars - used) // max(1, len(page_groups)))) + for page_number in sorted(page_groups): + candidates = page_groups[page_number] + representative = max( + candidates, + key=lambda item: ( + 1 if item.get("kind") in {"paragraph", "title"} else 0, + min(len(str(item.get("text") or "")), 600), + ), + ) + add(representative, max_length=representative_quota) + + section_priority = { + "abstract": 0, + "conclusion": 1, + "conclusions": 1, + "discussion": 1, + "method": 2, + "methods": 2, + "methodology": 2, + "approach": 2, + "architecture": 2, + "framework": 2, + "system overview": 2, + "introduction": 3, + "background": 3, + "experiment": 4, + "experiments": 4, + "evaluation": 4, + "results": 4, + } + + def priority(item: dict[str, Any]) -> tuple[int, int, str]: + section = str(item.get("section_hint") or "").casefold() + text = str(item.get("text") or "") + topology_definition = bool(re.search( + r"\b(?:has|have|consists? of|comprises?|contains?)\b.{0,80}\b(?:stages?|components?|modules?|steps?)\b" + r"|\b(?:assisted[ -]manual|semi[ -]automatic|fully automatic)\b" + r"|\b(?:input|image|text|query|prompt)\b.{0,100}\b(?:encoder|decoder|retriever|generator)\b" + r"|\b(?:encoder|decoder|retriever|generator)\b.{0,100}\b(?:output|prediction|representation|embedding)\b", + text, + re.IGNORECASE, + )) + rank = min((value for name, value in section_priority.items() if name in section), default=5) + if topology_definition: + rank = -1 + if item.get("kind") == "heading": + rank = min(rank, 2) + return rank, int(item.get("page") or 0), str(item.get("block_id") or "") + + for item in sorted(block_candidates, key=priority): + add(item) + + document_order = { + str(item["id"]): index + for index, item in enumerate(block_candidates) + } + caption_count = sum(1 for item in evidence if item.get("kind") == "caption") + caption_items = evidence[:caption_count] + body_items = sorted(evidence[caption_count:], key=lambda item: (int(item.get("page") or 0), document_order.get(str(item.get("id")), 10**9), str(item.get("block_id") or ""))) + evidence = caption_items + body_items for index, item in enumerate(evidence, 1): item["legacy_id"] = f"E{index:04d}" return evidence @@ -633,6 +815,8 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "readable_page_ratio": readable_ratio, "empty_or_low_text_pages": [page["page"] for page in pages if page not in readable], "ocr_candidate_pages": details.get("ocr_candidate_pages", []), + "ocr_priority_pages": details.get("ocr_priority_pages", []), + "ocr_priority": details.get("ocr_priority", []), "ocr_pages": details.get("ocr_pages", []), "ocr_page_durations": details.get("ocr_page_durations", []), "mean_ocr_confidence": mean_ocr_confidence, @@ -674,6 +858,10 @@ def parse_paper( evidence = _build_evidence(pages, sections, document_index, max_chars) all_text = "\n\n".join(page.get("text", "") for page in pages) extraction_report = _extraction_report(source, pages, document_index, details) + evidence_pages = {int(item.get("page") or 0) for item in evidence if item.get("page")} + extraction_report["evidence_char_count"] = sum(int(item.get("char_count") or 0) for item in evidence) + extraction_report["evidence_page_count"] = len(evidence_pages) + extraction_report["evidence_page_coverage_ratio"] = round(len(evidence_pages) / max(1, len(pages)), 4) return { "summary": "Paper parsed into a structured, page-aware evidence model.", "source_path": str(source), diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index a2e947e..9b3b45c 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 2 +DOCUMENT_CACHE_VERSION = 3 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index bff2ff8..ea26a42 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -553,7 +553,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 4, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 5, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -752,6 +752,18 @@ def prepare_paper_figure_contract( "contract_source": contract_source, "cache_hit": bool(planner_metadata.get("cached")), "document_cache_hit": document_cache_hit, + "extraction_quality": { + "pdf_type": parsed["extraction_report"].get("pdf_type"), + "page_count": parsed["extraction_report"].get("page_count"), + "readable_page_ratio": parsed["extraction_report"].get("readable_page_ratio"), + "evidence_page_coverage_ratio": parsed["extraction_report"].get("evidence_page_coverage_ratio"), + "evidence_char_count": parsed["extraction_report"].get("evidence_char_count"), + "section_coverage": parsed["extraction_report"].get("section_coverage", {}), + "missing_priority_sections": [name for name, present in parsed["extraction_report"].get("section_coverage", {}).items() if not present], + "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), + "ocr_scheduled_count": len(parsed["extraction_report"].get("ocr_priority_pages", [])), + "ocr_completed_count": len(parsed["extraction_report"].get("ocr_pages", [])), + }, "provider": provider_summary, "contract_completion": { "overview_term_coverage": completion_report.get("overview_term_coverage"), diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 6f97e3a..883f175 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -87,6 +87,8 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertEqual(result["aggregate"]["case_count"], 1) self.assertIn("mean_total_seconds", result["aggregate"]) self.assertIn("provider_success_rate", result["aggregate"]) + self.assertIn("mean_evidence_page_coverage_ratio", result["aggregate"]) + self.assertIn("ocr_scheduled_page_total", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) def test_paper_to_image_score_uses_hard_scientific_gates(self): diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 4da876c..b590d90 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -5,7 +5,7 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _prioritize_ocr_candidates, parse_paper from rfs.paper_to_image.inspection import inspect_paper @@ -21,6 +21,21 @@ def _pdf(self, draw) -> Path: self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) return target + def _multipage_pdf(self, page_texts: list[list[str]]) -> Path: + root = Path(tempfile.mkdtemp()) + target = root / "paper.pdf" + document = fitz.open() + for blocks in page_texts: + page = document.new_page(width=600, height=800) + top = 50 + for text in blocks: + page.insert_textbox((45, top, 555, top + 110), text, fontsize=10) + top += 120 + document.save(target) + document.close() + self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) + return target + def test_structured_pdf_preserves_page_blocks_and_captions(self): def draw(page): page.insert_text((50, 60), "1 Introduction", fontsize=14) @@ -152,6 +167,62 @@ def test_evidence_ids_are_stable_across_repeated_parses(self): self.assertEqual([item["id"] for item in first["evidence"]], [item["id"] for item in second["evidence"]]) self.assertTrue(all(item["id"].startswith("E_P") for item in first["evidence"])) + def test_evidence_budget_preserves_late_document_coverage(self): + path = self._multipage_pdf([ + ["Abstract", "Early context " * 90], + ["1 Introduction", "Our data engine has three stages:", "assisted-manual, semi-automatic, and fully automatic.", "Introduction evidence " * 24], + ["2 Method", "The encoder transforms inputs into representations. " * 18], + ["3 Experiments", "Experimental settings and evaluation metrics. " * 18], + ["4 Analysis", "Ablation and error analysis evidence. " * 18], + ["5 Conclusion", "The proposed method improves the final prediction while preserving scientific evidence. " * 12], + ]) + + parsed = parse_paper(path, max_chars=1800) + evidence_pages = {item["page"] for item in parsed["evidence"]} + + self.assertEqual(evidence_pages, {1, 2, 3, 4, 5, 6}) + self.assertTrue(any(item["page"] == 6 and "Conclusion" in item["text"] for item in parsed["evidence"])) + self.assertTrue(any("three stages" in item["text"] for item in parsed["evidence"])) + self.assertTrue(any("assisted-manual" in item["text"] for item in parsed["evidence"])) + self.assertEqual(parsed["extraction_report"]["evidence_page_coverage_ratio"], 1.0) + self.assertLessEqual(parsed["extraction_report"]["evidence_char_count"], 1800) + + def test_ocr_priority_spreads_fully_scanned_long_document(self): + pages = [{"page": index, "text": ""} for index in range(1, 13)] + + selected, details = _prioritize_ocr_candidates(pages, list(range(1, 13)), 6) + + self.assertEqual(selected, [1, 2, 3, 6, 9, 10]) + self.assertEqual([item["rank"] for item in details], [1, 2, 3, 4, 5, 6]) + + def test_ocr_priority_prefers_semantic_pages_before_coverage_anchors(self): + pages = [{"page": index, "text": ""} for index in range(1, 11)] + pages[3]["text"] = "Figure 1: Overview of the proposed architecture and system pipeline." + pages[7]["text"] = "8 Conclusion We summarize the method and its limitations." + + selected, details = _prioritize_ocr_candidates(pages, [1, 4, 6, 8, 10], 3) + + self.assertEqual(selected[:2], [4, 8]) + self.assertIn("overview_figure", details[0]["reasons"]) + self.assertIn("conclusion", details[1]["reasons"]) + + def test_scanned_pdf_report_records_adaptive_ocr_schedule(self): + path = self._multipage_pdf([[] for _ in range(12)]) + + def adapter(image_path, _lang): + page_number = int(Path(image_path).stem.rsplit("_", 1)[-1]) + return [{ + "text": f"Page {page_number} Abstract Method architecture evidence with enough reliable text for adaptive OCR scheduling and document recovery.", + "confidence": 0.97, + "quad": [[20, 20], [560, 20], [560, 70], [20, 70]], + }] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=6) + + self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 6, 9, 10]) + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 6, 9, 10]) + self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 6, 9, 10]) + if __name__ == "__main__": unittest.main() From 8aac6828a7b96e351ffa854a24c1823b0d28ebf8 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 03:46:51 +0800 Subject: [PATCH 25/74] Handle rotated and multi-column paper layouts --- README.md | 2 +- docs/paper-to-image.md | 4 +- rfs/evaluation/benchmarking.py | 2 + rfs/paper_to_image/analyzer.py | 106 ++++++++++++++++++++++++--- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 22 +++++- tests/unit/test_benchmarking.py | 1 + tests/unit/test_paper_analyzer.py | 36 +++++++++ tests/unit/test_paper_contract.py | 32 ++++++++ 9 files changed, 190 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ee5ceb8..f14b436 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index c1c732e..ed30e76 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -10,7 +10,7 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --j The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. Run all or selected paper benchmarks with aggregate reliability metrics: @@ -20,7 +20,7 @@ rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json ``` -`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, and OCR candidate/scheduled/completed totals. +`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, and OCR candidate/scheduled/completed totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. These repairs are paper-name agnostic and are covered by the NLP suite. When a paper exceeds the evidence character budget, the extractor reserves representative evidence from every page, then prioritizes topology-defining statements and Abstract, Conclusion, Method, Introduction, and Experiments content. This prevents long introductions or appendices from silently excluding later conclusions and framework definitions. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 25ced8d..a1c3efd 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -803,6 +803,8 @@ def percentile(values: list[float], fraction: float) -> float: "mean_readable_page_ratio": _mean(extraction_metric("readable_page_ratio")), "mean_evidence_page_coverage_ratio": _mean(extraction_metric("evidence_page_coverage_ratio")), "mean_evidence_char_count": _mean(extraction_metric("evidence_char_count")), + "max_detected_column_count": max((int(value) for value in extraction_metric("max_column_count")), default=0), + "multi_column_page_total": sum(int(value) for value in extraction_metric("multi_column_page_count")), "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 1ec6416..fb55952 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -65,40 +65,120 @@ def _block_kind(text: str, width_ratio: float = 0.0) -> str: return "paragraph" +def _column_groups(items: list[dict[str, Any]], page_width: float, max_columns: int = 3) -> list[list[dict[str, Any]]]: + if len(items) < 4: + return [items] + positioned = sorted(items, key=lambda item: float(item["bbox"][0])) + gaps = [ + (float(positioned[index + 1]["bbox"][0]) - float(positioned[index]["bbox"][0]), index) + for index in range(len(positioned) - 1) + ] + split_indexes = sorted(index for gap, index in sorted(gaps, reverse=True)[: max_columns - 1] if gap >= page_width * 0.12) + if not split_indexes: + return [items] + groups: list[list[dict[str, Any]]] = [] + start = 0 + for split in split_indexes: + groups.append(positioned[start:split + 1]) + start = split + 1 + groups.append(positioned[start:]) + if len(groups) > max_columns or any(len(group) < 2 for group in groups): + return [items] + + def median(values: list[float]) -> float: + ordered = sorted(values) + middle = len(ordered) // 2 + return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2.0 + + ordered_groups = sorted(groups, key=lambda group: median([float(item["bbox"][0]) for item in group])) + minimum_vertical_span = page_width * 0.12 + minimum_text = 40 + for group in ordered_groups: + vertical_span = max(float(item["bbox"][3]) for item in group) - min(float(item["bbox"][1]) for item in group) + text_count = sum(len(_normalized_compare_text(str(item.get("text") or ""))) for item in group) + if vertical_span < minimum_vertical_span or text_count < minimum_text: + return [items] + for left, right in zip(ordered_groups, ordered_groups[1:]): + left_right = median([float(item["bbox"][2]) for item in left]) + right_left = median([float(item["bbox"][0]) for item in right]) + if left_right > right_left + page_width * 0.04: + return [items] + return ordered_groups + + def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[list[dict[str, Any]], float]: if not blocks: return [], 0.0 usable = [item for item in blocks if _clean(item.get("text", ""))] if not usable: return [], 0.0 - midpoint = page_width / 2.0 spanning = [item for item in usable if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.62] - left = [item for item in usable if item not in spanning and (float(item["bbox"][0]) + float(item["bbox"][2])) / 2 <= midpoint] - right = [item for item in usable if item not in spanning and item not in left] - two_column = len(left) >= 2 and len(right) >= 2 - if not two_column: + non_spanning = [item for item in usable if item not in spanning] + detection_items = [ + item + for item in non_spanning + if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.20 + and len(_normalized_compare_text(str(item.get("text") or ""))) >= 20 + ] + detected_columns = _column_groups(detection_items, page_width) + if len(detected_columns) == 1: ordered = sorted(usable, key=lambda item: (round(float(item["bbox"][1]), 1), float(item["bbox"][0]))) confidence = 0.96 + for item in ordered: + item["column"] = 0 else: + def median_x0(column: list[dict[str, Any]]) -> float: + values = sorted(float(item["bbox"][0]) for item in column) + middle = len(values) // 2 + return values[middle] if len(values) % 2 else (values[middle - 1] + values[middle]) / 2.0 + + anchors = [median_x0(column) for column in detected_columns] + columns = [[] for _ in anchors] + for item in non_spanning: + column_index = min(range(len(anchors)), key=lambda index: abs(float(item["bbox"][0]) - anchors[index])) + columns[column_index].append(item) + for column_index, column in enumerate(columns): + for item in column: + item["column"] = column_index + for item in spanning: + item["column"] = -1 boundaries = sorted(spanning, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))) ordered = [] cursor = float("-inf") for boundary in boundaries + [None]: limit = float(boundary["bbox"][1]) if boundary else float("inf") segment = [item for item in usable if item not in spanning and cursor <= float(item["bbox"][1]) < limit] - ordered.extend(sorted((item for item in segment if item in left), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) - ordered.extend(sorted((item for item in segment if item in right), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) + for column in columns: + ordered.extend(sorted((item for item in segment if item in column), key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0])))) if boundary: ordered.append(boundary) cursor = float(boundary["bbox"][3]) seen: set[int] = set() ordered = [item for item in ordered if not (id(item) in seen or seen.add(id(item)))] - confidence = 0.88 if spanning else 0.82 + confidence = (0.88 if spanning else 0.82) if len(columns) == 2 else (0.84 if spanning else 0.78) for index, item in enumerate(ordered, 1): item["reading_order"] = index return ordered, confidence +def _rotate_blocks_to_page(blocks: list[dict[str, Any]], page: Any) -> list[dict[str, Any]]: + if not int(getattr(page, "rotation", 0) or 0): + return blocks + try: + import fitz + + matrix = page.rotation_matrix + rotated = [] + for item in blocks: + value = dict(item) + rect = fitz.Rect(*item["bbox"]) * matrix + value["bbox"] = [round(float(rect.x0), 3), round(float(rect.y0), 3), round(float(rect.x1), 3), round(float(rect.y1), 3)] + rotated.append(value) + return rotated + except Exception: + return blocks + + def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: records = [] payload = page.get_text("dict") @@ -445,10 +525,10 @@ def _read_pdf_pages( ocr_candidates: list[int] = [] try: for index, page in enumerate(document, 1): - raw_blocks = _pymupdf_line_blocks(page) + raw_blocks = _rotate_blocks_to_page(_pymupdf_line_blocks(page), page) ordered, order_confidence = _reading_order(raw_blocks, float(page.rect.width)) if len(_normalized_compare_text(" ".join(item["text"] for item in ordered))) < 80: - fallback_blocks = _pdfplumber_blocks(path, index) + fallback_blocks = _rotate_blocks_to_page(_pdfplumber_blocks(path, index), page) fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, float(page.rect.width)) if len(_normalized_compare_text(" ".join(item["text"] for item in fallback_ordered))) > len(_normalized_compare_text(" ".join(item["text"] for item in ordered))): ordered, order_confidence = fallback_ordered, min(fallback_confidence, 0.9) @@ -467,6 +547,8 @@ def _read_pdf_pages( "page": index, "width": round(float(page.rect.width), 3), "height": round(float(page.rect.height), 3), + "rotation": int(page.rotation or 0), + "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, "blocks": ordered, "text": page_text, "char_count": len(page_text), @@ -518,6 +600,7 @@ def _read_pdf_pages( "replacement_character_rate": _replacement_rate(ocr_text), "mojibake_rate": _mojibake_rate(ocr_text), "reading_order_confidence": order_confidence, + "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, "used_ocr": True, "ocr_engine": used_engine, }) @@ -825,6 +908,9 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "mojibake_rate": round(sum(page.get("mojibake_rate", 0.0) for page in pages) / max(1, len(pages)), 6), "cross_extractor_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, "reading_order_confidence": round(sum(page.get("reading_order_confidence", 0.0) for page in pages) / max(1, len(pages)), 4), + "max_column_count": max((int(page.get("column_count") or 1) for page in pages), default=1), + "multi_column_page_count": sum(int(page.get("column_count") or 1) > 1 for page in pages), + "rotated_pages": [int(page["page"]) for page in pages if int(page.get("rotation") or 0) % 360 != 0], "section_coverage": coverage, "figure_caption_count": len(index.get("figures", [])), "table_caption_count": len(index.get("tables", [])), diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 9b3b45c..80e666a 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 3 +DOCUMENT_CACHE_VERSION = 4 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index ea26a42..7930b32 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -125,6 +125,19 @@ def _stable_id(prefix: str, value: str, index: int) -> str: return f"{prefix}_{slug or index}" +def _statement_object(value: Any, fallback: Any = None) -> dict[str, Any]: + candidate = value if value is not None else fallback + if isinstance(candidate, dict): + item = dict(candidate) + text = str(item.get("text") or item.get("statement") or "unknown").strip() or "unknown" + item["text"] = text + item["evidence_ids"] = list(item.get("evidence_ids")) if isinstance(item.get("evidence_ids"), list) else [] + return item + if isinstance(candidate, str) and candidate.strip(): + return {"text": candidate.strip(), "evidence_ids": []} + return {"text": "unknown", "evidence_ids": [], "status": "unknown"} + + def _infer_topology(spec: dict[str, Any]) -> str: relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] if any("feedback" in str(item.get("type") or "").casefold() for item in relations): @@ -228,8 +241,8 @@ def _complete_from_overview_caption(spec: dict[str, Any], parsed: dict[str, Any] def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} - spec.setdefault("research_problem", summary.get("research_problem") or {"text": "unknown", "evidence_ids": []}) - spec.setdefault("central_claim", summary.get("central_claim") or {"text": "unknown", "evidence_ids": []}) + spec["research_problem"] = _statement_object(spec.get("research_problem"), summary.get("research_problem")) + spec["central_claim"] = _statement_object(spec.get("central_claim"), summary.get("central_claim")) if not isinstance(spec.get("inputs"), list) or not spec.get("inputs"): spec["inputs"] = summary.get("inputs") if isinstance(summary.get("inputs"), list) else [] if not isinstance(spec.get("outputs"), list) or not spec.get("outputs"): @@ -553,7 +566,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 5, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 6, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -758,6 +771,9 @@ def prepare_paper_figure_contract( "readable_page_ratio": parsed["extraction_report"].get("readable_page_ratio"), "evidence_page_coverage_ratio": parsed["extraction_report"].get("evidence_page_coverage_ratio"), "evidence_char_count": parsed["extraction_report"].get("evidence_char_count"), + "max_column_count": parsed["extraction_report"].get("max_column_count"), + "multi_column_page_count": parsed["extraction_report"].get("multi_column_page_count"), + "rotated_pages": parsed["extraction_report"].get("rotated_pages", []), "section_coverage": parsed["extraction_report"].get("section_coverage", {}), "missing_priority_sections": [name for name, present in parsed["extraction_report"].get("section_coverage", {}).items() if not present], "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 883f175..0d45e00 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -89,6 +89,7 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("provider_success_rate", result["aggregate"]) self.assertIn("mean_evidence_page_coverage_ratio", result["aggregate"]) self.assertIn("ocr_scheduled_page_total", result["aggregate"]) + self.assertIn("max_detected_column_count", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) def test_paper_to_image_score_uses_hard_scientific_gates(self): diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index b590d90..62e2f87 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -76,6 +76,42 @@ def draw(page): self.assertLess(text.index("LEFT COLUMN FIRST"), text.index("LEFT COLUMN THIRD")) self.assertLess(text.index("LEFT COLUMN THIRD"), text.index("RIGHT COLUMN SECOND")) self.assertGreaterEqual(parsed["pages"][0]["reading_order_confidence"], 0.8) + self.assertEqual(parsed["pages"][0]["column_count"], 2) + + def test_three_column_blocks_are_sorted_column_major(self): + def draw(page): + page.insert_textbox((400, 100, 560, 170), "COLUMN THREE TOP complete scientific paragraph.", fontsize=9) + page.insert_textbox((220, 100, 380, 170), "COLUMN TWO TOP complete scientific paragraph.", fontsize=9) + page.insert_textbox((40, 100, 200, 170), "COLUMN ONE TOP complete scientific paragraph.", fontsize=9) + page.insert_textbox((400, 220, 560, 290), "COLUMN THREE BOTTOM complete scientific paragraph.", fontsize=9) + page.insert_textbox((220, 220, 380, 290), "COLUMN TWO BOTTOM complete scientific paragraph.", fontsize=9) + page.insert_textbox((40, 220, 200, 290), "COLUMN ONE BOTTOM complete scientific paragraph.", fontsize=9) + page.insert_textbox((40, 40, 560, 75), "3 Method Architecture and System Overview Across Three Columns", fontsize=13) + + parsed = parse_paper(self._pdf(draw)) + text = parsed["pages"][0]["text"] + + self.assertLess(text.index("COLUMN ONE TOP"), text.index("COLUMN ONE BOTTOM")) + self.assertLess(text.index("COLUMN ONE BOTTOM"), text.index("COLUMN TWO TOP")) + self.assertLess(text.index("COLUMN TWO BOTTOM"), text.index("COLUMN THREE TOP")) + self.assertEqual(parsed["pages"][0]["column_count"], 3) + self.assertEqual(parsed["extraction_report"]["max_column_count"], 3) + + def test_rotated_page_uses_display_coordinate_space(self): + def draw(page): + page.insert_textbox((40, 40, 560, 85), "Rotated Paper Architecture Overview", fontsize=14) + page.insert_textbox((40, 120, 270, 220), "First scientific block with enough evidence for extraction.", fontsize=10) + page.insert_textbox((330, 120, 560, 220), "Second scientific block with enough evidence for extraction.", fontsize=10) + page.set_rotation(90) + + parsed = parse_paper(self._pdf(draw)) + page = parsed["pages"][0] + + self.assertEqual(page["rotation"], 90) + self.assertEqual((page["width"], page["height"]), (800.0, 600.0)) + self.assertTrue(all(0 <= block["bbox"][0] <= block["bbox"][2] <= page["width"] for block in page["blocks"])) + self.assertTrue(all(0 <= block["bbox"][1] <= block["bbox"][3] <= page["height"] for block in page["blocks"])) + self.assertEqual(parsed["extraction_report"]["rotated_pages"], [1]) def test_unicode_normalization_keeps_math_symbols(self): def draw(page): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 4c4718f..ad59473 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -42,6 +42,38 @@ def test_contract_adds_topology_labels_and_separate_flows(self): self.assertEqual(overlay["connectors"][0]["source"], "output") self.assertTrue(any(item["text"] == "Refined Output" for item in overlay["labels"])) + def test_contract_normalizes_scalar_claim_and_grounds_it_from_evidence(self): + parsed = { + "evidence": [ + { + "id": "E0001", + "page": 3, + "text": "The same architecture is used for pre-training and fine-tuning with task-specific output layers.", + "confidence": 1.0, + } + ] + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": "Unified architecture for pre-training and fine-tuning.", + "central_claim": "The same architecture is used for pre-training and fine-tuning with task-specific output layers.", + "modules": [{"id": "encoder", "name": "Encoder", "evidence_ids": ["E0001"]}], + "inputs": [], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + validation = validate_plan_grounding(plan, parsed) + + self.assertEqual(spec["central_claim"]["evidence_ids"], ["E0001"]) + self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0001"]) + self.assertTrue(validation["ok"]) + def test_grounding_rejects_relation_without_evidence(self): plan = { "figure_specification": { From 0e1a81f5fcf639e25809456c15f4963fac0a0cfb Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 05:42:56 +0800 Subject: [PATCH 26/74] Speed up OCR and preserve sampled scan contracts --- README.md | 2 +- docs/paper-to-image.md | 6 +- rfs/cli.py | 8 +- rfs/evaluation/benchmarking.py | 2 + rfs/paper_to_image/analyzer.py | 171 ++++++++++++++++++---- rfs/paper_to_image/contract_completion.py | 32 ++++ rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/planner.py | 3 + rfs/paper_to_image/preparation.py | 12 +- rfs/paper_to_image/workflow.py | 2 + rfs/reference_text_extractor.py | 31 +++- tests/integration/test_paper_to_image.py | 58 ++++++++ tests/unit/test_benchmarking.py | 1 + tests/unit/test_paper_analyzer.py | 89 ++++++++++- tests/unit/test_paper_contract.py | 26 ++++ 15 files changed, 403 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index f14b436..c4812aa 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. Cross-parser gating compares supported lexical content rather than raw string order, so normal two-column reordering, equation layout, and hidden accessibility text do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index ed30e76..15ede99 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -10,7 +10,9 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --j The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. + +If a long fully scanned paper cannot reach the 60% full-document readability gate within the deadline, the workflow may continue only when at least six high-confidence sampled pages cover both Abstract and Method. The result records `semantic_scope: sampled_pages_only`, includes an explicit uncertainty about unprocessed pages, and can never be production-ready. Missing Abstract/Method coverage or weak OCR still stops semantic planning. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. Run all or selected paper benchmarks with aggregate reliability metrics: @@ -20,7 +22,7 @@ rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json ``` -`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, and OCR candidate/scheduled/completed totals. +`fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. These repairs are paper-name agnostic and are covered by the NLP suite. When a paper exceeds the evidence character budget, the extractor reserves representative evidence from every page, then prioritizes topology-defining statements and Abstract, Conclusion, Method, Introduction, and Experiments content. This prevents long introductions or appendices from silently excluding later conclusions and framework definitions. diff --git a/rfs/cli.py b/rfs/cli.py index 42eaa84..1745783 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -78,7 +78,13 @@ def _doctor() -> dict: "pdftotext": {"available": bool(shutil.which("pdftotext")), "path": shutil.which("pdftotext")}, "pdftoppm": {"available": bool(shutil.which("pdftoppm")), "path": shutil.which("pdftoppm")}, "easyocr": {"available": importlib.util.find_spec("easyocr") is not None, "models": easyocr_models}, - "rapidocr": {"available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, "runtime": "onnxruntime", "thread_policy": "1 intra-op / 1 inter-op"}, + "rapidocr": { + "available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, + "runtime": "onnxruntime", + "worker_policy": "up to 3 page workers; 2 intra-op threads for a single page", + "worker_override": os.getenv("RFS_OCR_WORKERS"), + "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), + }, "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, "fast_contract_cache": {"available": True, "path": str(Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")))}, } diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index a1c3efd..2c3fc86 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -808,6 +808,8 @@ def percentile(values: list[float], fraction: float) -> float: "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), + "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), + "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), "provider_call_count": len(provider_calls), "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index fb55952..4e7bace 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -2,12 +2,15 @@ import hashlib import json +import os import re import shutil import subprocess import tempfile import time import unicodedata +from collections import Counter +from concurrent.futures import ThreadPoolExecutor from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable @@ -42,6 +45,16 @@ def _normalized_compare_text(text: str) -> str: return re.sub(r"[^a-z0-9]+", "", _clean(text).casefold()) +def _token_overlap_agreement(left: str, right: str) -> float | None: + token_pattern = re.compile(r"[a-z]{3,}|[\u3400-\u9fff]{2,}", re.IGNORECASE) + left_tokens = token_pattern.findall(_clean(left).casefold()) + right_tokens = token_pattern.findall(_clean(right).casefold()) + if not left_tokens or not right_tokens: + return None + overlap = sum((Counter(left_tokens) & Counter(right_tokens)).values()) + return round(overlap / max(1, len(left_tokens)), 4) + + def _replacement_rate(text: str) -> float: value = str(text or "") return round(value.count("\ufffd") / max(1, len(value)), 6) @@ -262,7 +275,7 @@ def _pdf_metadata(path: Path) -> dict[str, Any]: return {"page_count": None, "encrypted": None, "metadata": {}, "warning": "pypdf unavailable"} -def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None) -> tuple[list[dict[str, Any]], str]: +def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str]: if adapter: return list(adapter(image_path, lang) or []), "adapter" from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr @@ -279,7 +292,7 @@ def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | N except Exception: requested = "paddle" if requested == "rapidocr": - return run_rapidocr(image_path, lang), "rapidocr" + return run_rapidocr(image_path, lang, threads=rapidocr_threads), "rapidocr" if requested == "easyocr": return run_easyocr(image_path, lang), "easyocr" if requested == "paddle": @@ -387,14 +400,45 @@ def _assign_ocr_parent_blocks(lines: list[dict[str, Any]], image_width: float) - return ordered -def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None) -> tuple[list[dict[str, Any]], str, str | None]: +def _filter_ocr_margin_noise(records: list[dict[str, Any]], image_width: float) -> tuple[list[dict[str, Any]], int]: + anchors = [ + item + for item in records + if len(_normalized_compare_text(str(item.get("text") or ""))) >= 20 + and float(item["bbox"][2]) - float(item["bbox"][0]) >= image_width * 0.30 + ] + if len(anchors) < 3: + return records, 0 + left = sorted(float(item["bbox"][0]) for item in anchors)[max(0, len(anchors) // 10 - 1)] + right_values = sorted(float(item["bbox"][2]) for item in anchors) + right = right_values[min(len(right_values) - 1, len(right_values) - max(1, len(anchors) // 10))] + margin = image_width * 0.035 + kept = [] + removed = 0 + for item in records: + bbox = item.get("bbox") or [0, 0, 0, 0] + outside = float(bbox[2]) < left - margin or float(bbox[0]) > right + margin + width = max(1.0, float(bbox[2]) - float(bbox[0])) + height = max(1.0, float(bbox[3]) - float(bbox[1])) + short = len(_normalized_compare_text(str(item.get("text") or ""))) < 12 + vertical = height > width * 1.15 + outer_band = float(bbox[2]) < image_width * 0.12 or float(bbox[0]) > image_width * 0.88 + compact_margin_fragment = outer_band and short and (vertical or width < image_width * 0.08) + if (outside and (short or vertical)) or compact_margin_fragment: + removed += 1 + continue + kept.append(item) + return kept, removed + + +def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, str | None, dict[str, Any]]: try: with tempfile.TemporaryDirectory() as temp: target = Path(temp) / f"page_{page_number:03d}.png" dpi = 72 if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 pixmap = page.get_pixmap(dpi=dpi, alpha=False) pixmap.save(str(target)) - records, used_engine = _ocr_records(target, engine, lang, adapter) + records, used_engine = _ocr_records(target, engine, lang, adapter, rapidocr_threads=rapidocr_threads) if used_engine != "rapidocr": records = _group_ocr_words(records, float(pixmap.width)) else: @@ -407,6 +451,7 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call ys = [float(point[1]) for point in quad] normalized_records.append({"bbox": [min(xs), min(ys), max(xs), max(ys)], "text": _clean(record.get("text", "")), "confidence": float(record.get("confidence") or 0.0)}) records = _assign_ocr_parent_blocks(normalized_records, float(pixmap.width)) + records, margin_noise_removed = _filter_ocr_margin_noise(records, float(pixmap.width)) blocks = [] scale_x = float(page.rect.width) / max(1, pixmap.width) scale_y = float(page.rect.height) / max(1, pixmap.height) @@ -421,9 +466,24 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call "confidence": round(float(record.get("confidence") or 0.0), 4), "parent_block": record.get("parent_block"), }) - return blocks, used_engine, None + return blocks, used_engine, None, {"margin_noise_removed": margin_noise_removed} except Exception as exc: - return [], engine, str(exc) + return [], engine, str(exc), {"margin_noise_removed": 0} + + +def _rapidocr_worker_count(engine: str, adapter: Callable | None, page_count: int) -> int: + if adapter is not None or page_count < 2 or engine not in {"auto", "rapidocr"}: + return 1 + if engine == "auto": + try: + import rapidocr_onnxruntime # noqa: F401 + except Exception: + return 1 + try: + configured = int(os.getenv("RFS_OCR_WORKERS") or 3) + except ValueError: + configured = 3 + return max(1, min(int(page_count), configured, int(os.cpu_count() or 1))) def _prioritize_ocr_candidates( @@ -539,8 +599,9 @@ def _read_pdf_pages( poppler_page = poppler_text[index - 1] if index - 1 < len(poppler_text) else "" native_norm = _normalized_compare_text(page_text) poppler_norm = _normalized_compare_text(poppler_page) - agreement = round(SequenceMatcher(None, native_norm[:20000], poppler_norm[:20000]).ratio(), 4) if native_norm and poppler_norm else None - abnormal = len(native_norm) < 80 or _replacement_rate(page_text) > 0.005 or _mojibake_rate(page_text) > 0.005 or (agreement is not None and agreement < 0.35) + order_agreement = round(SequenceMatcher(None, native_norm[:20000], poppler_norm[:20000]).ratio(), 4) if native_norm and poppler_norm else None + agreement = _token_overlap_agreement(page_text, poppler_page) + abnormal = len(native_norm) < 80 or _replacement_rate(page_text) > 0.005 or _mojibake_rate(page_text) > 0.005 or (agreement is not None and agreement < 0.65) if abnormal: ocr_candidates.append(index) pages.append({ @@ -555,6 +616,7 @@ def _read_pdf_pages( "replacement_character_rate": _replacement_rate(page_text), "mojibake_rate": _mojibake_rate(page_text), "poppler_agreement": agreement, + "poppler_order_agreement": order_agreement, "reading_order_confidence": order_confidence, "used_ocr": False, }) @@ -568,30 +630,23 @@ def _read_pdf_pages( ocr_page_durations: list[dict[str, Any]] = [] prioritized: list[int] = [] ocr_priority: list[dict[str, Any]] = [] + ocr_worker_count = 1 if ocr_engine != "off": prioritized, ocr_priority = _prioritize_ocr_candidates(pages, ocr_candidates, max_ocr_pages) - for page_number in prioritized: - if deadline_at is not None: - remaining = deadline_at - time.monotonic() - completed_times = [float(item["elapsed_seconds"]) for item in ocr_page_durations if isinstance(item.get("elapsed_seconds"), (int, float))] - estimated = (sum(completed_times) / len(completed_times) * 1.25) if completed_times else 45.0 - if remaining < estimated + 20: - warnings.append("OCR stopped to preserve deadline validation budget") - break - page = document[page_number - 1] - ocr_started = time.monotonic() - blocks, used_engine, error = _ocr_page(page, page_number, ocr_engine, ocr_lang, ocr_adapter) - ocr_elapsed = round(time.monotonic() - ocr_started, 3) - ocr_page_durations.append({"page": page_number, "engine": used_engine, "elapsed_seconds": ocr_elapsed, "success": not bool(error)}) + ocr_worker_count = _rapidocr_worker_count(ocr_engine, ocr_adapter, len(prioritized)) + + def consume_ocr_result(page_number: int, result: tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], ocr_elapsed: float) -> None: + blocks, used_engine, error, ocr_diagnostics = result + ocr_page_durations.append({"page": page_number, "engine": used_engine, "elapsed_seconds": round(ocr_elapsed, 3), "success": not bool(error), **ocr_diagnostics}) if error: warnings.append(f"page {page_number} OCR unavailable: {error}") - continue - ordered, order_confidence = _reading_order(blocks, float(page.rect.width)) + return + current = pages[page_number - 1] + ordered, order_confidence = _reading_order(blocks, float(current["width"])) for block_index, block in enumerate(ordered, 1): block["id"] = f"P{page_number:03d}_B{block_index:03d}" - block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) + block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(current["width"]))) ocr_text = "\n\n".join(item["text"] for item in ordered) - current = pages[page_number - 1] if len(_normalized_compare_text(ocr_text)) > len(_normalized_compare_text(current["text"])): current.update({ "blocks": ordered, @@ -603,8 +658,50 @@ def _read_pdf_pages( "column_count": max((int(item.get("column", 0)) for item in ordered), default=0) + 1, "used_ocr": True, "ocr_engine": used_engine, + "ocr_margin_noise_removed": int(ocr_diagnostics.get("margin_noise_removed") or 0), }) ocr_pages.append(page_number) + + def deadline_allows_wave() -> bool: + if deadline_at is not None: + remaining = deadline_at - time.monotonic() + completed_times = [float(item["elapsed_seconds"]) for item in ocr_page_durations if isinstance(item.get("elapsed_seconds"), (int, float))] + estimated = (max(completed_times) * 1.25) if completed_times else 45.0 + if remaining < estimated + 20: + warnings.append("OCR stopped to preserve deadline validation budget") + return False + return True + + if ocr_worker_count > 1: + def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]: + ocr_started = time.monotonic() + try: + local_document = fitz.open(str(path)) + try: + result = _ocr_page(local_document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=1) + return result, time.monotonic() - ocr_started + finally: + local_document.close() + except Exception as exc: + return ([], ocr_engine, str(exc), {"margin_noise_removed": 0}), time.monotonic() - ocr_started + + with ThreadPoolExecutor(max_workers=ocr_worker_count) as pool: + for start in range(0, len(prioritized), ocr_worker_count): + if not deadline_allows_wave(): + break + batch = prioritized[start:start + ocr_worker_count] + futures = {page_number: pool.submit(isolated_ocr, page_number) for page_number in batch} + for page_number in batch: + result, ocr_elapsed = futures[page_number].result() + consume_ocr_result(page_number, result, ocr_elapsed) + else: + rapidocr_threads = 2 if ocr_engine in {"auto", "rapidocr"} and ocr_adapter is None else 1 + for page_number in prioritized: + if not deadline_allows_wave(): + break + ocr_started = time.monotonic() + result = _ocr_page(document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=rapidocr_threads) + consume_ocr_result(page_number, result, time.monotonic() - ocr_started) document.close() elapsed = round(time.monotonic() - started, 3) return pages, { @@ -614,6 +711,7 @@ def _read_pdf_pages( "ocr_priority": ocr_priority, "ocr_pages": ocr_pages, "ocr_page_durations": ocr_page_durations, + "ocr_worker_count": ocr_worker_count, "warnings": warnings, "poppler_available": bool(poppler_text), "elapsed_seconds": elapsed, @@ -870,7 +968,9 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st readable_ratio = round(len(readable) / max(1, len(pages)), 4) ocr_count = sum(1 for page in pages if page.get("used_ocr")) candidate_count = len(details.get("ocr_candidate_pages", [])) - pdf_type = "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count or (pages and candidate_count == len(pages)) else "born_digital" + fully_scanned_source = bool(pages and candidate_count == len(pages)) + partial_scan = bool(fully_scanned_source and 0 < ocr_count < len(pages)) + pdf_type = "scanned" if fully_scanned_source else "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" coverage = _section_coverage(pages, index.get("sections", [])) warnings = list(details.get("warnings", [])) ocr_blocks = [block for page in pages if page.get("used_ocr") for block in page.get("blocks", []) if isinstance(block, dict)] @@ -878,8 +978,18 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st mean_ocr_confidence = round(sum(ocr_confidences) / max(1, len(ocr_confidences)), 4) if ocr_blocks else None short_ocr_blocks = [block for block in ocr_blocks if len(_normalized_compare_text(str(block.get("text") or ""))) < 3] short_ocr_block_ratio = round(len(short_ocr_blocks) / max(1, len(ocr_blocks)), 4) if ocr_blocks else None - if readable_ratio < 0.6: + sampled_scan_ready = bool( + partial_scan + and len(readable) >= min(6, len(pages)) + and coverage.get("abstract") + and coverage.get("method") + and mean_ocr_confidence is not None + and mean_ocr_confidence >= 0.75 + ) + if readable_ratio < 0.6 and not sampled_scan_ready: warnings.append("fewer than 60% of pages contain reliable text") + if sampled_scan_ready: + warnings.append("fully scanned document was only partially OCRed; semantic scope is limited to sampled pages") if mean_ocr_confidence is not None and mean_ocr_confidence < 0.5: warnings.append("mean OCR confidence is below 0.50") if short_ocr_block_ratio is not None and short_ocr_block_ratio > 0.3: @@ -887,7 +997,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st if not coverage.get("abstract") or not coverage.get("method"): warnings.append("abstract or method-like section was not confidently located") agreements = [page["poppler_agreement"] for page in pages if isinstance(page.get("poppler_agreement"), (int, float))] - report_status = "fail" if readable_ratio < 0.6 or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) else "warning" if warnings else "pass" + report_status = "fail" if (readable_ratio < 0.6 and not sampled_scan_ready) or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) else "warning" if warnings else "pass" return { "summary": "PDF extraction quality and fallback report.", "status": report_status, @@ -896,12 +1006,17 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "page_count": len(pages), "readable_page_count": len(readable), "readable_page_ratio": readable_ratio, + "semantic_scope": "sampled_pages_only" if partial_scan else "full_document", + "scientific_scope_complete": not partial_scan, + "sampled_scan_ready": sampled_scan_ready, "empty_or_low_text_pages": [page["page"] for page in pages if page not in readable], "ocr_candidate_pages": details.get("ocr_candidate_pages", []), "ocr_priority_pages": details.get("ocr_priority_pages", []), "ocr_priority": details.get("ocr_priority", []), "ocr_pages": details.get("ocr_pages", []), "ocr_page_durations": details.get("ocr_page_durations", []), + "ocr_worker_count": int(details.get("ocr_worker_count") or 1), + "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), "mean_ocr_confidence": mean_ocr_confidence, "short_ocr_block_ratio": short_ocr_block_ratio, "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 048dd02..068bfd9 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -367,12 +367,43 @@ def _ground_statement(item: object, relevant: list[dict[str, Any]]) -> list[str] return evidence_ids +def _ground_declared_entities(spec: dict[str, Any], relevant: list[dict[str, Any]]) -> list[str]: + grounded: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict) or item.get("evidence_ids"): + continue + label = _label(item) + normalized_label = _normalized(label) + if len(normalized_label) < 8 or len(re.findall(r"[a-z0-9]+", label.casefold())) < 2: + continue + match: list[dict[str, Any]] = [] + for start in range(len(relevant)): + page = relevant[start].get("page") + for size in range(1, 4): + window = relevant[start:start + size] + if len(window) != size or any(value.get("page") != page for value in window): + break + combined = _normalized(" ".join(str(value.get("text") or "") for value in window)) + if normalized_label in combined: + match = window + break + if match: + break + evidence_ids = [str(value.get("id")) for value in match if value.get("id")] + if evidence_ids: + item["evidence_ids"] = evidence_ids + grounded.append(str(item.get("id") or label)) + return grounded + + def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: selected, relevant = _relevant_evidence(parsed) grounded_statements = { field: _ground_statement(spec.get(field), relevant) for field in ("research_problem", "central_claim") } + grounded_entities = _ground_declared_entities(spec, relevant) found: dict[str, dict[str, Any]] = {} added_entities: list[str] = [] upgraded_entities: list[str] = [] @@ -474,5 +505,6 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) "added_relations": added_relations, "repaired_relations": repaired_relations, "grounded_statements": {field: ids for field, ids in grounded_statements.items() if ids}, + "grounded_entities": grounded_entities, "rules_are_evidence_gated": True, } diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 80e666a..9d1a754 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 4 +DOCUMENT_CACHE_VERSION = 9 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index cb49af1..6a0b443 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -222,6 +222,9 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in Candidate overview captions: {json.dumps(candidates, ensure_ascii=False, indent=2)} +Extraction scope (do not claim coverage beyond this scope): +{json.dumps({key: parsed.get('extraction_report', {}).get(key) for key in ('pdf_type', 'semantic_scope', 'readable_page_ratio', 'ocr_pages', 'warnings')}, ensure_ascii=False, indent=2)} + User preferences relevant to wording only: {json.dumps({key: preferences.get(key) for key in ('language', 'must_show', 'must_not_show')}, ensure_ascii=False, indent=2)} diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 7930b32..05b1ca6 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -429,6 +429,8 @@ def choose_input_target(label: str) -> dict[str, Any] | None: valid_records = [record for record in evidence if record] if valid_records and all(float(record.get("confidence", 1.0)) < 0.75 and str(record.get("source") or "").casefold() in {"easyocr", "paddle", "adapter"} for record in valid_records): uncertainties.append(f"{_item_id(item, field, 0)} relies only on low-confidence OCR evidence") + if parsed.get("extraction_report", {}).get("semantic_scope") == "sampled_pages_only": + uncertainties.append("The source is a long scanned document; only scheduled sample pages were OCRed, so unprocessed pages may contain additional scientific details.") spec["uncertainties"] = list(dict.fromkeys(str(item) for item in uncertainties if str(item).strip())) spec.setdefault("forbidden_inventions", []) plan["figure_specification"] = spec @@ -566,7 +568,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 6, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 12, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -728,7 +730,8 @@ def prepare_paper_figure_contract( elapsed = round(time.monotonic() - started, 3) deadline_reached = time.monotonic() >= deadline_at - production_ready = bool((review_metadata.get("mode") == "vlm" or planner_metadata.get("mode") == "vlm") and planning_validation.get("ok") and not deadline_reached) + scientific_scope_complete = bool(parsed["extraction_report"].get("scientific_scope_complete", True)) + production_ready = bool((review_metadata.get("mode") == "vlm" or planner_metadata.get("mode") == "vlm") and planning_validation.get("ok") and not deadline_reached and scientific_scope_complete) has_warning = bool(planner_metadata.get("warning") or review_metadata.get("warning") or parsed["extraction_report"].get("status") == "warning") status = "complete" if production_ready and not has_warning else "completed_with_warnings" contract_source = "cache" if planner_metadata.get("cached") else "vlm" if planner_metadata.get("mode") == "vlm" else "vlm_review_deterministic_compile" if planner_metadata.get("mode") == "review_grounded_vlm" else "deterministic_evidence_rules" @@ -769,6 +772,9 @@ def prepare_paper_figure_contract( "pdf_type": parsed["extraction_report"].get("pdf_type"), "page_count": parsed["extraction_report"].get("page_count"), "readable_page_ratio": parsed["extraction_report"].get("readable_page_ratio"), + "semantic_scope": parsed["extraction_report"].get("semantic_scope", "full_document"), + "scientific_scope_complete": scientific_scope_complete, + "sampled_scan_ready": parsed["extraction_report"].get("sampled_scan_ready", False), "evidence_page_coverage_ratio": parsed["extraction_report"].get("evidence_page_coverage_ratio"), "evidence_char_count": parsed["extraction_report"].get("evidence_char_count"), "max_column_count": parsed["extraction_report"].get("max_column_count"), @@ -779,6 +785,8 @@ def prepare_paper_figure_contract( "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), "ocr_scheduled_count": len(parsed["extraction_report"].get("ocr_priority_pages", [])), "ocr_completed_count": len(parsed["extraction_report"].get("ocr_pages", [])), + "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), + "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), }, "provider": provider_summary, "contract_completion": { diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 957b494..2f0b227 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -68,6 +68,8 @@ def run_paper_to_image( planner_metadata = prepared["planner_metadata"] planning_validation = prepared["planning_validation"] production_mode = asset_mode == "image2" + if production_mode and not parsed.get("extraction_report", {}).get("scientific_scope_complete", True): + raise RuntimeError("Production Image2 generation requires full-document scientific scope; sampled scanned-paper contracts are engineering-only") coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=production_mode) write_json(root / "review_coverage_report.json", coverage) if production_mode and review_metadata.get("mode") != "vlm": diff --git a/rfs/reference_text_extractor.py b/rfs/reference_text_extractor.py index 48bd7b2..68ccda0 100644 --- a/rfs/reference_text_extractor.py +++ b/rfs/reference_text_extractor.py @@ -2,6 +2,7 @@ import math import os +import threading from pathlib import Path from typing import Any, Callable @@ -10,7 +11,14 @@ OCR_LOW_CONFIDENCE = 0.65 _EASYOCR_READERS: dict[tuple[str, ...], Any] = {} -_RAPIDOCR_ENGINE: Any | None = None +_RAPIDOCR_LOCAL = threading.local() + + +def _positive_ocr_setting(explicit: int | None, env_name: str, default: int) -> int: + try: + return max(1, int(explicit if explicit is not None else os.getenv(env_name) or default)) + except (TypeError, ValueError): + return max(1, int(default)) def _round4(value: float) -> float: @@ -181,14 +189,25 @@ def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: return records -def run_rapidocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: +def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = None, batch_size: int | None = None) -> list[dict[str, Any]]: del lang - global _RAPIDOCR_ENGINE - if _RAPIDOCR_ENGINE is None: + thread_count = _positive_ocr_setting(threads, "RFS_RAPIDOCR_THREADS", 1) + recognition_batch = _positive_ocr_setting(batch_size, "RFS_RAPIDOCR_BATCH", 6) + engines = getattr(_RAPIDOCR_LOCAL, "engines", None) + if engines is None: + engines = {} + _RAPIDOCR_LOCAL.engines = engines + engine_key = (thread_count, recognition_batch) + if engine_key not in engines: from rapidocr_onnxruntime import RapidOCR # type: ignore - _RAPIDOCR_ENGINE = RapidOCR(intra_op_num_threads=1, inter_op_num_threads=1, det_limit_side_len=512) - result, _timings = _RAPIDOCR_ENGINE(str(image_path)) + engines[engine_key] = RapidOCR( + intra_op_num_threads=thread_count, + inter_op_num_threads=1, + det_limit_side_len=512, + rec_batch_num=recognition_batch, + ) + result, _timings = engines[engine_key](str(image_path)) records = [] for item in result or []: if not isinstance(item, (list, tuple)) or len(item) < 3: diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index e428d46..7742f87 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -8,6 +8,7 @@ from pathlib import Path from unittest.mock import patch +import fitz from PIL import Image from rfs.cli import _doctor, build_parser @@ -159,6 +160,63 @@ def test_fast_framework_prompt_writes_contract_without_generation(self): overlay = json.loads((out / "overlay_spec.json").read_text(encoding="utf-8")) self.assertTrue(overlay["labels"]) + def test_long_scanned_paper_produces_explicit_sampled_engineering_contract(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paper = root / "scan.pdf" + document = fitz.open() + for _ in range(12): + document.new_page(width=600, height=800) + document.save(paper) + document.close() + + def adapter(image_path, _lang): + page_number = int(Path(image_path).stem.rsplit("_", 1)[-1]) + return [{ + "text": f"Page {page_number} Abstract Method Input Image enters a CNN Backbone and Transformer Encoder Decoder to produce Class Predictions and Bounding Box Predictions with Bipartite Matching.", + "confidence": 0.97, + "quad": [[20, 20], [920, 20], [920, 100], [20, 100]], + }] + + out = root / "fast" + result = run_fast_framework_prompt( + paper=paper, + out=out, + deadline_seconds=180, + planner_mode="heuristic", + ocr_engine="easyocr", + ocr_adapter=adapter, + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["status"], "completed_with_warnings") + self.assertFalse(result["production_ready"]) + self.assertEqual(result["extraction_quality"]["semantic_scope"], "sampled_pages_only") + self.assertTrue(result["extraction_quality"]["sampled_scan_ready"]) + self.assertTrue(any("sample pages" in item for item in result["uncertainties"])) + + def test_production_image_generation_rejects_sampled_scan_scope(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + prepared = { + "ok": True, + "root": root, + "paper": str(root / "paper.pdf"), + "archived_positive": [], + "archived_negative": [], + "preferences": {"aspect_ratio": "16:9"}, + "parsed": {"extraction_report": {"scientific_scope_complete": False}}, + "selected_domain": {"id": "general"}, + "paper_review": {}, + "review_metadata": {"mode": "vlm"}, + "plan": _plan(), + "planner_metadata": {"mode": "vlm"}, + "planning_validation": {"ok": True}, + } + with patch("rfs.paper_to_image.workflow.prepare_paper_figure_contract", return_value=prepared): + with self.assertRaisesRegex(RuntimeError, "full-document scientific scope"): + run_paper_to_image(paper=root / "paper.pdf", out=root / "run", asset_mode="image2") + def test_fast_framework_cli_defaults(self): parser = build_parser() args = parser.parse_args(["fast-framework-prompt", "--paper", "paper.pdf", "--out", "output/fast"]) diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 0d45e00..831132b 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -90,6 +90,7 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("mean_evidence_page_coverage_ratio", result["aggregate"]) self.assertIn("ocr_scheduled_page_total", result["aggregate"]) self.assertIn("max_detected_column_count", result["aggregate"]) + self.assertIn("max_ocr_worker_count", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) def test_paper_to_image_score_uses_hard_scientific_gates(self): diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 62e2f87..7c2dc55 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -5,8 +5,9 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _prioritize_ocr_candidates, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _filter_ocr_margin_noise, _prioritize_ocr_candidates, _rapidocr_worker_count, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper +from rfs.reference_text_extractor import _positive_ocr_setting class PaperAnalyzerTests(unittest.TestCase): @@ -78,6 +79,24 @@ def draw(page): self.assertGreaterEqual(parsed["pages"][0]["reading_order_confidence"], 0.8) self.assertEqual(parsed["pages"][0]["column_count"], 2) + def test_cross_extractor_token_agreement_ignores_column_order(self): + native = "left column first passage left column second passage right column first passage" + poppler = "left column first passage right column first passage left column second passage" + + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_cross_extractor_token_agreement_ignores_equation_variable_layout(self): + native = "The projected point gives the origin and direction. ax x z ay y z az bz." + poppler = "The projected point gives the origin and direction. a x x/z a y y/z a z + b z." + + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_cross_extractor_token_agreement_allows_poppler_hidden_text(self): + native = "The network architecture contains an encoder and decoder." + poppler = native + " hidden latex accessibility expansion with many additional formula tokens" + + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + def test_three_column_blocks_are_sorted_column_major(self): def draw(page): page.insert_textbox((400, 100, 560, 170), "COLUMN THREE TOP complete scientific paragraph.", fontsize=9) @@ -140,6 +159,28 @@ def test_low_text_page_uses_local_ocr_adapter(self): self.assertTrue(parsed["pages"][0]["used_ocr"]) self.assertIn("Image Encoder", parsed["pages"][0]["text"]) + def test_cross_extractor_content_disagreement_still_triggers_ocr(self): + path = self._pdf(lambda page: page.insert_textbox( + (40, 80, 560, 220), + "Abstract Method The native parser returns a sufficiently long but unsupported passage about an encoder and decoder.", + fontsize=10, + )) + ocr_text = "Abstract Method The verified page contains an image encoder, transformer decoder, training objective, inference output, and complete scientific evidence for the recovered framework." + with patch("rfs.paper_to_image.analyzer._poppler_pages", return_value=(["Completely unrelated alternate parser content about chemistry molecules and laboratory measurements."], None)): + parsed = parse_paper( + path, + ocr_engine="easyocr", + ocr_adapter=lambda _image, _lang: [{ + "text": ocr_text, + "confidence": 0.96, + "quad": [[20, 20], [950, 20], [950, 120], [20, 120]], + }], + ) + + self.assertEqual(parsed["extraction_report"]["ocr_candidate_pages"], [1]) + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1]) + self.assertIn("verified page", parsed["pages"][0]["text"]) + def test_empty_ocr_result_returns_failed_document_model_instead_of_raising(self): path = self._pdf(lambda _page: None) @@ -176,6 +217,38 @@ def test_rapidocr_caption_group_survives_sentence_boundaries(self): self.assertEqual({item["parent_block"] for item in grouped[:3]}, {1}) self.assertNotEqual(grouped[2]["parent_block"], grouped[3]["parent_block"]) + def test_ocr_margin_filter_removes_vertical_watermark_fragments(self): + records = [ + {"bbox": [130, 100 + index * 30, 730, 124 + index * 30], "text": f"Complete scientific body line number {index} with reliable method evidence."} + for index in range(6) + ] + records.extend([ + {"bbox": [35, 120, 70, 190], "text": "May"}, + {"bbox": [38, 210, 68, 245], "text": "28"}, + {"bbox": [90, 400, 180, 425], "text": "CNN"}, + ]) + + filtered, removed = _filter_ocr_margin_noise(records, 842) + + self.assertEqual(removed, 2) + self.assertIn("CNN", [item["text"] for item in filtered]) + + def test_ocr_margin_filter_removes_fragments_even_when_body_starts_left(self): + records = [ + {"bbox": [70, 300 + index * 30, 770, 324 + index * 30], "text": f"Long body line {index} establishes the left boundary near the page margin."} + for index in range(6) + ] + records.extend([ + {"bbox": [35, 120, 70, 190], "text": "May"}, + {"bbox": [40, 220, 66, 246], "text": "S"}, + {"bbox": [63, 40, 790, 75], "text": "End-to-End Object Detection with Transformers"}, + ]) + + filtered, removed = _filter_ocr_margin_noise(records, 842) + + self.assertEqual(removed, 2) + self.assertIn("End-to-End Object Detection with Transformers", [item["text"] for item in filtered]) + def test_inspection_reuses_matching_document_cache(self): path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "Abstract Method A sufficiently long scientific paragraph for deterministic PDF inspection.", fontsize=10)) with tempfile.TemporaryDirectory() as temp: @@ -231,6 +304,16 @@ def test_ocr_priority_spreads_fully_scanned_long_document(self): self.assertEqual(selected, [1, 2, 3, 6, 9, 10]) self.assertEqual([item["rank"] for item in details], [1, 2, 3, 4, 5, 6]) + def test_rapidocr_worker_count_is_bounded_and_adapter_safe(self): + with patch.dict("os.environ", {"RFS_OCR_WORKERS": "3"}, clear=False): + self.assertEqual(_rapidocr_worker_count("rapidocr", None, 6), min(3, __import__("os").cpu_count() or 1)) + self.assertEqual(_rapidocr_worker_count("rapidocr", lambda *_args: [], 6), 1) + self.assertEqual(_rapidocr_worker_count("easyocr", None, 6), 1) + + def test_invalid_rapidocr_environment_setting_falls_back(self): + with patch.dict("os.environ", {"RFS_RAPIDOCR_THREADS": "invalid"}, clear=False): + self.assertEqual(_positive_ocr_setting(None, "RFS_RAPIDOCR_THREADS", 1), 1) + def test_ocr_priority_prefers_semantic_pages_before_coverage_anchors(self): pages = [{"page": index, "text": ""} for index in range(1, 11)] pages[3]["text"] = "Figure 1: Overview of the proposed architecture and system pipeline." @@ -258,6 +341,10 @@ def adapter(image_path, _lang): self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 6, 9, 10]) self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 6, 9, 10]) self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 6, 9, 10]) + self.assertEqual(parsed["extraction_report"]["status"], "warning") + self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") + self.assertEqual(parsed["extraction_report"]["semantic_scope"], "sampled_pages_only") + self.assertFalse(parsed["extraction_report"]["scientific_scope_complete"]) if __name__ == "__main__": diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index ad59473..1c6506b 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -74,6 +74,32 @@ def test_contract_normalizes_scalar_claim_and_grounds_it_from_evidence(self): self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0001"]) self.assertTrue(validation["ok"]) + def test_contract_grounds_declared_multword_entity_by_exact_paper_term(self): + parsed = { + "page_count": 6, + "evidence": [ + {"id": "E0001", "page": 6, "text": "Transformer encoder. First, a convolution reduces the channel dimension.", "confidence": 0.98}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "modules": [{"id": "encoder", "name": "Transformer Encoder", "evidence_ids": []}], + "inputs": [], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["modules"][0]["evidence_ids"], ["E0001"]) + self.assertIn("encoder", plan["contract_completion_report"]["grounded_entities"]) + def test_grounding_rejects_relation_without_evidence(self): plan = { "figure_specification": { From 1d274a534aae6af279beca7ad64f8ddb835a6b6e Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 06:02:16 +0800 Subject: [PATCH 27/74] Recover two-column OCR layouts from title fragments --- README.md | 2 +- docs/paper-to-image.md | 2 +- rfs/paper_to_image/analyzer.py | 56 ++++++++++++++++------------ rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 2 +- tests/unit/test_paper_analyzer.py | 16 +++++++- 6 files changed, 51 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index c4812aa..2f1760e 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. Cross-parser gating compares supported lexical content rather than raw string order, so normal two-column reordering, equation layout, and hidden accessibility text do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Cross-parser gating compares supported lexical content rather than raw string order, so normal two-column reordering, equation layout, and hidden accessibility text do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 15ede99..806e7bd 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -10,7 +10,7 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --j The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. If a long fully scanned paper cannot reach the 60% full-document readability gate within the deadline, the workflow may continue only when at least six high-confidence sampled pages cover both Abstract and Method. The result records `semantic_scope: sampled_pages_only`, includes an explicit uncertainty about unprocessed pages, and can never be production-ready. Missing Abstract/Method coverage or weak OCR still stops semantic planning. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 4e7bace..8558d21 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -86,16 +86,8 @@ def _column_groups(items: list[dict[str, Any]], page_width: float, max_columns: (float(positioned[index + 1]["bbox"][0]) - float(positioned[index]["bbox"][0]), index) for index in range(len(positioned) - 1) ] - split_indexes = sorted(index for gap, index in sorted(gaps, reverse=True)[: max_columns - 1] if gap >= page_width * 0.12) - if not split_indexes: - return [items] - groups: list[list[dict[str, Any]]] = [] - start = 0 - for split in split_indexes: - groups.append(positioned[start:split + 1]) - start = split + 1 - groups.append(positioned[start:]) - if len(groups) > max_columns or any(len(group) < 2 for group in groups): + qualified_gaps = [(gap, index) for gap, index in sorted(gaps, reverse=True) if gap >= page_width * 0.12] + if not qualified_gaps: return [items] def median(values: list[float]) -> float: @@ -103,20 +95,36 @@ def median(values: list[float]) -> float: middle = len(ordered) // 2 return ordered[middle] if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2.0 - ordered_groups = sorted(groups, key=lambda group: median([float(item["bbox"][0]) for item in group])) - minimum_vertical_span = page_width * 0.12 - minimum_text = 40 - for group in ordered_groups: - vertical_span = max(float(item["bbox"][3]) for item in group) - min(float(item["bbox"][1]) for item in group) - text_count = sum(len(_normalized_compare_text(str(item.get("text") or ""))) for item in group) - if vertical_span < minimum_vertical_span or text_count < minimum_text: - return [items] - for left, right in zip(ordered_groups, ordered_groups[1:]): - left_right = median([float(item["bbox"][2]) for item in left]) - right_left = median([float(item["bbox"][0]) for item in right]) - if left_right > right_left + page_width * 0.04: - return [items] - return ordered_groups + def validated_groups(split_count: int) -> list[list[dict[str, Any]]] | None: + split_indexes = sorted(index for _, index in qualified_gaps[:split_count]) + groups: list[list[dict[str, Any]]] = [] + start = 0 + for split in split_indexes: + groups.append(positioned[start:split + 1]) + start = split + 1 + groups.append(positioned[start:]) + if len(groups) > max_columns or any(len(group) < 2 for group in groups): + return None + ordered_groups = sorted(groups, key=lambda group: median([float(item["bbox"][0]) for item in group])) + minimum_vertical_span = page_width * 0.12 + minimum_text = 40 + for group in ordered_groups: + vertical_span = max(float(item["bbox"][3]) for item in group) - min(float(item["bbox"][1]) for item in group) + text_count = sum(len(_normalized_compare_text(str(item.get("text") or ""))) for item in group) + if vertical_span < minimum_vertical_span or text_count < minimum_text: + return None + for left, right in zip(ordered_groups, ordered_groups[1:]): + left_right = median([float(item["bbox"][2]) for item in left]) + right_left = median([float(item["bbox"][0]) for item in right]) + if left_right > right_left + page_width * 0.04: + return None + return ordered_groups + + for split_count in range(min(max_columns - 1, len(qualified_gaps)), 0, -1): + groups = validated_groups(split_count) + if groups: + return groups + return [items] def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[list[dict[str, Any]], float]: diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 9d1a754..cc12624 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 9 +DOCUMENT_CACHE_VERSION = 10 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 05b1ca6..cd4109a 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -568,7 +568,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 12, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 13, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 7c2dc55..d37b0e8 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -5,7 +5,7 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _filter_ocr_margin_noise, _prioritize_ocr_candidates, _rapidocr_worker_count, _token_overlap_agreement, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _column_groups, _filter_ocr_margin_noise, _prioritize_ocr_candidates, _rapidocr_worker_count, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper from rfs.reference_text_extractor import _positive_ocr_setting @@ -116,6 +116,20 @@ def draw(page): self.assertEqual(parsed["pages"][0]["column_count"], 3) self.assertEqual(parsed["extraction_report"]["max_column_count"], 3) + def test_invalid_three_column_split_falls_back_to_two_columns(self): + items = [] + for index in range(6): + items.append({"bbox": [95, 300 + index * 30, 395, 322 + index * 30], "text": f"Complete left column scientific sentence number {index}."}) + items.append({"bbox": [412, 300 + index * 30, 713, 322 + index * 30], "text": f"Complete right column scientific sentence number {index}."}) + items.extend([ + {"bbox": [157, 96, 650, 120], "text": "BERT Pre-training of Deep Bidirectional Transformers"}, + {"bbox": [294, 125, 512, 148], "text": "Language Understanding"}, + ]) + + groups = _column_groups(items, 804) + + self.assertEqual(len(groups), 2) + def test_rotated_page_uses_display_coordinate_space(self): def draw(page): page.insert_textbox((40, 40, 560, 85), "Rotated Paper Architecture Overview", fontsize=14) From 186f0396aed6def13b8877b7d2a09393b473c23e Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 06:38:35 +0800 Subject: [PATCH 28/74] Support CJK papers without speculative relation repair --- README.md | 2 +- docs/paper-to-image.md | 2 +- rfs/paper_to_image/analyzer.py | 53 +++++++++++++++++++--------- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 24 +++++++------ tests/unit/test_paper_analyzer.py | 18 +++++++++- tests/unit/test_paper_contract.py | 53 ++++++++++++++++++++++++++++ 7 files changed, 123 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 2f1760e..ed6bc5e 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Cross-parser gating compares supported lexical content rather than raw string order, so normal two-column reordering, equation layout, and hidden accessibility text do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 806e7bd..e336491 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -10,7 +10,7 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --j The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Quality gates count all Unicode letters and numbers. English, Chinese, Japanese, and Korean section aliases are recognized, Chinese `图/表` captions are indexed, and CJK cross-extractor agreement uses character bigrams so per-character spacing differences do not resemble corruption. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. If a long fully scanned paper cannot reach the 60% full-document readability gate within the deadline, the workflow may continue only when at least six high-confidence sampled pages cover both Abstract and Method. The result records `semantic_scope: sampled_pages_only`, includes an explicit uncertainty about unprocessed pages, and can never be production-ready. Missing Abstract/Method coverage or weak OCR still stops semantic planning. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 8558d21..9df5a21 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -18,11 +18,11 @@ MOJIBAKE_MARKERS = ("\ufffd", "Ã", "Â", "â€", "锟", "鈥", "銆", "鏅") SECTION_ALIASES = { - "abstract": ("abstract",), - "introduction": ("introduction", "background"), - "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework"), - "experiments": ("experiment", "experiments", "evaluation", "results"), - "conclusion": ("conclusion", "conclusions", "discussion"), + "abstract": ("abstract", "摘要", "概要", "초록"), + "introduction": ("introduction", "background", "引言", "介绍", "背景", "はじめに", "序論", "서론", "배경"), + "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework", "方法", "方法论", "架构", "系统概述", "框架", "手法", "方法論", "アーキテクチャ", "방법", "방법론", "아키텍처"), + "experiments": ("experiment", "experiments", "evaluation", "results", "实验", "评估", "结果", "実験", "評価", "結果", "실험", "평가", "결과"), + "conclusion": ("conclusion", "conclusions", "discussion", "结论", "讨论", "总结", "結論", "考察", "まとめ", "결론", "논의", "요약"), } @@ -42,13 +42,23 @@ def _clean(text: str) -> str: def _normalized_compare_text(text: str) -> str: - return re.sub(r"[^a-z0-9]+", "", _clean(text).casefold()) + return "".join(char for char in _clean(text).casefold() if char.isalnum()) + + +def _lexical_units(text: str) -> list[str]: + clean = _clean(text).casefold() + units = [f"latin:{value}" for value in re.findall(r"[a-z]{3,}", clean)] + cjk = "".join(re.findall(r"[\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]", clean)) + if len(cjk) == 1: + units.append(f"cjk:{cjk}") + else: + units.extend(f"cjk:{cjk[index:index + 2]}" for index in range(len(cjk) - 1)) + return units def _token_overlap_agreement(left: str, right: str) -> float | None: - token_pattern = re.compile(r"[a-z]{3,}|[\u3400-\u9fff]{2,}", re.IGNORECASE) - left_tokens = token_pattern.findall(_clean(left).casefold()) - right_tokens = token_pattern.findall(_clean(right).casefold()) + left_tokens = _lexical_units(left) + right_tokens = _lexical_units(right) if not left_tokens or not right_tokens: return None overlap = sum((Counter(left_tokens) & Counter(right_tokens)).values()) @@ -67,8 +77,11 @@ def _mojibake_rate(text: str) -> float: def _block_kind(text: str, width_ratio: float = 0.0) -> str: value = _clean(text) - if re.match(r"^(figure|fig\.|table)\s*[a-z0-9]+", value, re.IGNORECASE): - return "caption" if value.casefold().startswith(("figure", "fig.")) else "table" + if re.match(r"^(figure|fig\.|table|图|圖|表)\s*[a-z0-9一二三四五六七八九十]+", value, re.IGNORECASE): + return "caption" if value.casefold().startswith(("figure", "fig.", "图", "圖")) else "table" + section_aliases = tuple(alias.casefold() for aliases in SECTION_ALIASES.values() for alias in aliases) + if len(value) <= 100 and any(value.casefold() == alias or value.casefold().startswith((f"{alias} ", f"{alias}:", f"{alias}:")) for alias in section_aliases): + return "heading" if re.match(r"^(abstract|\d+(?:\.\d+)*\s+|[ivx]+[.)]\s+)(.{2,100})$", value, re.IGNORECASE) and len(value) <= 140: return "heading" if len(value) <= 180 and width_ratio >= 0.55 and not value.endswith((".", ",", ";")): @@ -766,12 +779,18 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: continue explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) - compact_line = re.sub(r"[^a-z]", "", line.casefold()) - known = len(line) <= 60 and not line.endswith((".", ",", ";")) and any(compact_line.startswith(re.sub(r"[^a-z]", "", alias.casefold())) for aliases in SECTION_ALIASES.values() for alias in aliases) + compact_line = _normalized_compare_text(line) + known = len(line) <= 60 and not line.endswith((".", ",", ";")) and any( + compact_alias and compact_line.startswith(compact_alias) + for aliases in SECTION_ALIASES.values() + for alias in aliases + for compact_alias in [_normalized_compare_text(alias)] + ) if numbered: title_candidate = numbered.group(2).strip() first_alpha = next((char for char in title_candidate if char.isalpha()), "") - if not first_alpha or not first_alpha.isupper() or re.search(r"[=±∑∫]", title_candidate) or title_candidate.endswith(".") or len(title_candidate) > 80 or len(title_candidate.split()) > 10: + has_case = bool(first_alpha and first_alpha.lower() != first_alpha.upper()) + if not first_alpha or (has_case and not first_alpha.isupper()) or re.search(r"[=±∑∫]", title_candidate) or title_candidate.endswith(".") or len(title_candidate) > 80 or len(title_candidate.split()) > 10: numbered = None if not (explicit or numbered or known): continue @@ -795,7 +814,7 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) parent = block.get("parent_block") continuation = [] for following in page_blocks[block_index + 1:]: - if parent is None or following.get("parent_block") != parent or re.match(r"^(figure|fig\.|table)\s*\d", str(following.get("text") or ""), re.IGNORECASE): + if parent is None or following.get("parent_block") != parent or re.match(r"^(figure|fig\.|table|图|圖|表)\s*[\d一二三四五六七八九十]", str(following.get("text") or ""), re.IGNORECASE): break continuation.append(str(following.get("text") or "")) if continuation: @@ -810,9 +829,9 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, bool]: candidates = ("\n".join(item["title"] for item in sections) + "\n" + "\n".join(page.get("text", "") for page in pages)).casefold() - compact = re.sub(r"[^a-z]", "", candidates) + compact = _normalized_compare_text(candidates) return { - name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) or re.sub(r"[^a-z]", "", alias.casefold()) in compact for alias in aliases) + name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) or (_normalized_compare_text(alias) and _normalized_compare_text(alias) in compact) for alias in aliases) for name, aliases in SECTION_ALIASES.items() } diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index cc12624..92a58d0 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 10 +DOCUMENT_CACHE_VERSION = 11 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index cd4109a..71dad1d 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -358,20 +358,24 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d def choose_input_target(label: str) -> dict[str, Any] | None: preferences = [] low = label.casefold() - if "prompt" in low: - preferences = ["prompt encoder", "encoder", "decoder"] - elif "image" in low or "video" in low: - preferences = ["patch", "image encoder", "backbone", "encoder"] - elif any(term in low for term in ("text", "audio", "depth", "thermal", "imu")): - preferences = [f"{next((term for term in ('text', 'audio', 'depth', 'thermal', 'imu') if term in low), '')} encoder", "modality encoders", "encoder"] + if any(term in low for term in ("prompt", "提示", "プロンプト", "프롬프트")): + preferences = ["prompt encoder", "提示编码器", "提示編碼器", "プロンプトエンコーダ", "프롬프트 인코더", "encoder", "decoder"] + elif any(term in low for term in ("image", "video", "图像", "圖像", "影像", "视频", "影片", "画像", "動画", "이미지", "영상")): + preferences = ["patch", "image encoder", "图像编码器", "圖像編碼器", "画像エンコーダ", "이미지 인코더", "backbone", "encoder"] + elif any(term in low for term in ("text", "文本", "文字", "テキスト", "텍스트")): + preferences = ["text encoder", "文本编码器", "文本編碼器", "テキストエンコーダ", "텍스트 인코더", "modality encoders", "encoder"] + elif any(term in low for term in ("audio", "音频", "音訊", "音声", "오디오")): + preferences = ["audio encoder", "音频编码器", "音訊編碼器", "音声エンコーダ", "오디오 인코더", "modality encoders", "encoder"] + elif any(term in low for term in ("depth", "thermal", "imu", "深度", "热成像", "熱成像")): + preferences = ["modality encoders", "encoder", "编码器", "編碼器", "エンコーダ", "인코더"] for preference in preferences: match = next((module for module in modules if preference and preference in _item_label(module).casefold()), None) if match: return match - return modules[0] if modules else None + return None for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: source = str(item.get("id")) - if not source: + if not source or source in outgoing: continue target = choose_input_target(_item_label(item)) if target: @@ -385,7 +389,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: sink_modules = [item for item in modules if str(item.get("id")) not in module_sources] for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []: target_id = str(item.get("id")) - if not target_id: + if not target_id or target_id in incoming: continue output_label = _item_label(item).casefold() source = next((module for module in reversed(modules) if any(term in _item_label(module).casefold() for term in ("head", "decoder", "refine", "classifier", "predict"))), None) @@ -568,7 +572,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 13, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 15, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index d37b0e8..e97073e 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -5,7 +5,7 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _column_groups, _filter_ocr_margin_noise, _prioritize_ocr_candidates, _rapidocr_worker_count, _token_overlap_agreement, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _section_coverage, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper from rfs.reference_text_extractor import _positive_ocr_setting @@ -97,6 +97,22 @@ def test_cross_extractor_token_agreement_allows_poppler_hidden_text(self): self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + def test_unicode_quality_gates_and_cjk_parser_agreement(self): + native = "摘要 本文提出一个多模态框架,用于图像编码与文本生成。" + poppler = "摘 要 本 文 提 出 一 个 多 模 态 框 架,用 于 图 像 编 码 与 文 本 生 成。" + + self.assertGreater(len(_normalized_compare_text(native)), 20) + self.assertEqual(_token_overlap_agreement(native, poppler), 1.0) + + def test_cjk_section_coverage_and_caption_kinds(self): + pages = [{"page": 1, "text": "摘要\n1 引言\n2 方法\n3 实验\n4 结论"}] + coverage = _section_coverage(pages, []) + + self.assertTrue(all(coverage.values())) + self.assertEqual(_block_kind("图 1 系统框架概览"), "caption") + self.assertEqual(_block_kind("表 2 实验结果"), "table") + self.assertEqual(_block_kind("2 方法"), "heading") + def test_three_column_blocks_are_sorted_column_major(self): def draw(page): page.insert_textbox((400, 100, 560, 170), "COLUMN THREE TOP complete scientific paragraph.", fontsize=9) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 1c6506b..a8b4dfa 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -100,6 +100,59 @@ def test_contract_grounds_declared_multword_entity_by_exact_paper_term(self): self.assertEqual(spec["modules"][0]["evidence_ids"], ["E0001"]) self.assertIn("encoder", plan["contract_completion_report"]["grounded_entities"]) + def test_contract_does_not_add_cross_modal_edges_to_connected_inputs(self): + parsed = {"evidence": [{"id": "E0001", "page": 1, "text": "图像进入图像编码器,论文文本进入文本编码器。", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [ + {"id": "image", "name": "图像", "evidence_ids": ["E0001"]}, + {"id": "text", "name": "论文文本", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "image_encoder", "name": "图像编码器", "evidence_ids": ["E0001"]}, + {"id": "text_encoder", "name": "文本编码器", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "image", "target": "image_encoder", "evidence_ids": ["E0001"]}, + {"source": "text", "target": "text_encoder", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(pairs, {("image", "image_encoder"), ("text", "text_encoder")}) + + def test_contract_leaves_unknown_input_unconnected_instead_of_guessing(self): + parsed = {"evidence": [{"id": "E0001", "page": 1, "text": "A scientific system contains two modules.", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "signal", "name": "Scientific Signal", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "first", "name": "First Module", "evidence_ids": ["E0001"]}, + {"id": "second", "name": "Second Module", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertFalse(any(item["source"] == "signal" for item in spec["relations"])) + def test_grounding_rejects_relation_without_evidence(self): plan = { "figure_specification": { From dce151c2f54e2ea4a208e0ebf404aa432557818e Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 07:21:24 +0800 Subject: [PATCH 29/74] Harden paper contracts against VLM schema drift --- README.md | 2 +- docs/paper-to-image.md | 2 +- rfs/paper_to_image/contract_completion.py | 9 +- rfs/paper_to_image/preparation.py | 151 ++++++++++++++++++++-- tests/unit/test_paper_contract.py | 75 +++++++++++ 5 files changed, 224 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ed6bc5e..5ea9841 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ rfs fast-framework-prompt ` --json ``` -This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index e336491..8e2c35b 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -23,7 +23,7 @@ rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id ``` `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. -The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. These repairs are paper-name agnostic and are covered by the NLP suite. +The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. When a paper exceeds the evidence character budget, the extractor reserves representative evidence from every page, then prioritizes topology-defining statements and Abstract, Conclusion, Method, Introduction, and Experiments content. This prevents long introductions or appendices from silently excluding later conclusions and framework definitions. diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 068bfd9..cabf6c6 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -375,7 +375,8 @@ def _ground_declared_entities(spec: dict[str, Any], relevant: list[dict[str, Any continue label = _label(item) normalized_label = _normalized(label) - if len(normalized_label) < 8 or len(re.findall(r"[a-z0-9]+", label.casefold())) < 2: + acronym = bool(label.isupper() and 3 <= len(normalized_label) <= 12) + if not acronym and (len(normalized_label) < 8 or len(re.findall(r"[a-z0-9]+", label.casefold())) < 2): continue match: list[dict[str, Any]] = [] for start in range(len(relevant)): @@ -384,8 +385,10 @@ def _ground_declared_entities(spec: dict[str, Any], relevant: list[dict[str, Any window = relevant[start:start + size] if len(window) != size or any(value.get("page") != page for value in window): break - combined = _normalized(" ".join(str(value.get("text") or "") for value in window)) - if normalized_label in combined: + combined_text = " ".join(str(value.get("text") or "") for value in window) + combined = _normalized(combined_text) + exact_acronym = acronym and re.search(rf"(? list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for index, raw in enumerate(raw_items if isinstance(raw_items, list) else []): + if isinstance(raw, str) and raw.strip(): + item = {"name": raw.strip(), "evidence_ids": []} + elif isinstance(raw, dict): + item = dict(raw) + else: + continue + label = _item_label(item) + if not label: + continue + item["id"] = str(item.get("id") or _stable_id(field.rstrip("s"), label, index)).strip() + item["evidence_ids"] = list(item.get("evidence_ids")) if isinstance(item.get("evidence_ids"), list) else [] + normalized.append(item) + return normalized + + +def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: + aliases: dict[str, set[str]] = {} + endpoint_ids: set[str] = set() + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict) or not item.get("id"): + continue + item_id = str(item["id"]) + endpoint_ids.add(item_id) + for value in (item_id, _item_label(item)): + normalized = _normalized_label(str(value)) + if normalized: + aliases.setdefault(normalized, set()).add(item_id) + + def resolve(value: Any) -> str: + raw = str(value or "").strip() + if raw in endpoint_ids: + return raw + matches = aliases.get(_normalized_label(raw), set()) + return next(iter(matches)) if len(matches) == 1 else raw + + normalized_relations: list[dict[str, Any]] = [] + for raw in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(raw, dict): + continue + item = dict(raw) + source = resolve(item.get("source") or item.get("source_id")) + target = resolve(item.get("target") or item.get("target_id")) + if not source or not target: + continue + item["source"] = source + item["target"] = target + item["type"] = str(item.get("type") or item.get("relation_type") or "data_flow") + item["label"] = str(item.get("label") or "") + item["evidence_ids"] = list(item.get("evidence_ids")) if isinstance(item.get("evidence_ids"), list) else [] + normalized_relations.append(item) + return normalized_relations + + +def _repair_contract_relation_endpoints(spec: dict[str, Any]) -> dict[str, list[str]]: + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + label_matches: dict[str, set[str]] = {} + for item_id, item in endpoints.items(): + normalized = _normalized_label(_item_label(item)) + if normalized: + label_matches.setdefault(normalized, set()).add(item_id) + + replacements: dict[str, str] = {} + repaired: list[str] = [] + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + for relation in relations: + target = str(relation.get("target") or "") + if target in endpoints: + continue + label = _normalized_label(str(relation.get("label") or "")) + matches = label_matches.get(label, set()) + if target and label and len(matches) == 1: + replacement = next(iter(matches)) + replacements[target] = replacement + relation["target"] = replacement + repaired.append(f"{target}->{replacement}") + + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + if source in replacements: + relation["source"] = replacements[source] + if target in replacements: + relation["target"] = replacements[target] + + deduplicated: list[dict[str, Any]] = [] + by_key: dict[tuple[str, str, str], dict[str, Any]] = {} + removed_duplicates: list[str] = [] + removed_unresolved: list[str] = [] + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + relation_type = str(relation.get("type") or "data_flow") + if source not in endpoints or target not in endpoints: + removed_unresolved.append(f"{source}->{target}") + continue + key = (source, target, relation_type) + existing = by_key.get(key) + if existing: + existing["evidence_ids"] = list(dict.fromkeys(list(existing.get("evidence_ids", [])) + list(relation.get("evidence_ids", [])))) + if not existing.get("label") and relation.get("label"): + existing["label"] = relation["label"] + removed_duplicates.append(f"{source}->{target}:{relation_type}") + continue + by_key[key] = relation + deduplicated.append(relation) + spec["relations"] = deduplicated + return { + "repaired": list(dict.fromkeys(repaired)), + "removed_duplicates": list(dict.fromkeys(removed_duplicates)), + "removed_unresolved": list(dict.fromkeys(removed_unresolved)), + } + def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} @@ -247,15 +369,12 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d spec["inputs"] = summary.get("inputs") if isinstance(summary.get("inputs"), list) else [] if not isinstance(spec.get("outputs"), list) or not spec.get("outputs"): spec["outputs"] = summary.get("outputs") if isinstance(summary.get("outputs"), list) else [] - for field in ("inputs", "outputs", "innovations"): - normalized_items = [] - for index, raw in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): - if not isinstance(raw, dict): - continue - item = dict(raw) - item.setdefault("id", _item_id(item, field, index)) - normalized_items.append(item) - spec[field] = normalized_items + if not isinstance(spec.get("modules"), list) or not spec.get("modules"): + summary_modules = summary.get("core_modules") if isinstance(summary.get("core_modules"), list) else summary.get("modules") + spec["modules"] = summary_modules if isinstance(summary_modules, list) else [] + for field in ("inputs", "modules", "outputs", "innovations"): + spec[field] = _normalize_contract_entities(spec.get(field), field) + spec["relations"] = _normalize_contract_relations(spec) completion_report = augment_contract_from_evidence(spec, parsed) plan["contract_completion_report"] = completion_report if len(completion_report.get("added_entities", [])) >= 3: @@ -274,6 +393,14 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d ] completion_report["removed_fallback_entities"] = sorted(fallback_ids) _complete_from_overview_caption(spec, parsed) + endpoint_report = _repair_contract_relation_endpoints(spec) + completion_report["relation_endpoint_repairs"] = endpoint_report["repaired"] + completion_report["removed_duplicate_relations"] = endpoint_report["removed_duplicates"] + completion_report["removed_unresolved_relations"] = endpoint_report["removed_unresolved"] + if endpoint_report["repaired"]: + secondary_completion = augment_contract_from_evidence(spec, parsed) + for key in ("added_entities", "upgraded_entities", "adopted_entities", "added_relations", "repaired_relations", "grounded_entities"): + completion_report[key] = list(dict.fromkeys(list(completion_report.get(key, [])) + list(secondary_completion.get(key, [])))) modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] encoder_modules = [item for item in modules if isinstance(item, dict) and "encoder" in _item_label(item).casefold() and "modality encoder" not in _item_label(item).casefold()] if len(encoder_modules) >= 3: @@ -420,6 +547,10 @@ def choose_input_target(label: str) -> dict[str, Any] | None: labels.append(_item_label(item)) spec["required_labels"] = list(dict.fromkeys(labels)) raw_uncertainties = list(summary.get("unknowns", []) if isinstance(summary.get("unknowns"), list) else []) + raw_uncertainties.extend( + f"Dropped unresolved relation {value} because its endpoint was not declared." + for value in completion_report.get("removed_unresolved_relations", []) + ) uncertainties = [ str(item.get("statement") or item.get("text") or item.get("id") or "unknown") if isinstance(item, dict) else str(item) for item in raw_uncertainties @@ -572,7 +703,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 15, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 17, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index a8b4dfa..25bc660 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -153,6 +153,40 @@ def test_contract_leaves_unknown_input_unconnected_instead_of_guessing(self): self.assertFalse(any(item["source"] == "signal" for item in spec["relations"])) + def test_contract_normalizes_string_entities_and_relation_alias_fields(self): + parsed = { + "evidence": [ + {"id": "E0001", "page": 1, "text": "The Image Encoder sends representations to the Mask Decoder.", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": { + "unknowns": [], + "core_modules": ["Image Encoder", "Mask Decoder"], + }, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [], + "outputs": [], + "innovations": [], + "relations": [ + {"source_id": "Image Encoder", "target_id": "Mask Decoder", "relation_type": "feature_flow", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + validation = validate_plan_grounding(plan, parsed) + + self.assertEqual([item["name"] for item in spec["modules"]], ["Image Encoder", "Mask Decoder"]) + self.assertEqual(spec["relations"][0]["source"], spec["modules"][0]["id"]) + self.assertEqual(spec["relations"][0]["target"], spec["modules"][1]["id"]) + self.assertEqual(spec["relations"][0]["type"], "feature_flow") + self.assertTrue(validation["ok"]) + def test_grounding_rejects_relation_without_evidence(self): plan = { "figure_specification": { @@ -431,6 +465,47 @@ def test_generic_completion_recovers_retrieval_conditioned_generation(self): self.assertIn((ids["Document Index"], ids["Retriever"]), pairs) self.assertIn((ids["Top-K Documents"], ids["Generator"]), pairs) + def test_rag_contract_repairs_latent_endpoint_alias_and_acronym_evidence(self): + caption = "Figure 1: Overview of our approach. Query Encoder and Document Index use Maximum Inner Product Search (MIPS) to find the top-K documents z. The Generator conditions on those latent documents." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "query_encoder", "name": "Query Encoder", "evidence_ids": ["E0001"]}, + {"id": "doc_index", "name": "Document Index", "evidence_ids": ["E0001"]}, + {"id": "generator", "name": "Generator", "evidence_ids": ["E0001"]}, + {"id": "mips", "name": "MIPS", "evidence_ids": []}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "mips", "target": "latent_z", "type": "branch", "label": "Top-K Documents", "evidence_ids": []}, + {"source": "latent_z", "target": "generator", "type": "conditioning", "label": "context", "evidence_ids": []}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + validation = validate_plan_grounding(plan, parsed) + ids = {item["name"]: item["id"] for item in spec["modules"]} + pairs = {(item["source"], item["target"]): item for item in spec["relations"]} + + self.assertTrue(next(item for item in spec["modules"] if item["name"] == "MIPS")["evidence_ids"]) + self.assertIn(("mips", ids["Top-K Documents"]), pairs) + self.assertIn((ids["Top-K Documents"], "generator"), pairs) + self.assertTrue(pairs[("mips", ids["Top-K Documents"])]["evidence_ids"]) + self.assertFalse(any("latent_z" in (item["source"], item["target"]) for item in spec["relations"])) + self.assertTrue(validation["ok"]) + def test_generic_completion_repairs_vlm_ids_labels_and_relation_grounding(self): caption = "Figure 1: Pre-training and fine-tuning procedures." parsed = { From 173a62eb9119b4443e7436eaa3d9188105234ffd Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 08:47:47 +0800 Subject: [PATCH 30/74] Constrain paper contracts to evidence-connected graphs --- README.md | 2 +- .../expected_semantics.json | 2 +- .../expected_semantics.json | 2 +- docs/architecture.md | 2 + rfs/evaluation/benchmarking.py | 95 ++++++--- rfs/paper_to_image/contract_completion.py | 114 +++++++++- rfs/paper_to_image/preparation.py | 2 +- tests/unit/test_benchmarking.py | 47 +++++ tests/unit/test_paper_contract.py | 195 ++++++++++++++++++ 9 files changed, 425 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 5ea9841..0d71c0d 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ rfs fast-framework-prompt ` --json ``` -This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. diff --git a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json index c881bb8..86f1af4 100644 --- a/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/105_sam_dense_multiframe/expected_semantics.json @@ -26,7 +26,7 @@ {"source": "fully_automatic", "target": "sa1b", "type": "data_generation", "required": true}, {"source": "sam_model", "target": "data_engine", "type": "annotation_support", "required": true} ], - "forbidden_labels": ["Text-to-Image Generation", "Object Detector Only", "Single-stage Manual Annotation", "Language Model"], + "forbidden_labels": ["Text-to-Image Generation", "Object Detector Only", "Single-stage Manual Annotation", "Language Model", "Class Descriptions", "Contrastive Learning", "Text Embeddings", "Zero-Shot Classifier"], "expected_panels": ["promptable_task", "sam_model", "data_engine"], "topology_requirements": {"panel_count": 3, "dense_multi_panel": true, "feedback_between_model_and_data_engine": true} } diff --git a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json index 93e8caf..c54db48 100644 --- a/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json +++ b/benchmarks/paper-to-image/cases/111_rag_retrieval_generation/expected_semantics.json @@ -18,7 +18,7 @@ {"source": "top_k_documents", "target": "generator", "type": "conditioning", "required": true}, {"source": "generator", "target": "output_sequence", "type": "generation", "required": true} ], - "forbidden_labels": ["Image Encoder", "Object Queries", "Mask Decoder", "Convolutional Backbone"], + "forbidden_labels": ["Image Encoder", "Object Queries", "Mask Decoder", "Convolutional Backbone", "Class Prediction", "Class Predictions", "Classification Head"], "expected_reading_order": ["input_query", "query_encoder", "document_index", "retriever", "top_k_documents", "generator", "output_sequence"], "topology_requirements": {"primary_flow": "retrieval_then_generation", "branch_count": 2, "feedback_loop_count": 0, "dense_multi_panel": false} } diff --git a/docs/architecture.md b/docs/architecture.md index 4a4a19f..ef82d44 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,6 +75,8 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. - Engineering previews are opt-in and never production-approved. - Critical text and relations remain editable PPT objects. - Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. +- Deterministic completion is graph-scoped: rich VLM contracts only gain overview-local or directly connected evidence-backed entities, while heuristic contracts grow from overview seeds or the largest connected method chain. +- Benchmark entity matching uses both label fidelity and relation consistency so relation labels and variable suffixes cannot steal mappings from real contract entities. - VLM correction is optional and bounded; deterministic reports remain available offline. - Historical sample-specific scripts stay under `experiments/legacy_reference_rebuilds/` and are not imported by production code. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 2c3fc86..a450727 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -63,6 +63,15 @@ def _normalized_text(value: object) -> str: return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) +def _normalized_label_variants(value: object) -> set[str]: + label = str(value or "").strip() + variants = {_normalized_text(label)} + without_suffix = re.sub(r"\s*\([^()]{1,24}\)\s*$", "", label).strip() + if without_suffix: + variants.add(_normalized_text(without_suffix)) + return {item for item in variants if item} + + def _score_planning_contract(expected: dict, specification: dict) -> dict[str, Any]: if not specification: return {"available": False} @@ -72,7 +81,7 @@ def _score_planning_contract(expected: dict, specification: dict) -> dict[str, A if not isinstance(item, dict): continue label = next((str(item.get(key) or "").strip() for key in ("visible_label", "name", "text", "statement", "label", "id") if str(item.get(key) or "").strip()), "") - actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label), "field": field}) + actual_entities.append({"id": str(item.get("id") or f"{field}_{index}"), "label": label, "normalized": _normalized_text(label), "normalized_variants": _normalized_label_variants(label), "field": field}) expanded_relations = set() for index, item in enumerate(specification.get("relations", []) if isinstance(specification.get("relations"), list) else []): if not isinstance(item, dict): @@ -84,52 +93,90 @@ def _score_planning_contract(expected: dict, specification: dict) -> dict[str, A label = str(item.get("label") or "").strip() if source and target and label: label_id = f"relation_label_{index}" - actual_entities.append({"id": label_id, "label": label, "normalized": _normalized_text(label), "field": "relation_label"}) + actual_entities.append({"id": label_id, "label": label, "normalized": _normalized_text(label), "normalized_variants": _normalized_label_variants(label), "field": "relation_label"}) expanded_relations.add((source, label_id)) expanded_relations.add((label_id, target)) + actual_relations = expanded_relations + adjacency: dict[str, set[str]] = {} + for source, target in actual_relations: + adjacency.setdefault(source, set()).add(target) + + def connected(source: str, target: str, max_hops: int = 2) -> bool: + if (source, target) in actual_relations: + return True + frontier = {source} + visited = {source} + for _ in range(max_hops): + frontier = {neighbor for node in frontier for neighbor in adjacency.get(node, set()) if neighbor not in visited} + if target in frontier: + return True + visited.update(frontier) + return False + expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] mapping: dict[str, str] = {} matched = [] used_actual_ids: set[str] = set() + candidate_scores: dict[str, dict[str, int]] = {} for item in expected_entities: + expected_id = str(item.get("id")) primary = _normalized_text(item.get("label")) aliases = [_normalized_text(value) for value in item.get("aliases", []) if _normalized_text(value)] - def match_score(actual: dict[str, str]) -> int: + def match_score(actual: dict[str, Any]) -> int: value = actual["normalized"] - if not value or actual["id"] in used_actual_ids: + variants = actual.get("normalized_variants") or {value} + if not value: return 0 bonus = 2 if actual.get("field") in {"inputs", "modules", "outputs", "innovations"} else 1 if actual.get("field") == "relation_label" else 0 - if primary and primary == value: + if primary and primary in variants: return 5 + bonus - if any(alias == value for alias in aliases): + if any(alias in variants for alias in aliases): return 4 + bonus if any(alias in value or value in alias for alias in aliases): return 3 + bonus if primary and (primary in value or value in primary): return 1 + bonus return 0 - ranked = sorted(((match_score(actual), actual) for actual in actual_entities), key=lambda pair: pair[0], reverse=True) + candidate_scores[expected_id] = {actual["id"]: match_score(actual) for actual in actual_entities} + ranked = sorted( + ((score, actual) for actual in actual_entities if actual["id"] not in used_actual_ids and (score := candidate_scores[expected_id][actual["id"]]) > 0), + key=lambda pair: pair[0], + reverse=True, + ) match = ranked[0][1] if ranked and ranked[0][0] > 0 else None if match: - mapping[str(item.get("id"))] = match["id"] + mapping[expected_id] = match["id"] used_actual_ids.add(match["id"]) - matched.append(str(item.get("id"))) + matched.append(expected_id) expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] - actual_relations = expanded_relations - adjacency: dict[str, set[str]] = {} - for source, target in actual_relations: - adjacency.setdefault(source, set()).add(target) - def connected(source: str, target: str, max_hops: int = 2) -> bool: - if (source, target) in actual_relations: - return True - frontier = {source} - visited = {source} - for _ in range(max_hops): - frontier = {neighbor for node in frontier for neighbor in adjacency.get(node, set()) if neighbor not in visited} - if target in frontier: - return True - visited.update(frontier) - return False + for expected_id, current_actual in list(mapping.items()): + blocked = set(mapping.values()) - {current_actual} + + def relation_support(actual_id: str) -> int: + support = 0 + for relation in expected_relations: + source_expected = str(relation.get("source")) + target_expected = str(relation.get("target")) + if source_expected == expected_id and target_expected in mapping: + support += int(connected(actual_id, mapping[target_expected])) + elif target_expected == expected_id and source_expected in mapping: + support += int(connected(mapping[source_expected], actual_id)) + return support + + alternatives = [ + actual_id + for actual_id, score in candidate_scores.get(expected_id, {}).items() + if score > 0 and actual_id not in blocked + ] + if alternatives: + mapping[expected_id] = max( + alternatives, + key=lambda actual_id: ( + candidate_scores[expected_id][actual_id], + relation_support(actual_id), + actual_id == current_actual, + ), + ) relation_matches = 0 matched_relations = [] missing_relations = [] diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index cabf6c6..778948b 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -71,15 +71,15 @@ class ConceptRule: ConceptRule("prompt_encoder", "Prompt Encoder", "modules", "encoder", r"\bprompt encoder\b"), ConceptRule("mask_decoder", "Mask Decoder", "modules", "decoder", r"\bmask decoder\b"), ConceptRule("mlp", "MLP", "modules", "neural field", r"\bneural radiance field\b|\bfeeding[^.]{0,100}\bmlp\b|\bmlp to produce\b", ("neural radiance field", "neural network")), - ConceptRule("image_embeddings", "Image Embeddings", "modules", "representation", r"\bimage embeddings?\b|\bimage features?\b", (), True), - ConceptRule("text_embeddings", "Text Embeddings", "modules", "representation", r"\btext embeddings?\b|\btext features?\b"), + ConceptRule("image_embeddings", "Image Embeddings", "modules", "representation", r"\bimage embeddings?\b|\bimage features?\b", ("image embedding", "image features"), True), + ConceptRule("text_embeddings", "Text Embeddings", "modules", "representation", r"\btext embeddings?\b|\btext features?\b", ("text embedding", "text features")), ConceptRule("contrastive_objective", "Contrastive Learning", "modules", "training objective", r"\bcontrastive (?:pre-?training|learning|objective|loss)\b|\bcorrect pairings?\b", ("contrastive pre-training", "image-text contrastive loss", "correct pairings")), ConceptRule("bipartite_matching", "Bipartite Matching", "modules", "training objective", r"\bbipartite matching\b|\bhungarian matching\b", ("hungarian matching", "set prediction loss")), ConceptRule("volume_density", "Volume Density", "modules", "field output", r"\bvolume density\b", ("density", "sigma")), ConceptRule("color", "Color", "modules", "field output", r"\b(?:rgb|emitted) colou?r\b|\bradiance\b|\bcolou?r and volume density\b", ("rgb color", "radiance"), True), ConceptRule("volume_rendering", "Volume Rendering", "modules", "renderer", r"\b(?:differentiable\s+)?volume rendering\b", ("differentiable volume rendering",)), ConceptRule("class_predictions", "Class Predictions", "outputs", "output", r"\bclass (?:prediction|predictions|labels?)\b|\bpredicts?[^.]{0,80}\bclass\b", ("classes", "class labels")), - ConceptRule("class_prediction", "Class Prediction", "outputs", "output", r"\bimage classification predictions?\b|\bclassification predictions?\b|\bclass label\b", ("image classification predictions", "classification predictions", "class label")), + ConceptRule("class_prediction", "Class Prediction", "outputs", "output", r"\bimage classification predictions?\b|\bclassification predictions?\b|\bclass label\b", ("image classification predictions", "classification predictions", "class label", "class")), ConceptRule("box_predictions", "Bounding Box Predictions", "outputs", "output", r"\bbounding box(?:es| predictions?)?\b|\bbox predictions?\b", ("bounding boxes", "box predictions")), ConceptRule("classification", "Classification", "outputs", "output head", r"\bclassification head\b|\bpredicts? the class label\b", ("class label",)), ConceptRule("box_regression", "Bounding-box Regression", "outputs", "output head", r"\bbounding[ -]?box regression\b|\bbox regression\b", ("bounding box", "box branch")), @@ -92,7 +92,7 @@ class ConceptRule: ConceptRule("source_tokens", "Inputs", "inputs", "source sequence", r"\bencoder maps an input sequence(?: of symbol representations)?\b|\blearned embeddings to convert the input\s+tokens\b", ("input sequence", "source tokens")), ConceptRule("input_embedding", "Input Embedding", "modules", "embedding", r"\blearned embeddings to convert the input\s+tokens\b|\binput embeddings?\b", ("input embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), ConceptRule("encoder_stack", "Encoder Stack", "modules", "encoder", r"\bencoder is composed of a stack\b|\bencoder stack\b", ("encoder", "transformer encoder")), - ConceptRule("target_tokens", "Outputs (shifted right)", "inputs", "target sequence", r"\boutput embeddings? (?:are )?offset by one position\b|\binput\s+tokens and output tokens\b", ("target tokens", "target sequence", "outputs shifted right")), + ConceptRule("target_tokens", "Outputs (shifted right)", "inputs", "target sequence", r"\boutput embeddings? (?:are )?offset by one position\b|\binput\s+tokens and output tokens\b", ("target tokens", "target sequence", "output sequence", "outputs shifted right")), ConceptRule("output_embedding", "Output Embedding", "modules", "embedding", r"\boutput embeddings?\b|\binput\s+tokens and output tokens\b", ("output embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), ConceptRule("decoder_stack", "Decoder Stack", "modules", "decoder", r"\bdecoder is also composed of a stack\b|\bdecoder stack\b", ("decoder", "transformer decoder")), ConceptRule("output_linear", "Linear", "modules", "output projection", r"\blearned linear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("linear layer", "output projection")), @@ -126,6 +126,7 @@ class ConceptRule: ("class_token", "transformer_encoder", "conditioning"), ("transformer_encoder", "mlp_head", "data_flow"), ("mlp_head", "class_prediction", "data_flow"), + ("mlp_head", "classification", "data_flow"), ("input_image", "cnn_backbone", "data_flow"), ("input_image", "backbone", "data_flow"), ("cnn_backbone", "transformer_encoder", "data_flow"), @@ -278,7 +279,7 @@ def _find_matches(rule: ConceptRule, selected: list[dict[str, Any]], relevant: l pattern = re.compile(rule.pattern, re.IGNORECASE) if rule.context_pattern: context = re.compile(rule.context_pattern, re.IGNORECASE | re.DOTALL) - context_text = " ".join(str(item.get("text") or "") for item in relevant) + context_text = " ".join(str(item.get("text") or "") for item in [*selected, *relevant]) if not context.search(context_text): return [] primary = [item for item in selected if pattern.search(str(item.get("text") or ""))] @@ -301,10 +302,13 @@ def _find_existing(spec: dict[str, Any], rule: ConceptRule) -> tuple[str, dict[s for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: if not isinstance(item, dict): continue - value = _normalized(_label(item)) + raw_value = _label(item) + value = _normalized(raw_value) item_id = _normalized(item.get("id")) if item_id in accepted or value in accepted: return field, item + if rule.label.isupper() and re.match(rf"^\s*{re.escape(rule.label)}\s*(?:\(|$)", raw_value, re.IGNORECASE): + return field, item if rule.key != "self_feedback" and field == rule.field and len(value) >= 8 and any(len(candidate) >= 8 and (value in candidate or candidate in value) for candidate in accepted if candidate): return field, item return None @@ -400,8 +404,86 @@ def _ground_declared_entities(spec: dict[str, Any], relevant: list[dict[str, Any return grounded +def _heuristic_rule_scope( + selected: list[dict[str, Any]], + relevant: list[dict[str, Any]], + declared_rule_keys: set[str], +) -> tuple[set[str], dict[str, list[dict[str, Any]]]]: + matches = { + rule.key: found + for rule in CONCEPT_RULES + if (found := _find_matches(rule, selected, relevant)) + } + candidates = set(matches) + if not candidates: + return set(), matches + seeds = declared_rule_keys | { + rule.key + for rule in CONCEPT_RULES + if rule.key in candidates and _find_matches(rule, selected, []) + } + adjacency = {key: set() for key in candidates} + for source_key, target_key, _ in RELATION_RULES: + if source_key in candidates and target_key in candidates: + adjacency[source_key].add(target_key) + adjacency[target_key].add(source_key) + + def component(start: str) -> set[str]: + reached: set[str] = set() + pending = [start] + while pending: + current = pending.pop() + if current in reached: + continue + reached.add(current) + pending.extend(adjacency.get(current, set()) - reached) + return reached + + if seeds & candidates: + allowed: set[str] = set() + for seed in seeds & candidates: + allowed.update(component(seed)) + return allowed, matches + + components: list[set[str]] = [] + remaining = set(candidates) + while remaining: + group = component(next(iter(remaining))) + components.append(group) + remaining -= group + components.sort(key=lambda group: (len(group), sum(len(matches[key]) for key in group)), reverse=True) + return components[0], matches + + def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: selected, relevant = _relevant_evidence(parsed) + declared_entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + fallback_declared = any(str(item.get("role") or "") == "paper-derived stage requiring VLM verification" for item in declared_entities) + conservative_expansion = len(declared_entities) >= 3 and len([item for item in spec.get("relations", []) if isinstance(item, dict)]) >= 1 and not fallback_declared + selected_pages = {int(item.get("page") or 0) for item in selected if item.get("page")} + relevant_non_background = [ + item + for item in relevant + if not any(term in str(item.get("section_hint") or "").casefold() for term in ("related work", "reference", "acknowledg")) + and not ( + any(term in str(item.get("section_hint") or "").casefold() for term in ("introduction", "background")) + and re.search(r"\b(?:for example|e\.g\.|such as)\b[^\n]{0,160}\[\s*\d+", str(item.get("text") or ""), re.IGNORECASE) + ) + ] + initially_declared_rule_keys = { + rule.key + for rule in CONCEPT_RULES + if _find_existing(spec, rule) + } + heuristic_rule_keys: set[str] | None = None + heuristic_matches: dict[str, list[dict[str, Any]]] = {} + if not conservative_expansion: + heuristic_rule_keys, heuristic_matches = _heuristic_rule_scope(selected, relevant_non_background, initially_declared_rule_keys) grounded_statements = { field: _ground_statement(spec.get(field), relevant) for field in ("research_problem", "central_claim") @@ -412,11 +494,24 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) upgraded_entities: list[str] = [] adopted_entities: list[str] = [] for rule in CONCEPT_RULES: - matches = _find_matches(rule, selected, relevant) + existing = _find_existing(spec, rule) + if existing: + matches = _find_matches(rule, selected, relevant) + elif not conservative_expansion: + matches = heuristic_matches.get(rule.key, []) if rule.key in (heuristic_rule_keys or set()) else [] + else: + matches = _find_matches(rule, selected, []) + if not matches: + connected_to_declared = any( + (source_key == rule.key and target_key in initially_declared_rule_keys) + or (target_key == rule.key and source_key in initially_declared_rule_keys) + for source_key, target_key, _ in RELATION_RULES + ) + if connected_to_declared: + matches = _find_matches(rule, [], relevant_non_background) if not matches: continue evidence_ids = list(dict.fromkeys(str(item.get("id")) for item in matches if item.get("id"))) - existing = _find_existing(spec, rule) if existing: _, item = existing current = _label(item) @@ -509,5 +604,8 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) "repaired_relations": repaired_relations, "grounded_statements": {field: ids for field, ids in grounded_statements.items() if ids}, "grounded_entities": grounded_entities, + "conservative_expansion": conservative_expansion, + "expansion_page_scope": sorted(selected_pages), + "new_entities_require_declared_neighbor": conservative_expansion, "rules_are_evidence_gated": True, } diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 35f95ed..9cb4c7b 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -703,7 +703,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 17, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 25, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 831132b..417025c 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -48,6 +48,53 @@ def test_planning_contract_treats_relation_labels_as_intermediate_artifacts(self self.assertEqual(result["entity_recall"], 1.0) self.assertEqual(result["relation_recall"], 1.0) + def test_planning_contract_prefers_real_entity_with_variable_suffix_over_relation_label(self): + expected = { + "entities": [ + {"id": "refiner", "label": "Refine"}, + {"id": "result", "label": "Refined Output"}, + ], + "relations": [{"source": "refiner", "target": "result"}], + } + specification = { + "modules": [{"id": "refine", "name": "Refine"}], + "outputs": [{"id": "final", "name": "Refined Output (yn)"}], + "relations": [{"source": "refine", "target": "final", "label": "Refined Output"}], + } + + result = _score_planning_contract(expected, specification) + + self.assertEqual(result["entity_mapping"]["result"], "final") + self.assertEqual(result["relation_recall"], 1.0) + + def test_planning_contract_uses_relations_to_disambiguate_equal_input_labels(self): + expected = { + "entities": [ + {"id": "task_input", "label": "Input"}, + {"id": "generator", "label": "Generate"}, + ], + "relations": [{"source": "task_input", "target": "generator"}], + } + specification = { + "inputs": [ + {"id": "iteration_input", "name": "Input (y0)"}, + {"id": "task_input", "name": "Input"}, + ], + "modules": [ + {"id": "model", "name": "Model"}, + {"id": "generate", "name": "Generate"}, + ], + "relations": [ + {"source": "iteration_input", "target": "model"}, + {"source": "task_input", "target": "generate"}, + ], + } + + result = _score_planning_contract(expected, specification) + + self.assertEqual(result["entity_mapping"]["task_input"], "task_input") + self.assertEqual(result["relation_recall"], 1.0) + def test_bundled_cases_validate_and_list(self): p2i = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" i2p = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 25bc660..764921d 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -363,6 +363,160 @@ def test_generic_completion_keeps_clip_modalities_and_encoders_distinct(self): self.assertTrue(any(item.get("id") == "fallback_contrastive" for item in spec["modules"])) self.assertEqual(spec["topology"], "multimodal") + def test_rich_clip_contract_reuses_singular_embedding_entities(self): + caption = "Figure 1. CLIP trains an image encoder and a text encoder on paired image and text inputs, then embeds class descriptions for a zero-shot classifier." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "The image and text embeddings are aligned by a contrastive objective.", "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [ + {"id": "images", "name": "Image", "evidence_ids": ["E0001"]}, + {"id": "texts", "name": "Text", "evidence_ids": ["E0001"]}, + {"id": "classes", "name": "Class Descriptions", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "image_encoder", "name": "Image Encoder", "evidence_ids": ["E0001"]}, + {"id": "text_encoder", "name": "Text Encoder", "evidence_ids": ["E0001"]}, + {"id": "contrastive", "name": "Contrastive Learning", "evidence_ids": ["E0002"]}, + ], + "outputs": [ + {"id": "image_embedding", "name": "image embedding", "evidence_ids": ["E0002"]}, + {"id": "text_embedding", "name": "text embedding", "evidence_ids": ["E0002"]}, + {"id": "classifier", "name": "Zero-Shot Classifier", "evidence_ids": ["E0001"]}, + ], + "relations": [{"source": "image_embedding", "target": "classifier", "type": "classification", "evidence_ids": ["E0001", "E0002"]}], + "innovations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + embedding_labels = [ + item.get("name") + for field in ("inputs", "modules", "outputs") + for item in spec[field] + if "embedding" in str(item.get("name") or "").casefold() + ] + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(len([label for label in embedding_labels if "image" in label.casefold()]), 1) + self.assertEqual(len([label for label in embedding_labels if "text" in label.casefold()]), 1) + self.assertIn(("text_embedding", "classifier"), pairs) + + def test_rich_bert_contract_uses_selected_caption_for_context_rules(self): + caption = "Figure 2: BERT input representation. The input embeddings are the sum of the token embeddings, the segmentation embeddings and the position embeddings." + parsed = { + "page_count": 12, + "document_index": {"figures": [{"page": 5, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 5, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "tokens", "name": "Input Sequence", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "token_embeddings", "name": "Token Embeddings", "evidence_ids": ["E0001"]}, + {"id": "position_embeddings", "name": "Position Embeddings", "evidence_ids": ["E0001"]}, + {"id": "bert", "name": "Bidirectional Transformer Encoder", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "tokens", "target": "token_embeddings", "type": "embedding", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + ids = {item.get("name"): item.get("id") for field in ("inputs", "modules", "outputs") for item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn("Segment Embeddings", ids) + self.assertIn("Input Representation", ids) + self.assertIn((ids["Segment Embeddings"], ids["Input Representation"]), pairs) + + def test_rich_nerf_contract_reuses_parenthesized_mlp_and_recovers_distant_method_step(self): + overview = "Figure 2: A neural radiance field samples 5D coordinates along camera rays and uses an MLP to predict color and volume density for volume rendering." + encoding = "We transform each sampled 3D point with a positional encoding before passing it into the MLP." + parsed = { + "page_count": 10, + "document_index": {"figures": [{"page": 2, "caption": overview}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 6, "kind": "paragraph", "text": encoding, "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "ray", "name": "Camera Rays", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "points", "name": "Sampled 3D Points", "evidence_ids": ["E0001", "E0002"]}, + {"id": "field", "name": "MLP (F_theta)", "evidence_ids": ["E0001", "E0002"]}, + {"id": "render", "name": "Volume Rendering", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "image", "name": "Rendered Image", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [{"source": "ray", "target": "points", "type": "sampling", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + mlps = [item for item in spec["modules"] if str(item.get("name") or "").casefold().startswith("mlp")] + positional = next(item for item in spec["modules"] if item.get("name") == "Positional Encoding") + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(len(mlps), 1) + self.assertIn(("points", positional["id"]), pairs) + self.assertIn((positional["id"], "field"), pairs) + + def test_rich_transformer_contract_recovers_connected_steps_from_distant_method_evidence(self): + caption = "Figure 1: The Transformer follows an encoder-decoder architecture." + method = "We use learned embeddings to convert the input tokens and output tokens to vectors. Output embeddings are offset by one position. A linear transformation and softmax convert the decoder output to predicted next-token probabilities." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 6, "kind": "paragraph", "text": method, "section_hint": "Method", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [ + {"id": "input_embedding", "name": "Input Embedding", "evidence_ids": ["E0002"]}, + {"id": "output_embedding", "name": "Output Embedding", "evidence_ids": ["E0002"]}, + {"id": "target_sequence", "name": "output sequence", "evidence_ids": ["E0002"]}, + ], + "modules": [ + {"id": "encoder", "name": "Encoder Stack", "evidence_ids": ["E0001"]}, + {"id": "decoder", "name": "Decoder Stack", "evidence_ids": ["E0001"]}, + {"id": "softmax", "name": "Softmax", "evidence_ids": ["E0002"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "encoder", "target": "decoder", "type": "cross_attention", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + ids = {item.get("name"): item.get("id") for field in ("inputs", "modules", "outputs") for item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn((ids["Inputs"], "input_embedding"), pairs) + self.assertEqual(ids["Outputs (shifted right)"], "target_sequence") + self.assertIn(("target_sequence", "output_embedding"), pairs) + self.assertIn(("softmax", ids["Output Probabilities"]), pairs) + def test_generic_completion_recovers_nerf_branch_without_false_multimodal_topology(self): caption = "Fig. 2: An overview of our neural radiance field and differentiable rendering procedure. We synthesize images by sampling 5D coordinates along camera rays, feeding locations and viewing direction into an MLP to produce a color and volume density, and using volume rendering to composite these values into an image." positional = "Fig. 4: Our full model passes input coordinates through a high-frequency positional encoding." @@ -506,6 +660,47 @@ def test_rag_contract_repairs_latent_endpoint_alias_and_acronym_evidence(self): self.assertFalse(any("latent_z" in (item["source"], item["target"]) for item in spec["relations"])) self.assertTrue(validation["ok"]) + def test_rich_sam_contract_does_not_import_unrelated_clip_components(self): + sam_caption = "Figure 1: The promptable segmentation model contains an image encoder, prompt encoder, and mask decoder that produces a valid segmentation mask." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": sam_caption}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": sam_caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 1, "kind": "paragraph", "text": "CLIP uses class descriptions, a text encoder, text embeddings, and contrastive learning.", "section_hint": "Introduction", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "image", "name": "Image", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "image_encoder", "name": "Image Encoder", "evidence_ids": ["E0001"]}, + {"id": "prompt_encoder", "name": "Prompt Encoder", "evidence_ids": ["E0001"]}, + {"id": "mask_decoder", "name": "Mask Decoder", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "mask", "name": "Valid Segmentation Mask", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [ + {"source": "image", "target": "image_encoder", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "image_encoder", "target": "mask_decoder", "type": "feature_flow", "evidence_ids": ["E0001"]}, + {"source": "prompt_encoder", "target": "mask_decoder", "type": "conditioning", "evidence_ids": ["E0001"]}, + {"source": "mask_decoder", "target": "mask", "type": "prediction", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = {_item.get("name") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + + self.assertNotIn("Class Descriptions", labels) + self.assertNotIn("Text Encoder", labels) + self.assertNotIn("Contrastive Learning", labels) + self.assertTrue(plan["contract_completion_report"]["conservative_expansion"]) + def test_generic_completion_repairs_vlm_ids_labels_and_relation_grounding(self): caption = "Figure 1: Pre-training and fine-tuning procedures." parsed = { From bd123176a40dca9ab34cdd7447d2775e87106b93 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 09:19:39 +0800 Subject: [PATCH 31/74] Recover unnumbered PDF headings from typography --- README.md | 2 +- rfs/evaluation/benchmarking.py | 4 +++ rfs/paper_to_image/analyzer.py | 48 ++++++++++++++++++++++++++-- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 8 +++-- tests/unit/test_benchmarking.py | 3 ++ tests/unit/test_paper_analyzer.py | 20 ++++++++++++ 7 files changed, 80 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0d71c0d..4fc280f 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records `max_column_count`, `multi_column_page_count`, and `rotated_pages` so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, and rotation metrics so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index a450727..5b126b2 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -852,6 +852,10 @@ def percentile(values: list[float], fraction: float) -> float: "mean_evidence_char_count": _mean(extraction_metric("evidence_char_count")), "max_detected_column_count": max((int(value) for value in extraction_metric("max_column_count")), default=0), "multi_column_page_total": sum(int(value) for value in extraction_metric("multi_column_page_count")), + "mean_section_count": _mean(extraction_metric("section_count")), + "typographic_heading_total": sum(int(value) for value in extraction_metric("typographic_heading_count")), + "figure_caption_total": sum(int(value) for value in extraction_metric("figure_caption_count")), + "table_caption_total": sum(int(value) for value in extraction_metric("table_caption_count")), "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 9df5a21..bd37120 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -75,6 +75,24 @@ def _mojibake_rate(text: str) -> float: return round(sum(value.count(marker) for marker in MOJIBAKE_MARKERS) / max(1, len(value)), 6) +def _looks_like_typographic_heading(text: str) -> bool: + value = _clean(text) + words = re.findall(r"[A-Za-z][A-Za-z0-9+&/-]*", value) + first_alpha = next((char for char in value if char.isalpha()), "") + first_char = value[0] if value else "" + if not 2 <= len(value) <= 100 or not 1 <= len(words) <= 10: + return False + if not first_char.isalpha() or (first_char.isascii() and not re.search(r"[a-z]", value)): + return False + if re.match(r"^(?:figure|fig\.|table)\b", value, re.IGNORECASE) or re.search(r"[=<>]{1,2}|\bdoi\b|https?://", value, re.IGNORECASE): + return False + if value.endswith((",", ";", "?", "!")) or (value.endswith(".") and len(words) > 5): + return False + if first_alpha and first_alpha.lower() != first_alpha.upper() and not first_alpha.isupper(): + return False + return True + + def _block_kind(text: str, width_ratio: float = 0.0) -> str: value = _clean(text) if re.match(r"^(figure|fig\.|table|图|圖|表)\s*[a-z0-9一二三四五六七八九十]+", value, re.IGNORECASE): @@ -232,7 +250,18 @@ def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: round(max(float(value[2]) for value in bbox_values), 3), round(max(float(value[3]) for value in bbox_values), 3), ] - records.append({"bbox": bbox, "text": text, "source": "pymupdf", "confidence": 1.0, "parent_block": parent_index}) + sizes = [float(span.get("size") or 0.0) for span in spans if float(span.get("size") or 0.0) > 0] + fonts = [str(span.get("font") or "") for span in spans] + font_bold = any(int(span.get("flags") or 0) & 16 for span in spans) or any(re.search(r"(?:bold|demi|semi|medi)", font, re.IGNORECASE) for font in fonts) + records.append({ + "bbox": bbox, + "text": text, + "source": "pymupdf", + "confidence": 1.0, + "parent_block": parent_index, + "font_size": round(max(sizes), 3) if sizes else None, + "font_bold": font_bold, + }) return records @@ -771,6 +800,7 @@ def _read_non_pdf_pages(path: Path, max_chars: int) -> tuple[list[dict[str, Any] def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: headings = [] seen = set() + body_started = False for page in pages: for block in page.get("blocks", []): lines = [_clean(value) for value in str(block.get("text", "")).splitlines() if _clean(value)] @@ -786,19 +816,23 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: for alias in aliases for compact_alias in [_normalized_compare_text(alias)] ) + typographic = body_started and bool(block.get("font_bold")) and _looks_like_typographic_heading(line) if numbered: title_candidate = numbered.group(2).strip() first_alpha = next((char for char in title_candidate if char.isalpha()), "") + if not title_candidate or not title_candidate[0].isalpha(): + first_alpha = "" has_case = bool(first_alpha and first_alpha.lower() != first_alpha.upper()) if not first_alpha or (has_case and not first_alpha.isupper()) or re.search(r"[=±∑∫]", title_candidate) or title_candidate.endswith(".") or len(title_candidate) > 80 or len(title_candidate.split()) > 10: numbered = None - if not (explicit or numbered or known): + if not (explicit or numbered or known or typographic): continue - normalized = numbered.group(2).strip() if numbered else line.strip() + normalized = numbered.group(2).strip() if numbered else line.strip().rstrip(".:") key = normalized.casefold() if normalized and key not in seen: seen.add(key) headings.append({"id": f"section_{len(headings) + 1:03d}", "title": normalized, "page": page["page"], "block_id": block["id"], "line_index": line_index}) + body_started = body_started or bool(explicit or numbered or known) return headings[:80] @@ -999,6 +1033,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st partial_scan = bool(fully_scanned_source and 0 < ocr_count < len(pages)) pdf_type = "scanned" if fully_scanned_source else "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" coverage = _section_coverage(pages, index.get("sections", [])) + indexed_section_blocks = {(int(item.get("page") or 0), str(item.get("block_id") or "")) for item in index.get("sections", [])} warnings = list(details.get("warnings", [])) ocr_blocks = [block for page in pages if page.get("used_ocr") for block in page.get("blocks", []) if isinstance(block, dict)] ocr_confidences = [float(block.get("confidence") or 0.0) for block in ocr_blocks] @@ -1056,6 +1091,8 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "section_coverage": coverage, "figure_caption_count": len(index.get("figures", [])), "table_caption_count": len(index.get("tables", [])), + "section_count": len(index.get("sections", [])), + "typographic_heading_count": sum(bool(block.get("font_bold")) and (int(page["page"]), str(block.get("id") or "")) in indexed_section_blocks for page in pages for block in page.get("blocks", [])), "poppler_available": details.get("poppler_available", False), "warnings": warnings, "elapsed_seconds": details.get("elapsed_seconds", 0.0), @@ -1082,6 +1119,11 @@ def parse_paper( pages, loader = _read_non_pdf_pages(source, max_chars) details = {"warnings": [], "ocr_candidate_pages": [], "ocr_pages": [], "poppler_available": False, "elapsed_seconds": 0.0} sections = _heading_candidates(pages) + section_blocks = {(int(item.get("page") or 0), str(item.get("block_id") or "")) for item in sections} + for page in pages: + for block in page.get("blocks", []): + if (int(page["page"]), str(block.get("id") or "")) in section_blocks: + block["kind"] = "heading" document_index = _document_index(pages, sections) evidence = _build_evidence(pages, sections, document_index, max_chars) all_text = "\n\n".join(page.get("text", "") for page in pages) diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 92a58d0..85a26da 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 11 +DOCUMENT_CACHE_VERSION = 14 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 9cb4c7b..2027f9b 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -12,7 +12,7 @@ from ..utils import ensure_dir, read_json, write_json, write_text from .analyzer import paper_markdown, parse_paper from .contract_completion import augment_contract_from_evidence -from .document_cache import read_document_cache, write_document_cache +from .document_cache import DOCUMENT_CACHE_VERSION, read_document_cache, write_document_cache from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_fast_paper_contract, plan_paper_image, validate_plan_grounding from .review import build_paper_review, detect_domain_profile, validate_review_coverage @@ -703,7 +703,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 25, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 26, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -915,6 +915,10 @@ def prepare_paper_figure_contract( "max_column_count": parsed["extraction_report"].get("max_column_count"), "multi_column_page_count": parsed["extraction_report"].get("multi_column_page_count"), "rotated_pages": parsed["extraction_report"].get("rotated_pages", []), + "section_count": parsed["extraction_report"].get("section_count", 0), + "typographic_heading_count": parsed["extraction_report"].get("typographic_heading_count", 0), + "figure_caption_count": parsed["extraction_report"].get("figure_caption_count", 0), + "table_caption_count": parsed["extraction_report"].get("table_caption_count", 0), "section_coverage": parsed["extraction_report"].get("section_coverage", {}), "missing_priority_sections": [name for name, present in parsed["extraction_report"].get("section_coverage", {}).items() if not present], "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 417025c..16c30db 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -137,6 +137,9 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("mean_evidence_page_coverage_ratio", result["aggregate"]) self.assertIn("ocr_scheduled_page_total", result["aggregate"]) self.assertIn("max_detected_column_count", result["aggregate"]) + self.assertIn("mean_section_count", result["aggregate"]) + self.assertIn("typographic_heading_total", result["aggregate"]) + self.assertIn("figure_caption_total", result["aggregate"]) self.assertIn("max_ocr_worker_count", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index e97073e..ea1b1d8 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -52,6 +52,26 @@ def draw(page): self.assertIn("Introduction", parsed["headings"]) self.assertTrue(all(item.get("block_id") for item in parsed["evidence"])) + def test_bold_unnumbered_headings_create_section_boundaries(self): + def draw(page): + page.insert_text((50, 55), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((50, 75, 550, 145), "This paper introduces a reliable architecture with enough text for extraction quality checks.", fontsize=10) + page.insert_text((50, 185), "Model Architecture", fontname="hebo", fontsize=11) + page.insert_textbox((50, 205, 550, 285), "The encoder maps the input sequence into a representation used by the decoder and output layer.", fontsize=10) + page.insert_text((50, 325), "Encoder and Decoder Stacks", fontname="hebo", fontsize=10) + page.insert_textbox((50, 345, 550, 425), "Each stack contains attention and feed-forward modules with residual connections.", fontsize=10) + + parsed = parse_paper(self._pdf(draw)) + evidence_by_text = {item["text"]: item for item in parsed["evidence"]} + architecture_text = next(text for text in evidence_by_text if text.startswith("The encoder maps")) + stack_text = next(text for text in evidence_by_text if text.startswith("Each stack contains")) + + self.assertIn("Model Architecture", parsed["headings"]) + self.assertIn("Encoder and Decoder Stacks", parsed["headings"]) + self.assertEqual(evidence_by_text[architecture_text]["section_hint"], "Model Architecture") + self.assertEqual(evidence_by_text[stack_text]["section_hint"], "Encoder and Decoder Stacks") + self.assertGreaterEqual(parsed["extraction_report"]["typographic_heading_count"], 3) + def test_multiline_figure_caption_is_combined_into_priority_evidence(self): def draw(page): page.insert_textbox((50, 80, 550, 150), "Figure 1: Three interconnected components: a promptable task,\na model, and a data engine for collecting a large dataset.", fontsize=10) From 66de30fe2f2e02d49bf4e97b9642b54f7f471a86 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 10:38:24 +0800 Subject: [PATCH 32/74] Add reproducible PDF extraction stress benchmarks --- README.md | 8 + benchmarks/README.md | 7 +- rfs/cli.py | 10 +- rfs/evaluation/__init__.py | 2 + rfs/evaluation/benchmarking.py | 1 + rfs/evaluation/pdf_extraction_benchmark.py | 256 +++++++++++++++++++++ rfs/paper_to_image/analyzer.py | 93 +++++++- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 1 + tests/unit/test_benchmarking.py | 13 ++ 10 files changed, 378 insertions(+), 15 deletions(-) create mode 100644 rfs/evaluation/pdf_extraction_benchmark.py diff --git a/README.md b/README.md index 4fc280f..a228a24 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,14 @@ Run a repeatable fast suite with provider, cache, recall, and timing aggregation rfs benchmark fast-suite --root benchmarks --out output\benchmarks\fast-suite --planner-mode heuristic --json ``` +Run the generated multi-layout PDF extraction stress suite independently of paper semantics: + +```powershell +rfs benchmark pdf-suite --out output\benchmarks\pdf-extraction --ocr-engine auto --json +``` + +It generates fixed two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs. The deterministic four-case tier always runs; `--ocr-engine auto` adds a fifth real local-OCR probe. Each case writes its fixture, rendered preview, document model, extraction report, assertions, and timing so reading-order or OCR regressions are reproducible without committing third-party PDFs. + Use `paper-to-image` when the required endpoint is a generated raster framework figure rather than an editable PowerPoint file. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts diff --git a/benchmarks/README.md b/benchmarks/README.md index 31009f7..4fae871 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ # ResearchFigureStudio Benchmarks -ResearchFigureStudio uses two independent benchmark suites so failures can be attributed to either reference-image generation or editable reconstruction. +ResearchFigureStudio uses two independent product benchmark suites plus a generated PDF extraction stress suite, so failures can be attributed to paper parsing, reference-image generation, or editable reconstruction. ## Suites @@ -16,6 +16,10 @@ Measures rendered visual fidelity, object and relation reconstruction, text alig A full-slide copy of the reference image is a hard failure even if pixel similarity is perfect. +### Generated PDF extraction stress suite + +`benchmark pdf-suite` creates deterministic native two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs at runtime. It validates section boundaries, cross-column reading order, displayed coordinates, caption recovery, local OCR scheduling, and elapsed time. `--ocr-engine auto` adds a real installed-OCR probe after the deterministic adapter-backed tier. + ## Commands ```powershell @@ -24,6 +28,7 @@ rfs benchmark validate --case benchmarks/paper-to-image/cases/001_linear_pipelin rfs benchmark fetch --case benchmarks/paper-to-image/cases/101_vit_linear --json rfs benchmark fast --case benchmarks/paper-to-image/cases/101_vit_linear --out output/benchmarks/vit_fast --planner-mode heuristic --json rfs benchmark fast-suite --root benchmarks --out output/benchmarks/fast_suite --planner-mode heuristic --json +rfs benchmark pdf-suite --out output/benchmarks/pdf_extraction --ocr-engine auto --json rfs benchmark run --case benchmarks/paper-to-image/cases/001_linear_pipeline --out output/benchmarks/p2i_001 --json rfs benchmark score --case benchmarks/image-to-ppt/cases/001_three_stage_layout --run output/rebuild_case --json ``` diff --git a/rfs/cli.py b/rfs/cli.py index 1745783..dcf9708 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -11,7 +11,7 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable -from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case +from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, run_pdf_extraction_stress_suite, score_benchmark_case, validate_benchmark_case from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro @@ -304,7 +304,7 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") - benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "run", "score"]) + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score"]) benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") @@ -315,7 +315,7 @@ def build_parser() -> argparse.ArgumentParser: benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") - benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for benchmark fast.") + benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") benchmark.add_argument("--json", action="store_true", help="Emit JSON.") validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") @@ -577,6 +577,10 @@ def main(argv: list[str] | None = None) -> int: if not args.out: parser.error("benchmark fast-suite requires --out") result = run_fast_benchmark_suite(args.root, args.out, case_ids=args.case_id, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine) + elif args.benchmark_action == "pdf-suite": + if not args.out: + parser.error("benchmark pdf-suite requires --out") + result = run_pdf_extraction_stress_suite(args.out, ocr_engine=args.ocr_engine) else: if not args.case or not args.run_dir: parser.error("benchmark score requires --case and --run") diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index b138db0..44582bd 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -2,6 +2,7 @@ from .rebuild_visual import run_rebuild_visual_quality_check from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case +from .pdf_extraction_benchmark import run_pdf_extraction_stress_suite __all__ = [ "list_benchmark_cases", @@ -10,6 +11,7 @@ "run_benchmark_case", "run_fast_benchmark_case", "run_fast_benchmark_suite", + "run_pdf_extraction_stress_suite", "score_benchmark_case", "validate_benchmark_case", ] diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 5b126b2..86bc4f4 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -854,6 +854,7 @@ def percentile(values: list[float], fraction: float) -> float: "multi_column_page_total": sum(int(value) for value in extraction_metric("multi_column_page_count")), "mean_section_count": _mean(extraction_metric("section_count")), "typographic_heading_total": sum(int(value) for value in extraction_metric("typographic_heading_count")), + "merged_heading_line_total": sum(int(value) for value in extraction_metric("merged_heading_line_count")), "figure_caption_total": sum(int(value) for value in extraction_metric("figure_caption_count")), "table_caption_total": sum(int(value) for value in extraction_metric("table_caption_count")), "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py new file mode 100644 index 0000000..90bdec4 --- /dev/null +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import math +import time +from pathlib import Path +from typing import Any, Callable + +from ..paper_to_image.analyzer import parse_paper +from ..utils import ensure_dir, write_json + + +def _save_document(document: Any, target: Path) -> Path: + target.parent.mkdir(parents=True, exist_ok=True) + document.save(str(target)) + document.close() + return target + + +def _native_two_column_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 48), "Abstract", fontname="hebo", fontsize=13) + page.insert_textbox((45, 68, 555, 125), "A deterministic two-column extraction benchmark with enough scientific text for parser quality gates.", fontsize=10) + page.insert_text((45, 155), "2 Method", fontname="hebo", fontsize=13) + page.insert_textbox((45, 185, 275, 260), "LEFT_TOP encoder evidence begins the method flow and establishes the input representation.", fontsize=10) + page.insert_textbox((45, 300, 275, 375), "LEFT_BOTTOM decoder evidence follows the encoder in the left reading column.", fontsize=10) + page.insert_textbox((325, 185, 555, 260), "RIGHT_TOP retrieval evidence belongs after the complete left column.", fontsize=10) + page.insert_textbox((325, 300, 555, 375), "RIGHT_BOTTOM output evidence finishes the two-column reading order.", fontsize=10) + page.insert_textbox((45, 430, 555, 485), "Figure 1: Overview of the encoder, decoder, retrieval module, and final output.", fontsize=10) + page.insert_text((45, 540), "3 Conclusion", fontname="hebo", fontsize=13) + page.insert_textbox((45, 560, 555, 620), "The benchmark concludes with a complete evidence-grounded pipeline.", fontsize=10) + return _save_document(document, target) + + +def _bold_heading_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 70, 555, 130), "A paper with unnumbered bold sections validates typography-aware section recovery.", fontsize=10) + page.insert_text((45, 175), "Model Architecture", fontname="hebo", fontsize=11) + page.insert_textbox((45, 195, 555, 260), "The architecture contains an image encoder, a prompt encoder, and a mask decoder.", fontsize=10) + page.insert_text((45, 305), "Encoder and Decoder Stacks", fontname="hebo", fontsize=10) + page.insert_textbox((45, 325, 555, 390), "The encoder representation conditions the decoder before the final prediction.", fontsize=10) + page.insert_text((45, 435), "Conclusion", fontname="hebo", fontsize=11) + page.insert_textbox((45, 455, 555, 520), "Typography-based boundaries preserve the scientific section context.", fontsize=10) + return _save_document(document, target) + + +def _rotated_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 75, 555, 145), "A rotated scientific page must retain displayed coordinates and readable text.", fontsize=10) + page.insert_text((45, 190), "2 Method", fontname="hebo", fontsize=12) + page.insert_textbox((45, 215, 555, 290), "The rotated encoder sends its representation to a decoder and final output.", fontsize=10) + page.insert_textbox((45, 335, 555, 390), "Figure 1: Rotated architecture overview with encoder and decoder stages.", fontsize=10) + page.set_rotation(90) + return _save_document(document, target) + + +def _mixed_scan_fixture(target: Path) -> Path: + import fitz + + scan_source = fitz.open() + scan_page = scan_source.new_page(width=600, height=800) + scan_page.insert_text((50, 65), "2 Method", fontname="hebo", fontsize=18) + scan_page.insert_textbox((50, 105, 550, 190), "The scanned encoder passes evidence to the scanned decoder and output module.", fontsize=15) + scan_page.insert_textbox((50, 245, 550, 330), "Figure 1: Scanned architecture overview with encoder decoder and output.", fontsize=15) + pixmap = scan_page.get_pixmap(dpi=180, alpha=False) + scan_png = pixmap.tobytes("png") + scan_source.close() + + document = fitz.open() + page1 = document.new_page(width=600, height=800) + page1.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page1.insert_textbox((45, 75, 555, 155), "A mixed PDF combines native text with one scanned method page for local OCR fallback.", fontsize=10) + page2 = document.new_page(width=600, height=800) + page2.insert_image(page2.rect, stream=scan_png) + page3 = document.new_page(width=600, height=800) + page3.insert_text((45, 50), "3 Conclusion", fontname="hebo", fontsize=12) + page3.insert_textbox((45, 75, 555, 155), "The mixed extraction path preserves native pages and recovers the scanned method evidence.", fontsize=10) + return _save_document(document, target) + + +def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: + if "page_002" not in image_path.name: + return [] + lines = [ + ("2 Method", 110), + ("The scanned encoder passes evidence to the scanned decoder and output module.", 220), + ("Figure 1: Scanned architecture overview with encoder decoder and output.", 420), + ] + return [ + { + "text": text, + "confidence": 0.99, + "quad": [[90, top], [1240, top], [1240, top + 55], [90, top + 55]], + } + for text, top in lines + ] + + +def _render_preview(source: Path, target: Path, page_index: int = 0) -> Path: + import fitz + + document = fitz.open(str(source)) + try: + pixmap = document[page_index].get_pixmap(dpi=110, alpha=False) + target.parent.mkdir(parents=True, exist_ok=True) + pixmap.save(str(target)) + finally: + document.close() + return target + + +def _check(name: str, passed: bool, actual: Any, expected: Any) -> dict[str, Any]: + return {"name": name, "passed": bool(passed), "actual": actual, "expected": expected} + + +def _run_case( + case_id: str, + fixture: Path, + out_dir: Path, + checks: Callable[[dict[str, Any]], list[dict[str, Any]]], + *, + ocr_engine: str = "off", + ocr_adapter: Callable | None = None, + preview_page: int = 0, +) -> dict[str, Any]: + started = time.monotonic() + parsed = parse_paper(fixture, ocr_engine=ocr_engine, ocr_adapter=ocr_adapter) + elapsed = round(time.monotonic() - started, 4) + case_dir = ensure_dir(out_dir / case_id) + write_json(case_dir / "document_model.json", parsed) + write_json(case_dir / "extraction_report.json", parsed.get("extraction_report", {})) + preview = _render_preview(fixture, case_dir / f"preview_page_{preview_page + 1}.png", preview_page) + assertions = checks(parsed) + result = { + "case_id": case_id, + "ok": all(item.get("passed") for item in assertions), + "fixture": str(fixture), + "preview": str(preview), + "elapsed_seconds": elapsed, + "pdf_type": parsed.get("extraction_report", {}).get("pdf_type"), + "page_count": parsed.get("page_count"), + "section_count": parsed.get("extraction_report", {}).get("section_count"), + "figure_caption_count": parsed.get("extraction_report", {}).get("figure_caption_count"), + "ocr_pages": parsed.get("extraction_report", {}).get("ocr_pages", []), + "assertions": assertions, + } + write_json(case_dir / "benchmark_result.json", result) + return result + + +def _percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * fraction) - 1)) + return round(ordered[index], 4) + + +def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> dict[str, Any]: + root = ensure_dir(out).resolve() + fixtures = ensure_dir(root / "fixtures") + native = _native_two_column_fixture(fixtures / "native_two_column.pdf") + bold = _bold_heading_fixture(fixtures / "unnumbered_bold_sections.pdf") + rotated = _rotated_fixture(fixtures / "rotated_native_page.pdf") + mixed = _mixed_scan_fixture(fixtures / "mixed_scan.pdf") + + results = [ + _run_case( + "native_two_column", + native, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("cross_column_caption_order", parsed["pages"][0]["text"].index("RIGHT_BOTTOM") < parsed["pages"][0]["text"].index("Figure 1:") < parsed["pages"][0]["text"].index("3 Conclusion"), parsed["pages"][0]["text"], "RIGHT_BOTTOM < Figure 1 < Conclusion"), + _check("figure_caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "conclusion")), parsed["extraction_report"].get("section_coverage"), "abstract/method/conclusion"), + ], + ), + _run_case( + "unnumbered_bold_sections", + bold, + root, + lambda parsed: [ + _check("model_architecture_heading", "Model Architecture" in parsed.get("headings", []), parsed.get("headings", []), "Model Architecture"), + _check("stack_heading", "Encoder and Decoder Stacks" in parsed.get("headings", []), parsed.get("headings", []), "Encoder and Decoder Stacks"), + _check("section_hint", any(item.get("section_hint") == "Model Architecture" and "image encoder" in str(item.get("text") or "").casefold() for item in parsed.get("evidence", [])), [item.get("section_hint") for item in parsed.get("evidence", []) if "image encoder" in str(item.get("text") or "").casefold()], "Model Architecture"), + ], + ), + _run_case( + "rotated_native_page", + rotated, + root, + lambda parsed: [ + _check("rotation", parsed["extraction_report"].get("rotated_pages") == [1], parsed["extraction_report"].get("rotated_pages"), [1]), + _check("display_coordinates", all(0 <= block["bbox"][0] <= block["bbox"][2] <= parsed["pages"][0]["width"] and 0 <= block["bbox"][1] <= block["bbox"][3] <= parsed["pages"][0]["height"] for block in parsed["pages"][0]["blocks"]), parsed["pages"][0]["blocks"], "all blocks inside displayed page"), + _check("caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), + ], + ), + _run_case( + "mixed_scan_fixture_adapter", + mixed, + root, + lambda parsed: [ + _check("mixed_pdf", parsed["extraction_report"].get("pdf_type") == "mixed", parsed["extraction_report"].get("pdf_type"), "mixed"), + _check("local_ocr", parsed["extraction_report"].get("ocr_pages") == [2], parsed["extraction_report"].get("ocr_pages"), [2]), + _check("method_recovered", parsed["extraction_report"].get("section_coverage", {}).get("method") is True, parsed["extraction_report"].get("section_coverage"), "method=true"), + _check("scanned_caption", any("Scanned architecture overview" in str(item.get("caption") or "") for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "scanned Figure 1 caption"), + ], + ocr_engine="easyocr", + ocr_adapter=_fixture_ocr_adapter, + preview_page=1, + ), + ] + + if ocr_engine != "off": + results.append(_run_case( + "mixed_scan_runtime_ocr", + mixed, + root, + lambda parsed: [ + _check("ocr_completed", 2 in parsed["extraction_report"].get("ocr_pages", []), parsed["extraction_report"].get("ocr_pages", []), "contains page 2"), + _check("method_token", "method" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains method"), + _check("encoder_token", "encoder" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains encoder"), + ], + ocr_engine=ocr_engine, + preview_page=1, + )) + + elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] + report = { + "summary": "Deterministic multi-layout PDF extraction stress suite completed.", + "ok": all(item.get("ok") for item in results), + "out_dir": str(root), + "ocr_engine": ocr_engine, + "aggregate": { + "case_count": len(results), + "passed_case_count": sum(bool(item.get("ok")) for item in results), + "mean_elapsed_seconds": round(sum(elapsed) / max(1, len(elapsed)), 4), + "p95_elapsed_seconds": _percentile(elapsed, 0.95), + "ocr_case_count": sum(bool(item.get("ocr_pages")) for item in results), + }, + "cases": results, + } + write_json(root / "pdf_extraction_stress_report.json", report) + return report diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index bd37120..288f788 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -86,6 +86,8 @@ def _looks_like_typographic_heading(text: str) -> bool: return False if re.match(r"^(?:figure|fig\.|table)\b", value, re.IGNORECASE) or re.search(r"[=<>]{1,2}|\bdoi\b|https?://", value, re.IGNORECASE): return False + if value.endswith(":") and len(words) <= 2: + return False if value.endswith((",", ";", "?", "!")) or (value.endswith(".") and len(words) > 5): return False if first_alpha and first_alpha.lower() != first_alpha.upper() and not first_alpha.isupper(): @@ -164,7 +166,7 @@ def _reading_order(blocks: list[dict[str, Any]], page_width: float) -> tuple[lis usable = [item for item in blocks if _clean(item.get("text", ""))] if not usable: return [], 0.0 - spanning = [item for item in usable if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.62] + spanning = [item for item in usable if float(item["bbox"][2]) - float(item["bbox"][0]) >= page_width * 0.56] non_spanning = [item for item in usable if item not in spanning] detection_items = [ item @@ -251,8 +253,15 @@ def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: round(max(float(value[3]) for value in bbox_values), 3), ] sizes = [float(span.get("size") or 0.0) for span in spans if float(span.get("size") or 0.0) > 0] - fonts = [str(span.get("font") or "") for span in spans] - font_bold = any(int(span.get("flags") or 0) & 16 for span in spans) or any(re.search(r"(?:bold|demi|semi|medi)", font, re.IGNORECASE) for font in fonts) + span_bold = [ + bool(int(span.get("flags") or 0) & 16) or bool(re.search(r"(?:bold|demi|semi|medi)", str(span.get("font") or ""), re.IGNORECASE)) + for span in spans + ] + span_lengths = [max(1, len(_clean(span.get("text", "")))) for span in spans] + bold_characters = sum(length for length, bold in zip(span_lengths, span_bold) if bold) + total_characters = sum(span_lengths) + font_bold_ratio = round(bold_characters / max(1, total_characters), 4) + font_bold = font_bold_ratio >= 0.8 records.append({ "bbox": bbox, "text": text, @@ -261,6 +270,7 @@ def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: "parent_block": parent_index, "font_size": round(max(sizes), 3) if sizes else None, "font_bold": font_bold, + "font_bold_ratio": font_bold_ratio, }) return records @@ -801,7 +811,11 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: headings = [] seen = set() body_started = False + body_anchor: tuple[int, float] | None = None for page in pages: + font_sizes = sorted(float(block.get("font_size")) for block in page.get("blocks", []) if isinstance(block.get("font_size"), (int, float)) and len(_normalized_compare_text(str(block.get("text") or ""))) >= 20) + middle = len(font_sizes) // 2 + body_font_size = (font_sizes[middle] if len(font_sizes) % 2 else (font_sizes[middle - 1] + font_sizes[middle]) / 2.0) if font_sizes else None for block in page.get("blocks", []): lines = [_clean(value) for value in str(block.get("text", "")).splitlines() if _clean(value)] for line_index, line in enumerate(lines): @@ -810,16 +824,27 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: explicit = re.match(r"^(abstract|references|acknowledg(?:e)?ments?|appendix)$", line, re.IGNORECASE) numbered = re.match(r"^\s*((?:\d+(?:\.\d+)*)|(?:[ivx]+))[.)]?\s+(.{2,120})$", line, re.IGNORECASE) compact_line = _normalized_compare_text(line) + block_bold = bool(block.get("font_bold")) + font_size = float(block.get("font_size") or 0.0) + first_line_alpha = next((char for char in line if char.isalpha()), "") + heading_case = not first_line_alpha or not first_line_alpha.isascii() or first_line_alpha.isupper() known = len(line) <= 60 and not line.endswith((".", ",", ";")) and any( - compact_alias and compact_line.startswith(compact_alias) + compact_alias and heading_case and (compact_line == compact_alias or (block_bold and compact_line.startswith(compact_alias))) for aliases in SECTION_ALIASES.values() for alias in aliases for compact_alias in [_normalized_compare_text(alias)] ) - typographic = body_started and bool(block.get("font_bold")) and _looks_like_typographic_heading(line) + block_y = float((block.get("bbox") or [0, 0, 0, 0])[1]) + after_anchor = body_anchor is not None and (int(page["page"]) > body_anchor[0] or block_y >= body_anchor[1]) + position_ok = after_anchor if body_anchor is not None else body_started + size_ok = body_font_size is None or font_size >= body_font_size * 0.95 + typographic = position_ok and block_bold and size_ok and _looks_like_typographic_heading(line) if numbered: title_candidate = numbered.group(2).strip() first_alpha = next((char for char in title_candidate if char.isalpha()), "") + numbered_style = block.get("source") != "pymupdf" or body_font_size is None or block_bold or font_size >= body_font_size * 1.05 + if not numbered_style: + first_alpha = "" if not title_candidate or not title_candidate[0].isalpha(): first_alpha = "" has_case = bool(first_alpha and first_alpha.lower() != first_alpha.upper()) @@ -831,9 +856,51 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: key = normalized.casefold() if normalized and key not in seen: seen.add(key) - headings.append({"id": f"section_{len(headings) + 1:03d}", "title": normalized, "page": page["page"], "block_id": block["id"], "line_index": line_index}) + headings.append({ + "id": f"section_{len(headings) + 1:03d}", + "title": normalized, + "page": page["page"], + "block_id": block["id"], + "line_index": line_index, + "_candidate_type": "explicit" if explicit else "numbered" if numbered else "known" if known else "typographic", + "_bbox": block.get("bbox"), + "_font_size": font_size, + "_reading_order": int(block.get("reading_order") or 0), + }) + if body_anchor is None and (explicit or numbered or known): + body_anchor = (int(page["page"]), block_y) body_started = body_started or bool(explicit or numbered or known) - return headings[:80] + merged: list[dict[str, Any]] = [] + for heading in headings: + previous = merged[-1] if merged else None + previous_bbox = previous.get("_bbox") if previous else None + current_bbox = heading.get("_bbox") + same_multiline_heading = bool( + previous + and previous.get("_candidate_type") == "typographic" + and heading.get("_candidate_type") == "typographic" + and int(previous.get("page") or 0) == int(heading.get("page") or 0) + and int(heading.get("_reading_order") or 0) == int(previous.get("_reading_order") or 0) + 1 + and previous_bbox + and current_bbox + and float(current_bbox[1]) - float(previous_bbox[3]) <= max(float(previous.get("_font_size") or 0), float(heading.get("_font_size") or 0), 1.0) * 0.8 + and len(f"{previous.get('title', '')} {heading.get('title', '')}") <= 140 + ) + if same_multiline_heading: + previous["title"] = _clean(f"{previous.get('title', '')} {heading.get('title', '')}").rstrip(".:") + previous.setdefault("continuation_block_ids", []).append(str(heading.get("block_id") or "")) + previous["_bbox"] = [ + min(float(previous_bbox[0]), float(current_bbox[0])), + min(float(previous_bbox[1]), float(current_bbox[1])), + max(float(previous_bbox[2]), float(current_bbox[2])), + max(float(previous_bbox[3]), float(current_bbox[3])), + ] + previous["_reading_order"] = int(heading.get("_reading_order") or 0) + continue + merged.append(heading) + for index, heading in enumerate(merged, 1): + heading["id"] = f"section_{index:03d}" + return [{key: value for key, value in heading.items() if not key.startswith("_")} for heading in merged[:80]] def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, Any]: @@ -1033,7 +1100,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st partial_scan = bool(fully_scanned_source and 0 < ocr_count < len(pages)) pdf_type = "scanned" if fully_scanned_source else "mixed" if ocr_count and ocr_count < len(pages) else "scanned" if ocr_count else "born_digital" coverage = _section_coverage(pages, index.get("sections", [])) - indexed_section_blocks = {(int(item.get("page") or 0), str(item.get("block_id") or "")) for item in index.get("sections", [])} + bold_block_ids = {(int(page["page"]), str(block.get("id") or "")) for page in pages for block in page.get("blocks", []) if block.get("font_bold")} warnings = list(details.get("warnings", [])) ocr_blocks = [block for page in pages if page.get("used_ocr") for block in page.get("blocks", []) if isinstance(block, dict)] ocr_confidences = [float(block.get("confidence") or 0.0) for block in ocr_blocks] @@ -1092,7 +1159,8 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "figure_caption_count": len(index.get("figures", [])), "table_caption_count": len(index.get("tables", [])), "section_count": len(index.get("sections", [])), - "typographic_heading_count": sum(bool(block.get("font_bold")) and (int(page["page"]), str(block.get("id") or "")) in indexed_section_blocks for page in pages for block in page.get("blocks", [])), + "typographic_heading_count": sum((int(item.get("page") or 0), str(item.get("block_id") or "")) in bold_block_ids for item in index.get("sections", [])), + "merged_heading_line_count": sum(len(item.get("continuation_block_ids", [])) for item in index.get("sections", [])), "poppler_available": details.get("poppler_available", False), "warnings": warnings, "elapsed_seconds": details.get("elapsed_seconds", 0.0), @@ -1119,7 +1187,12 @@ def parse_paper( pages, loader = _read_non_pdf_pages(source, max_chars) details = {"warnings": [], "ocr_candidate_pages": [], "ocr_pages": [], "poppler_available": False, "elapsed_seconds": 0.0} sections = _heading_candidates(pages) - section_blocks = {(int(item.get("page") or 0), str(item.get("block_id") or "")) for item in sections} + section_blocks = { + (int(item.get("page") or 0), block_id) + for item in sections + for block_id in [str(item.get("block_id") or ""), *(str(value) for value in item.get("continuation_block_ids", []))] + if block_id + } for page in pages: for block in page.get("blocks", []): if (int(page["page"]), str(block.get("id") or "")) in section_blocks: diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 85a26da..5ed70c7 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 14 +DOCUMENT_CACHE_VERSION = 18 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 2027f9b..f1bc16a 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -917,6 +917,7 @@ def prepare_paper_figure_contract( "rotated_pages": parsed["extraction_report"].get("rotated_pages", []), "section_count": parsed["extraction_report"].get("section_count", 0), "typographic_heading_count": parsed["extraction_report"].get("typographic_heading_count", 0), + "merged_heading_line_count": parsed["extraction_report"].get("merged_heading_line_count", 0), "figure_caption_count": parsed["extraction_report"].get("figure_caption_count", 0), "table_caption_count": parsed["extraction_report"].get("table_caption_count", 0), "section_coverage": parsed["extraction_report"].get("section_coverage", {}), diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 16c30db..bea8567 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -15,6 +15,7 @@ score_benchmark_case, validate_benchmark_case, ) +from rfs.evaluation.pdf_extraction_benchmark import run_pdf_extraction_stress_suite ROOT = Path(__file__).resolve().parents[2] @@ -139,10 +140,22 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("max_detected_column_count", result["aggregate"]) self.assertIn("mean_section_count", result["aggregate"]) self.assertIn("typographic_heading_total", result["aggregate"]) + self.assertIn("merged_heading_line_total", result["aggregate"]) self.assertIn("figure_caption_total", result["aggregate"]) self.assertIn("max_ocr_worker_count", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) + def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr(self): + with tempfile.TemporaryDirectory() as tmp: + result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") + + self.assertTrue(result["ok"]) + self.assertEqual(result["aggregate"]["case_count"], 4) + self.assertEqual(result["aggregate"]["passed_case_count"], 4) + self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) + self.assertTrue((Path(tmp) / "mixed_scan_fixture_adapter" / "document_model.json").exists()) + self.assertTrue((Path(tmp) / "pdf_extraction_stress_report.json").exists()) + def test_paper_to_image_score_uses_hard_scientific_gates(self): case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" with tempfile.TemporaryDirectory() as tmp: From c01994cd5d939437f526fbd2724b4fadb6e3947c Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 10:56:29 +0800 Subject: [PATCH 33/74] Repair missing spaces in English OCR text --- README.md | 2 +- benchmarks/README.md | 2 +- pyproject.toml | 3 +- rfs/cli.py | 1 + rfs/evaluation/benchmarking.py | 1 + rfs/evaluation/pdf_extraction_benchmark.py | 7 ++- rfs/paper_to_image/analyzer.py | 60 +++++++++++++++++++++- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 1 + tests/unit/test_benchmarking.py | 1 + tests/unit/test_paper_analyzer.py | 25 ++++++++- 11 files changed, 96 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a228a24..58b7944 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, and rotation metrics so layout assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, and OCR-spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; the OCR extra includes `wordninja` for conservative English spacing recovery such as `2Method` to `2 Method`, while protecting CJK text, acronyms, camel-case model names, and common scientific compounds. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/benchmarks/README.md b/benchmarks/README.md index 4fae871..7611eb9 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -18,7 +18,7 @@ A full-slide copy of the reference image is a hard failure even if pixel similar ### Generated PDF extraction stress suite -`benchmark pdf-suite` creates deterministic native two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs at runtime. It validates section boundaries, cross-column reading order, displayed coordinates, caption recovery, local OCR scheduling, and elapsed time. `--ocr-engine auto` adds a real installed-OCR probe after the deterministic adapter-backed tier. +`benchmark pdf-suite` creates deterministic native two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs at runtime. It validates section boundaries, cross-column reading order, displayed coordinates, caption recovery, local OCR scheduling, English OCR spacing recovery, and elapsed time. `--ocr-engine auto` adds a real installed-OCR probe after the deterministic adapter-backed tier. ## Commands diff --git a/pyproject.toml b/pyproject.toml index c64c4e1..1c7f2a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,8 @@ dependencies = [ ocr = [ "rapidocr-onnxruntime", "paddleocr", - "easyocr" + "easyocr", + "wordninja" ] [project.scripts] diff --git a/rfs/cli.py b/rfs/cli.py index dcf9708..dafdd8c 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -86,6 +86,7 @@ def _doctor() -> dict: "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), }, "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, + "wordninja": {"available": importlib.util.find_spec("wordninja") is not None, "purpose": "conservative English spacing repair for OCR lines"}, "fast_contract_cache": {"available": True, "path": str(Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")))}, } ok = all(item["available"] for item in deps.values()) diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 86bc4f4..d25210b 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -862,6 +862,7 @@ def percentile(values: list[float], fraction: float) -> float: "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), + "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), "provider_call_count": len(provider_calls), "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index 90bdec4..0f74504 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -152,6 +152,7 @@ def _run_case( "section_count": parsed.get("extraction_report", {}).get("section_count"), "figure_caption_count": parsed.get("extraction_report", {}).get("figure_caption_count"), "ocr_pages": parsed.get("extraction_report", {}).get("ocr_pages", []), + "ocr_spacing_repair_count": parsed.get("extraction_report", {}).get("ocr_spacing_repair_count", 0), "assertions": assertions, } write_json(case_dir / "benchmark_result.json", result) @@ -230,8 +231,10 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> root, lambda parsed: [ _check("ocr_completed", 2 in parsed["extraction_report"].get("ocr_pages", []), parsed["extraction_report"].get("ocr_pages", []), "contains page 2"), - _check("method_token", "method" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains method"), - _check("encoder_token", "encoder" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains encoder"), + _check("method_spacing", "2 method" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "contains '2 Method'"), + _check("sentence_spacing", "scanned encoder passes evidence to the scanned decoder and output" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "spaced scientific sentence"), + _check("caption_spacing", "scanned architecture overview with encoder decoder and output" in parsed["pages"][1].get("text", "").casefold(), parsed["pages"][1].get("text", ""), "spaced caption"), + _check("repairs_recorded", parsed["extraction_report"].get("ocr_spacing_repair_count", 0) > 0, parsed["extraction_report"].get("ocr_spacing_repair_count", 0), ">0"), ], ocr_engine=ocr_engine, preview_page=1, diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 288f788..9b687d0 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -25,6 +25,19 @@ "conclusion": ("conclusion", "conclusions", "discussion", "结论", "讨论", "总结", "結論", "考察", "まとめ", "결론", "논의", "요약"), } +OCR_PROTECTED_WORDS = { + "acknowledgements", "architecture", "autoregressive", "bidirectional", "classification", "configuration", + "convolutional", "deterministic", "differentiable", "downstream", "embedding", "embeddings", "evaluation", + "experimental", "experiments", "finetuning", "implementation", "information", "initialization", "introduction", + "methodology", "multiframe", "multimodal", "optimization", "performance", "positional", "pretraining", + "probabilities", "representation", "representations", "retrieval", "scientific", "segmentation", "transformer", + "visualization", +} +OCR_SPLIT_ANCHORS = OCR_PROTECTED_WORDS | { + "and", "decoder", "encoder", "evidence", "for", "from", "generation", "into", "method", "model", "of", + "output", "retriever", "the", "to", "with", +} + def _clean(text: str) -> str: value = unicodedata.normalize("NFKC", str(text or "")).replace("\u00ad", "") @@ -41,6 +54,43 @@ def _clean(text: str) -> str: return value.strip() +def _repair_ocr_spacing(text: str, splitter: Callable[[str], list[str]] | None = None) -> tuple[str, int]: + value = _clean(text) + repairs = 0 + value, count = re.subn(r"\b(figure|fig\.|table)(?=\d)", r"\1 ", value, flags=re.IGNORECASE) + repairs += count + section_terms = "abstract|introduction|background|method|methods|approach|experiments|results|conclusion|discussion|references|appendix" + value, count = re.subn(rf"(? str: + nonlocal repairs + token = match.group(0) + normalized = token.casefold() + if normalized in OCR_PROTECTED_WORDS or token.isupper() or (token[1:] != token[1:].lower() and token[1:] != token[1:].upper()): + return token + pieces = [str(piece) for piece in splitter(token) if str(piece)] + lowered = [piece.casefold() for piece in pieces] + if len(pieces) < 2 or any(len(piece) < 2 for piece in pieces) or not any(piece in OCR_SPLIT_ANCHORS for piece in lowered): + return token + repairs += len(pieces) - 1 + return " ".join(pieces) + + value = re.sub(r"[A-Za-z]{8,160}", split_token, value) + return _clean(value), repairs + + def _normalized_compare_text(text: str) -> str: return "".join(char for char in _clean(text).casefold() if char.isalnum()) @@ -499,6 +549,11 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call pixmap = page.get_pixmap(dpi=dpi, alpha=False) pixmap.save(str(target)) records, used_engine = _ocr_records(target, engine, lang, adapter, rapidocr_threads=rapidocr_threads) + spacing_repairs = 0 + for record in records: + repaired, count = _repair_ocr_spacing(record.get("text", "")) + record["text"] = repaired + spacing_repairs += count if used_engine != "rapidocr": records = _group_ocr_words(records, float(pixmap.width)) else: @@ -526,9 +581,9 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call "confidence": round(float(record.get("confidence") or 0.0), 4), "parent_block": record.get("parent_block"), }) - return blocks, used_engine, None, {"margin_noise_removed": margin_noise_removed} + return blocks, used_engine, None, {"margin_noise_removed": margin_noise_removed, "spacing_repairs": spacing_repairs} except Exception as exc: - return [], engine, str(exc), {"margin_noise_removed": 0} + return [], engine, str(exc), {"margin_noise_removed": 0, "spacing_repairs": 0} def _rapidocr_worker_count(engine: str, adapter: Callable | None, page_count: int) -> int: @@ -1146,6 +1201,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "ocr_page_durations": details.get("ocr_page_durations", []), "ocr_worker_count": int(details.get("ocr_worker_count") or 1), "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), + "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), "mean_ocr_confidence": mean_ocr_confidence, "short_ocr_block_ratio": short_ocr_block_ratio, "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 5ed70c7..24e891b 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 18 +DOCUMENT_CACHE_VERSION = 20 def _cache_root() -> Path: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index f1bc16a..b157be7 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -927,6 +927,7 @@ def prepare_paper_figure_contract( "ocr_completed_count": len(parsed["extraction_report"].get("ocr_pages", [])), "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), + "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), }, "provider": provider_summary, "contract_completion": { diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index bea8567..f470637 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -143,6 +143,7 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("merged_heading_line_total", result["aggregate"]) self.assertIn("figure_caption_total", result["aggregate"]) self.assertIn("max_ocr_worker_count", result["aggregate"]) + self.assertIn("ocr_spacing_repair_total", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr(self): diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index ea1b1d8..a849724 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -5,7 +5,7 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _section_coverage, _token_overlap_agreement, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper from rfs.reference_text_extractor import _positive_ocr_setting @@ -192,6 +192,29 @@ def draw(page): self.assertEqual(parsed["extraction_report"]["replacement_character_rate"], 0.0) self.assertEqual(parsed["extraction_report"]["mojibake_rate"], 0.0) + def test_ocr_spacing_repair_restores_scientific_english_boundaries(self): + mapping = { + "Thescannedencoderpassesevidencetothescanned": ["The", "scanned", "encoder", "passes", "evidence", "to", "the", "scanned"], + "architectureoverviewwithencoder": ["architecture", "overview", "with", "encoder"], + "decoderandoutput": ["decoder", "and", "output"], + } + + repaired, count = _repair_ocr_spacing( + "2Method Thescannedencoderpassesevidencetothescanned decoderandoutput. Figure1:Scanned architectureoverviewwithencoder.", + splitter=lambda token: mapping.get(token, [token]), + ) + + self.assertIn("2 Method", repaired) + self.assertIn("The scanned encoder passes evidence to the scanned decoder and output", repaired) + self.assertIn("Figure 1: Scanned architecture overview with encoder", repaired) + self.assertGreaterEqual(count, 10) + + def test_ocr_spacing_repair_preserves_cjk_acronyms_and_known_compounds(self): + source = "多模态 ImageBind BIDIRECTIONALTRANSFORMER multimodal representation" + repaired, _ = _repair_ocr_spacing(source, splitter=lambda token: ["multi", "modal"]) + + self.assertEqual(repaired, source) + def test_low_text_page_uses_local_ocr_adapter(self): path = self._pdf(lambda _page: None) From 47250465bd575669928b0556e2023a6fc353af49 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 11:47:38 +0800 Subject: [PATCH 34/74] Profile RapidOCR extraction stages --- docs/paper-to-image.md | 11 ++++++ rfs/cli.py | 3 ++ rfs/evaluation/pdf_extraction_benchmark.py | 11 ++++++ rfs/paper_to_image/analyzer.py | 28 ++++++++----- rfs/reference_text_extractor.py | 46 ++++++++++++++++++++-- tests/integration/test_paper_to_image.py | 1 + tests/unit/test_benchmarking.py | 10 +++++ tests/unit/test_paper_analyzer.py | 27 ++++++++++++- 8 files changed, 123 insertions(+), 14 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 8e2c35b..e92dae3 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -22,6 +22,17 @@ rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json ``` +For OCR performance diagnosis, `extraction_report.json` and `benchmark pdf-suite` now record page rendering, text detection, orientation classification, text recognition, and post-processing time separately. RapidOCR defaults are deliberately conservative: detector limit `512`, recognition batch `6`, and up to three page workers. Advanced local experiments can override these without changing repository defaults: + +```powershell +$env:RFS_RAPIDOCR_DET_LIMIT = "512" +$env:RFS_RAPIDOCR_BATCH = "6" +$env:RFS_OCR_WORKERS = "3" +rfs benchmark pdf-suite --out output/benchmarks/pdf_profile --ocr-engine rapidocr --json +``` + +Lower detector limits are not assumed to be faster. On dense two-column scans, recognition usually dominates and smaller detector settings can have unstable latency; accept a change only after text agreement, section recovery, caption recovery, confidence, and wall-clock gates all pass. + `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. diff --git a/rfs/cli.py b/rfs/cli.py index dafdd8c..f02e54f 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -19,6 +19,7 @@ from .presentations_qa import run_presentations_qa from .rebuild_vlm_adapters import build_rebuild_vlm_adapters from .rebuild_eval import evaluate_rebuild_vlm +from .reference_text_extractor import _rapidocr_detector_limit from .utils import env_present, mask_secret from .validator import validate_output from .workflow import make_framework @@ -84,6 +85,8 @@ def _doctor() -> dict: "worker_policy": "up to 3 page workers; 2 intra-op threads for a single page", "worker_override": os.getenv("RFS_OCR_WORKERS"), "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), + "detector_limit_override": os.getenv("RFS_RAPIDOCR_DET_LIMIT"), + "effective_detector_limit": _rapidocr_detector_limit(), }, "paddleocr": {"available": importlib.util.find_spec("paddleocr") is not None}, "wordninja": {"available": importlib.util.find_spec("wordninja") is not None, "purpose": "conservative English spacing repair for OCR lines"}, diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index 0f74504..e9041d4 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -141,6 +141,7 @@ def _run_case( write_json(case_dir / "extraction_report.json", parsed.get("extraction_report", {})) preview = _render_preview(fixture, case_dir / f"preview_page_{preview_page + 1}.png", preview_page) assertions = checks(parsed) + ocr_page_durations = parsed.get("extraction_report", {}).get("ocr_page_durations", []) result = { "case_id": case_id, "ok": all(item.get("passed") for item in assertions), @@ -153,6 +154,11 @@ def _run_case( "figure_caption_count": parsed.get("extraction_report", {}).get("figure_caption_count"), "ocr_pages": parsed.get("extraction_report", {}).get("ocr_pages", []), "ocr_spacing_repair_count": parsed.get("extraction_report", {}).get("ocr_spacing_repair_count", 0), + "ocr_page_durations": ocr_page_durations, + "ocr_stage_seconds": { + name: round(sum(float(item.get(name) or 0.0) for item in ocr_page_durations), 4) + for name in ("render_seconds", "detection_seconds", "classification_seconds", "recognition_seconds", "postprocess_seconds", "inference_seconds") + }, "assertions": assertions, } write_json(case_dir / "benchmark_result.json", result) @@ -241,6 +247,10 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> )) elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] + ocr_stage_totals = { + name: round(sum(float(item.get("ocr_stage_seconds", {}).get(name) or 0.0) for item in results), 4) + for name in ("render_seconds", "detection_seconds", "classification_seconds", "recognition_seconds", "postprocess_seconds", "inference_seconds") + } report = { "summary": "Deterministic multi-layout PDF extraction stress suite completed.", "ok": all(item.get("ok") for item in results), @@ -252,6 +262,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> "mean_elapsed_seconds": round(sum(elapsed) / max(1, len(elapsed)), 4), "p95_elapsed_seconds": _percentile(elapsed, 0.95), "ocr_case_count": sum(bool(item.get("ocr_pages")) for item in results), + "ocr_stage_seconds": ocr_stage_totals, }, "cases": results, } diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 9b687d0..802a216 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -385,10 +385,10 @@ def _pdf_metadata(path: Path) -> dict[str, Any]: return {"page_count": None, "encrypted": None, "metadata": {}, "warning": "pypdf unavailable"} -def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str]: +def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, dict[str, Any]]: if adapter: - return list(adapter(image_path, lang) or []), "adapter" - from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr + return list(adapter(image_path, lang) or []), "adapter", {} + from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr_detailed requested = engine if requested == "auto": @@ -402,12 +402,13 @@ def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | N except Exception: requested = "paddle" if requested == "rapidocr": - return run_rapidocr(image_path, lang, threads=rapidocr_threads), "rapidocr" + records, diagnostics = run_rapidocr_detailed(image_path, lang, threads=rapidocr_threads) + return records, "rapidocr", diagnostics if requested == "easyocr": - return run_easyocr(image_path, lang), "easyocr" + return run_easyocr(image_path, lang), "easyocr", {} if requested == "paddle": - return run_paddle_ocr(image_path, lang), "paddle" - return [], "off" + return run_paddle_ocr(image_path, lang), "paddle", {} + return [], "off", {} def _group_ocr_words(records: list[dict[str, Any]], image_width: float) -> list[dict[str, Any]]: @@ -546,9 +547,12 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call with tempfile.TemporaryDirectory() as temp: target = Path(temp) / f"page_{page_number:03d}.png" dpi = 72 if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 + render_started = time.monotonic() pixmap = page.get_pixmap(dpi=dpi, alpha=False) pixmap.save(str(target)) - records, used_engine = _ocr_records(target, engine, lang, adapter, rapidocr_threads=rapidocr_threads) + render_seconds = time.monotonic() - render_started + records, used_engine, engine_diagnostics = _ocr_records(target, engine, lang, adapter, rapidocr_threads=rapidocr_threads) + postprocess_started = time.monotonic() spacing_repairs = 0 for record in records: repaired, count = _repair_ocr_spacing(record.get("text", "")) @@ -581,7 +585,13 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call "confidence": round(float(record.get("confidence") or 0.0), 4), "parent_block": record.get("parent_block"), }) - return blocks, used_engine, None, {"margin_noise_removed": margin_noise_removed, "spacing_repairs": spacing_repairs} + return blocks, used_engine, None, { + "margin_noise_removed": margin_noise_removed, + "spacing_repairs": spacing_repairs, + "render_seconds": round(render_seconds, 4), + "postprocess_seconds": round(time.monotonic() - postprocess_started, 4), + **engine_diagnostics, + } except Exception as exc: return [], engine, str(exc), {"margin_noise_removed": 0, "spacing_repairs": 0} diff --git a/rfs/reference_text_extractor.py b/rfs/reference_text_extractor.py index 68ccda0..d76a634 100644 --- a/rfs/reference_text_extractor.py +++ b/rfs/reference_text_extractor.py @@ -21,6 +21,12 @@ def _positive_ocr_setting(explicit: int | None, env_name: str, default: int) -> return max(1, int(default)) +def _rapidocr_detector_limit(explicit: int | None = None) -> int: + """Return a bounded detector resize limit suitable for RapidOCR.""" + value = _positive_ocr_setting(explicit, "RFS_RAPIDOCR_DET_LIMIT", 512) + return max(256, min(2048, value)) + + def _round4(value: float) -> float: return round(float(value), 4) @@ -189,25 +195,33 @@ def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: return records -def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = None, batch_size: int | None = None) -> list[dict[str, Any]]: +def run_rapidocr_detailed( + image_path: str | Path, + lang: str, + *, + threads: int | None = None, + batch_size: int | None = None, + detector_limit: int | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: del lang thread_count = _positive_ocr_setting(threads, "RFS_RAPIDOCR_THREADS", 1) recognition_batch = _positive_ocr_setting(batch_size, "RFS_RAPIDOCR_BATCH", 6) + detector_side_limit = _rapidocr_detector_limit(detector_limit) engines = getattr(_RAPIDOCR_LOCAL, "engines", None) if engines is None: engines = {} _RAPIDOCR_LOCAL.engines = engines - engine_key = (thread_count, recognition_batch) + engine_key = (thread_count, recognition_batch, detector_side_limit) if engine_key not in engines: from rapidocr_onnxruntime import RapidOCR # type: ignore engines[engine_key] = RapidOCR( intra_op_num_threads=thread_count, inter_op_num_threads=1, - det_limit_side_len=512, + det_limit_side_len=detector_side_limit, rec_batch_num=recognition_batch, ) - result, _timings = engines[engine_key](str(image_path)) + result, timings = engines[engine_key](str(image_path)) records = [] for item in result or []: if not isinstance(item, (list, tuple)) or len(item) < 3: @@ -216,6 +230,30 @@ def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = Non text = str(item[1] or "").strip() if quad and text: records.append({"text": text, "confidence": float(item[2] or 0.0), "quad": quad}) + timing_values = [float(value or 0.0) for value in (timings or [])] + while len(timing_values) < 3: + timing_values.append(0.0) + diagnostics = { + "detector_limit": detector_side_limit, + "thread_count": thread_count, + "recognition_batch": recognition_batch, + "detection_seconds": round(timing_values[0], 4), + "classification_seconds": round(timing_values[1], 4), + "recognition_seconds": round(timing_values[2], 4), + "inference_seconds": round(sum(timing_values[:3]), 4), + "region_count": len(records), + } + return records, diagnostics + + +def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = None, batch_size: int | None = None, detector_limit: int | None = None) -> list[dict[str, Any]]: + records, _diagnostics = run_rapidocr_detailed( + image_path, + lang, + threads=threads, + batch_size=batch_size, + detector_limit=detector_limit, + ) return records diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 7742f87..bb4ce07 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -134,6 +134,7 @@ def test_doctor_reports_easyocr_model_readiness(self): self.assertIn("en_ready", models) self.assertIn("en_ch_ready", models) self.assertIn("allow_download", models) + self.assertIn("effective_detector_limit", report["pdf_tools"]["rapidocr"]) def test_fast_framework_prompt_writes_contract_without_generation(self): with tempfile.TemporaryDirectory() as temp: diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index f470637..8dd75c4 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -146,6 +146,16 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("ocr_spacing_repair_total", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) + def test_pdf_stress_suite_reports_ocr_stage_timings(self): + with tempfile.TemporaryDirectory() as temp: + result = run_pdf_extraction_stress_suite(Path(temp) / "pdf", ocr_engine="off") + + self.assertIn("ocr_stage_seconds", result["aggregate"]) + self.assertIn("recognition_seconds", result["aggregate"]["ocr_stage_seconds"]) + adapter_case = next(item for item in result["cases"] if item["case_id"] == "mixed_scan_fixture_adapter") + self.assertIn("ocr_page_durations", adapter_case) + self.assertIn("render_seconds", adapter_case["ocr_stage_seconds"]) + def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr(self): with tempfile.TemporaryDirectory() as tmp: result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index a849724..e951ff2 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -7,7 +7,7 @@ from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper -from rfs.reference_text_extractor import _positive_ocr_setting +from rfs.reference_text_extractor import _positive_ocr_setting, _rapidocr_detector_limit, run_rapidocr_detailed class PaperAnalyzerTests(unittest.TestCase): @@ -387,6 +387,31 @@ def test_invalid_rapidocr_environment_setting_falls_back(self): with patch.dict("os.environ", {"RFS_RAPIDOCR_THREADS": "invalid"}, clear=False): self.assertEqual(_positive_ocr_setting(None, "RFS_RAPIDOCR_THREADS", 1), 1) + def test_rapidocr_detector_limit_is_bounded_and_overridable(self): + with patch.dict("os.environ", {"RFS_RAPIDOCR_DET_LIMIT": "384"}, clear=False): + self.assertEqual(_rapidocr_detector_limit(), 384) + self.assertEqual(_rapidocr_detector_limit(128), 256) + self.assertEqual(_rapidocr_detector_limit(4096), 2048) + + def test_rapidocr_detailed_exposes_stage_timings(self): + fake_engine = lambda _path: ([([[0, 0], [10, 0], [10, 10], [0, 10]], "Method", 0.95)], [0.2, 0.0, 0.3]) + local = __import__("rfs.reference_text_extractor", fromlist=["_RAPIDOCR_LOCAL"])._RAPIDOCR_LOCAL + prior = getattr(local, "engines", None) + local.engines = {(1, 6, 384): fake_engine} + try: + records, diagnostics = run_rapidocr_detailed("unused.png", "en", detector_limit=384) + finally: + if prior is None: + delattr(local, "engines") + else: + local.engines = prior + + self.assertEqual(records[0]["text"], "Method") + self.assertEqual(diagnostics["detector_limit"], 384) + self.assertEqual(diagnostics["detection_seconds"], 0.2) + self.assertEqual(diagnostics["recognition_seconds"], 0.3) + self.assertEqual(diagnostics["inference_seconds"], 0.5) + def test_ocr_priority_prefers_semantic_pages_before_coverage_anchors(self): pages = [{"page": index, "text": ""} for index in range(1, 11)] pages[3]["text"] = "Figure 1: Overview of the proposed architecture and system pipeline." From fafc8a088140790f2d17557d34c2ca79ad2dce28 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 11:59:27 +0800 Subject: [PATCH 35/74] Avoid caching incomplete OCR runs --- docs/paper-to-image.md | 2 ++ rfs/evaluation/benchmarking.py | 2 ++ rfs/paper_to_image/analyzer.py | 14 ++++++++++++++ rfs/paper_to_image/document_cache.py | 11 ++++++++--- rfs/paper_to_image/preparation.py | 14 +++++++++++--- tests/unit/test_benchmarking.py | 2 ++ tests/unit/test_paper_analyzer.py | 26 ++++++++++++++++++++++++++ 7 files changed, 65 insertions(+), 6 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index e92dae3..cea243f 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -33,6 +33,8 @@ rfs benchmark pdf-suite --out output/benchmarks/pdf_profile --ocr-engine rapidoc Lower detector limits are not assumed to be faster. On dense two-column scans, recognition usually dominates and smaller detector settings can have unstable latency; accept a change only after text agreement, section recovery, caption recovery, confidence, and wall-clock gates all pass. +Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedule_complete=false` / `ocr_run_complete=false`. Such document models are never written to the global document cache, and scientifically partial scan results are never written to the semantic contract cache; a transient slow or failed OCR run therefore cannot poison later fast runs. + `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index d25210b..321e452 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -859,7 +859,9 @@ def percentile(values: list[float], fraction: float) -> float: "table_caption_total": sum(int(value) for value in extraction_metric("table_caption_count")), "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), + "ocr_attempted_page_total": sum(int(value) for value in extraction_metric("ocr_attempted_count")), "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), + "ocr_incomplete_run_count": sum(not bool(item.get("extraction_quality", {}).get("ocr_run_complete", True)) for item in results), "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 802a216..4182151 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -829,6 +829,12 @@ def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str consume_ocr_result(page_number, result, time.monotonic() - ocr_started) document.close() elapsed = round(time.monotonic() - started, 3) + attempted_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page")] + successful_attempt_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page") and item.get("success")] + schedule_complete = set(prioritized).issubset(attempted_pages) + run_complete = schedule_complete and set(attempted_pages).issubset(successful_attempt_pages) + if not schedule_complete and "OCR schedule was not completed before the extraction deadline" not in warnings: + warnings.append("OCR schedule was not completed before the extraction deadline") return pages, { "metadata": metadata, "ocr_candidate_pages": ocr_candidates, @@ -836,6 +842,10 @@ def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str "ocr_priority": ocr_priority, "ocr_pages": ocr_pages, "ocr_page_durations": ocr_page_durations, + "ocr_attempted_pages": attempted_pages, + "ocr_successful_attempt_pages": successful_attempt_pages, + "ocr_schedule_complete": schedule_complete, + "ocr_run_complete": run_complete, "ocr_worker_count": ocr_worker_count, "warnings": warnings, "poppler_available": bool(poppler_text), @@ -1209,6 +1219,10 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "ocr_priority": details.get("ocr_priority", []), "ocr_pages": details.get("ocr_pages", []), "ocr_page_durations": details.get("ocr_page_durations", []), + "ocr_attempted_pages": details.get("ocr_attempted_pages", []), + "ocr_successful_attempt_pages": details.get("ocr_successful_attempt_pages", []), + "ocr_schedule_complete": bool(details.get("ocr_schedule_complete", True)), + "ocr_run_complete": bool(details.get("ocr_run_complete", True)), "ocr_worker_count": int(details.get("ocr_worker_count") or 1), "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 24e891b..041caf7 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,12 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 20 +DOCUMENT_CACHE_VERSION = 21 + + +def document_model_cacheable(parsed: dict[str, Any]) -> bool: + report = parsed.get("extraction_report", {}) + return bool(report.get("status") != "fail" and report.get("ocr_run_complete", True)) def _cache_root() -> Path: @@ -50,7 +55,7 @@ def read_document_cache( if not path.exists(): return None cached = read_json(path) - if not isinstance(cached, dict) or cached.get("extraction_report", {}).get("status") == "fail": + if not isinstance(cached, dict) or not document_model_cacheable(cached): return None active = Path(source).resolve() cached["source_path"] = str(active) @@ -69,7 +74,7 @@ def write_document_cache( max_chars: int = 90000, max_ocr_pages: int = 6, ) -> Path | None: - if parsed.get("extraction_report", {}).get("status") == "fail": + if not document_model_cacheable(parsed): return None path = document_cache_path(source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, max_chars=max_chars, max_ocr_pages=max_ocr_pages) write_json(path, parsed) diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index b157be7..aa5191a 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -703,12 +703,17 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 26, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 27, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" +def _semantic_cache_safe(parsed: dict[str, Any]) -> bool: + report = parsed.get("extraction_report", {}) + return bool(report.get("scientific_scope_complete", True) and report.get("ocr_run_complete", True)) + + def prepare_paper_figure_contract( paper: str | Path, out: str | Path, @@ -784,7 +789,7 @@ def prepare_paper_figure_contract( remaining = max(10, int(deadline_at - time.monotonic() - 15)) effective_mode = planner_mode if remaining >= 25 else "heuristic" cache_path = _fast_cache_path(parsed, str(planner_model or ""), preferences) - if cache_path.exists() and effective_mode == "vlm": + if cache_path.exists() and effective_mode == "vlm" and _semantic_cache_safe(parsed): plan = read_json(cache_path) planner_metadata = {"requested_mode": "vlm", "mode": "vlm", "model": planner_model, "warning": None, "prompt": "", "cached": True} else: @@ -850,7 +855,7 @@ def prepare_paper_figure_contract( write_json(root / f"{name}.json", plan[name]) planning_validation = validate_plan_grounding(plan, parsed) write_json(root / "planning_validation_report.json", planning_validation) - if fast_mode and planning_validation.get("ok") and planner_metadata.get("mode") in {"vlm", "review_grounded_vlm"}: + if fast_mode and planning_validation.get("ok") and planner_metadata.get("mode") in {"vlm", "review_grounded_vlm"} and _semantic_cache_safe(parsed): write_json(cache_path, plan) overlay = build_overlay_spec(plan) write_json(root / "overlay_spec.json", overlay) @@ -925,6 +930,9 @@ def prepare_paper_figure_contract( "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), "ocr_scheduled_count": len(parsed["extraction_report"].get("ocr_priority_pages", [])), "ocr_completed_count": len(parsed["extraction_report"].get("ocr_pages", [])), + "ocr_attempted_count": len(parsed["extraction_report"].get("ocr_attempted_pages", [])), + "ocr_schedule_complete": parsed["extraction_report"].get("ocr_schedule_complete", True), + "ocr_run_complete": parsed["extraction_report"].get("ocr_run_complete", True), "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 8dd75c4..72d6652 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -137,6 +137,8 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("provider_success_rate", result["aggregate"]) self.assertIn("mean_evidence_page_coverage_ratio", result["aggregate"]) self.assertIn("ocr_scheduled_page_total", result["aggregate"]) + self.assertIn("ocr_attempted_page_total", result["aggregate"]) + self.assertIn("ocr_incomplete_run_count", result["aggregate"]) self.assertIn("max_detected_column_count", result["aggregate"]) self.assertIn("mean_section_count", result["aggregate"]) self.assertIn("typographic_heading_total", result["aggregate"]) diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index e951ff2..6ca4e0d 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -1,4 +1,5 @@ import tempfile +import time import unittest from pathlib import Path from unittest.mock import patch @@ -7,6 +8,8 @@ from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper +from rfs.paper_to_image.document_cache import write_document_cache +from rfs.paper_to_image.preparation import _semantic_cache_safe from rfs.reference_text_extractor import _positive_ocr_setting, _rapidocr_detector_limit, run_rapidocr_detailed @@ -443,6 +446,29 @@ def adapter(image_path, _lang): self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") self.assertEqual(parsed["extraction_report"]["semantic_scope"], "sampled_pages_only") self.assertFalse(parsed["extraction_report"]["scientific_scope_complete"]) + self.assertTrue(parsed["extraction_report"]["ocr_schedule_complete"]) + self.assertTrue(parsed["extraction_report"]["ocr_run_complete"]) + + def test_deadline_limited_ocr_is_not_cached_as_complete(self): + path = self._multipage_pdf([[] for _ in range(3)]) + + def adapter(_image_path, _lang): + return [{ + "text": "Abstract Method architecture evidence recovered by OCR.", + "confidence": 0.97, + "quad": [[20, 20], [560, 20], [560, 70], [20, 70]], + }] + + parsed = parse_paper(path, deadline_at=time.monotonic() - 1, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=3) + report = parsed["extraction_report"] + + self.assertFalse(report["ocr_schedule_complete"]) + self.assertFalse(report["ocr_run_complete"]) + self.assertEqual(report["ocr_attempted_pages"], []) + self.assertTrue(any("schedule was not completed" in warning for warning in report["warnings"])) + with tempfile.TemporaryDirectory() as cache, patch.dict("os.environ", {"RFS_CACHE_DIR": cache}, clear=False): + self.assertIsNone(write_document_cache(path, parsed, ocr_engine="easyocr", ocr_lang="en_ch")) + self.assertFalse(_semantic_cache_safe(parsed)) if __name__ == "__main__": From 2065f5663f911505dd39d2a3f7af036d6891bbbf Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 12:06:38 +0800 Subject: [PATCH 36/74] Enforce OCR deadlines with isolated workers --- docs/paper-to-image.md | 2 + rfs/cli.py | 2 +- rfs/paper_to_image/analyzer.py | 113 +++++++++++++++++++++++++++++- rfs/paper_to_image/ocr_worker.py | 47 +++++++++++++ tests/unit/test_paper_analyzer.py | 56 ++++++++++++++- 5 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 rfs/paper_to_image/ocr_worker.py diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index cea243f..1fecb5e 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -35,6 +35,8 @@ Lower detector limits are not assumed to be faster. On dense two-column scans, r Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedule_complete=false` / `ocr_run_complete=false`. Such document models are never written to the global document cache, and scientifically partial scan results are never written to the semantic contract cache; a transient slow or failed OCR run therefore cannot poison later fast runs. +When a deadline is active and multiple pages need local OCR, pages run in isolated worker processes. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. + `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. diff --git a/rfs/cli.py b/rfs/cli.py index f02e54f..e09b380 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -82,7 +82,7 @@ def _doctor() -> dict: "rapidocr": { "available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, "runtime": "onnxruntime", - "worker_policy": "up to 3 page workers; 2 intra-op threads for a single page", + "worker_policy": "up to 3 page workers; deadline runs use killable worker processes; 2 intra-op threads for a single page", "worker_override": os.getenv("RFS_OCR_WORKERS"), "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), "detector_limit_override": os.getenv("RFS_RAPIDOCR_DET_LIMIT"), diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 4182151..5d6d925 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -6,6 +6,7 @@ import re import shutil import subprocess +import sys import tempfile import time import unicodedata @@ -611,6 +612,109 @@ def _rapidocr_worker_count(engine: str, adapter: Callable | None, page_count: in return max(1, min(int(page_count), configured, int(os.cpu_count() or 1))) +def _deadline_ocr_wave( + path: Path, + page_numbers: list[int], + engine: str, + lang: str, + deadline_at: float, +) -> list[tuple[int, tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]]: + """Run OCR pages in killable child processes and preserve completed results.""" + cutoff = max(time.monotonic(), float(deadline_at) - 20.0) + creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0 + with tempfile.TemporaryDirectory() as temp: + processes: dict[int, dict[str, Any]] = {} + for page_number in page_numbers: + output = Path(temp) / f"page_{page_number:03d}.json" + command = [ + sys.executable, + "-m", + "rfs.paper_to_image.ocr_worker", + "--paper", + str(path), + "--page", + str(page_number), + "--engine", + engine, + "--lang", + lang, + "--threads", + "1", + "--out", + str(output), + ] + started = time.monotonic() + try: + process = subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + creationflags=creation_flags, + ) + processes[page_number] = {"process": process, "output": output, "started": started, "launch_error": None} + except Exception as exc: + processes[page_number] = {"process": None, "output": output, "started": started, "launch_error": str(exc)} + + pending = {page for page, item in processes.items() if item["process"] is not None} + while pending and time.monotonic() < cutoff: + finished = {page for page in pending if processes[page]["process"].poll() is not None} + pending.difference_update(finished) + if pending: + time.sleep(0.02) + + timed_out = set(pending) + for page_number in timed_out: + process = processes[page_number]["process"] + try: + process.terminate() + except Exception: + pass + for page_number in timed_out: + process = processes[page_number]["process"] + try: + process.wait(timeout=1.0) + except Exception: + try: + process.kill() + process.wait(timeout=1.0) + except Exception: + pass + + results = [] + for page_number in page_numbers: + item = processes[page_number] + process = item["process"] + elapsed = time.monotonic() - float(item["started"]) + stderr = "" + if process is not None: + try: + _stdout, stderr = process.communicate(timeout=0.2) + except Exception: + stderr = "" + if item["launch_error"]: + result = ([], engine, f"OCR worker launch failed: {item['launch_error']}", {"worker_process": True}) + elif page_number in timed_out: + result = ([], engine, "OCR exceeded extraction deadline", {"worker_process": True, "timed_out": True}) + elif item["output"].exists(): + try: + payload = json.loads(item["output"].read_text(encoding="utf-8")) + result = ( + list(payload.get("blocks") or []), + str(payload.get("engine") or engine), + str(payload.get("error")) if payload.get("error") else None, + {"worker_process": True, **dict(payload.get("diagnostics") or {})}, + ) + elapsed = float(payload.get("elapsed_seconds") or elapsed) + except Exception as exc: + result = ([], engine, f"OCR worker returned invalid output: {exc}", {"worker_process": True}) + else: + detail = stderr.strip() or f"worker exited with code {getattr(process, 'returncode', None)}" + result = ([], engine, f"OCR worker failed: {detail}", {"worker_process": True}) + results.append((page_number, result, elapsed)) + return results + + def _prioritize_ocr_candidates( pages: list[dict[str, Any]], candidates: list[int], @@ -797,7 +901,14 @@ def deadline_allows_wave() -> bool: return False return True - if ocr_worker_count > 1: + if ocr_worker_count > 1 and deadline_at is not None and ocr_adapter is None: + for start in range(0, len(prioritized), ocr_worker_count): + if not deadline_allows_wave(): + break + batch = prioritized[start:start + ocr_worker_count] + for page_number, result, ocr_elapsed in _deadline_ocr_wave(path, batch, ocr_engine, ocr_lang, deadline_at): + consume_ocr_result(page_number, result, ocr_elapsed) + elif ocr_worker_count > 1: def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]: ocr_started = time.monotonic() try: diff --git a/rfs/paper_to_image/ocr_worker.py b/rfs/paper_to_image/ocr_worker.py new file mode 100644 index 0000000..76618d0 --- /dev/null +++ b/rfs/paper_to_image/ocr_worker.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from ..utils import write_json +from .analyzer import _ocr_page + + +def main() -> int: + parser = argparse.ArgumentParser(description="Isolated, deadline-safe PDF OCR page worker.") + parser.add_argument("--paper", required=True) + parser.add_argument("--page", required=True, type=int) + parser.add_argument("--engine", required=True) + parser.add_argument("--lang", default="en_ch") + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--out", required=True) + args = parser.parse_args() + + import fitz + + started = time.monotonic() + document = fitz.open(str(Path(args.paper).resolve())) + try: + blocks, engine, error, diagnostics = _ocr_page( + document[args.page - 1], + args.page, + args.engine, + args.lang, + None, + rapidocr_threads=max(1, int(args.threads)), + ) + finally: + document.close() + write_json(Path(args.out), { + "blocks": blocks, + "engine": engine, + "error": error, + "diagnostics": diagnostics, + "elapsed_seconds": round(time.monotonic() - started, 4), + }) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 6ca4e0d..def551c 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -1,3 +1,4 @@ +import json import tempfile import time import unittest @@ -6,7 +7,7 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _deadline_ocr_wave, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper from rfs.paper_to_image.document_cache import write_document_cache from rfs.paper_to_image.preparation import _semantic_cache_safe @@ -470,6 +471,59 @@ def adapter(_image_path, _lang): self.assertIsNone(write_document_cache(path, parsed, ocr_engine="easyocr", ocr_lang="en_ch")) self.assertFalse(_semantic_cache_safe(parsed)) + def test_deadline_ocr_wave_preserves_completed_pages_and_terminates_slow_workers(self): + class FakeProcess: + def __init__(self, command, **_kwargs): + self.page = int(command[command.index("--page") + 1]) + self.output = Path(command[command.index("--out") + 1]) + self.returncode = None + self.terminated = False + if self.page == 1: + self.output.write_text(json.dumps({ + "blocks": [{"text": "Method", "bbox": [0, 0, 10, 10], "confidence": 0.9}], + "engine": "rapidocr", + "error": None, + "diagnostics": {"recognition_seconds": 0.1}, + "elapsed_seconds": 0.2, + }), encoding="utf-8") + self.returncode = 0 + + def poll(self): + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = -15 + + def wait(self, timeout=None): + del timeout + return self.returncode + + def kill(self): + self.returncode = -9 + + def communicate(self, timeout=None): + del timeout + return "", "" + + created = [] + + def fake_popen(command, **kwargs): + process = FakeProcess(command, **kwargs) + created.append(process) + return process + + with patch("rfs.paper_to_image.analyzer.subprocess.Popen", side_effect=fake_popen): + results = _deadline_ocr_wave(Path("paper.pdf"), [1, 2], "rapidocr", "en", time.monotonic() + 20.05) + + by_page = {page: result for page, result, _elapsed in results} + self.assertIsNone(by_page[1][2]) + self.assertEqual(by_page[1][0][0]["text"], "Method") + self.assertTrue(by_page[1][3]["worker_process"]) + self.assertEqual(by_page[2][2], "OCR exceeded extraction deadline") + self.assertTrue(by_page[2][3]["timed_out"]) + self.assertTrue(next(item for item in created if item.page == 2).terminated) + if __name__ == "__main__": unittest.main() From cb95ffb6dad93b04659f8ecbbfb3a630d5b57e4f Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 12:10:30 +0800 Subject: [PATCH 37/74] Bound single-page OCR by deadlines --- docs/paper-to-image.md | 2 +- rfs/paper_to_image/analyzer.py | 5 +++-- tests/unit/test_paper_analyzer.py | 8 ++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 1fecb5e..0421b6f 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -35,7 +35,7 @@ Lower detector limits are not assumed to be faster. On dense two-column scans, r Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedule_complete=false` / `ocr_run_complete=false`. Such document models are never written to the global document cache, and scientifically partial scan results are never written to the semantic contract cache; a transient slow or failed OCR run therefore cannot poison later fast runs. -When a deadline is active and multiple pages need local OCR, pages run in isolated worker processes. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. +When a deadline is active, every local OCR page runs in an isolated worker process. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. Single-page RapidOCR retains two intra-op threads; multi-page waves use one thread per worker. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 5d6d925..712bf1e 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -621,6 +621,7 @@ def _deadline_ocr_wave( ) -> list[tuple[int, tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]]: """Run OCR pages in killable child processes and preserve completed results.""" cutoff = max(time.monotonic(), float(deadline_at) - 20.0) + worker_threads = 2 if len(page_numbers) == 1 and engine in {"auto", "rapidocr"} else 1 creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0 with tempfile.TemporaryDirectory() as temp: processes: dict[int, dict[str, Any]] = {} @@ -639,7 +640,7 @@ def _deadline_ocr_wave( "--lang", lang, "--threads", - "1", + str(worker_threads), "--out", str(output), ] @@ -901,7 +902,7 @@ def deadline_allows_wave() -> bool: return False return True - if ocr_worker_count > 1 and deadline_at is not None and ocr_adapter is None: + if deadline_at is not None and ocr_adapter is None: for start in range(0, len(prioritized), ocr_worker_count): if not deadline_allows_wave(): break diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index def551c..716168a 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -475,6 +475,7 @@ def test_deadline_ocr_wave_preserves_completed_pages_and_terminates_slow_workers class FakeProcess: def __init__(self, command, **_kwargs): self.page = int(command[command.index("--page") + 1]) + self.threads = int(command[command.index("--threads") + 1]) self.output = Path(command[command.index("--out") + 1]) self.returncode = None self.terminated = False @@ -524,6 +525,13 @@ def fake_popen(command, **kwargs): self.assertTrue(by_page[2][3]["timed_out"]) self.assertTrue(next(item for item in created if item.page == 2).terminated) + created.clear() + with patch("rfs.paper_to_image.analyzer.subprocess.Popen", side_effect=fake_popen): + single = _deadline_ocr_wave(Path("paper.pdf"), [1], "rapidocr", "en", time.monotonic() + 20.05) + self.assertIsNone(single[0][1][2]) + self.assertEqual(created[0].page, 1) + self.assertEqual(created[0].threads, 2) + if __name__ == "__main__": unittest.main() From d3ad3dc0b9f74b87c307ed4c89299d5644e595bf Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 12:28:27 +0800 Subject: [PATCH 38/74] Recover elided embedding lists from scanned papers --- docs/paper-to-image.md | 2 ++ rfs/paper_to_image/analyzer.py | 4 ++-- rfs/paper_to_image/contract_completion.py | 13 +++++++----- rfs/paper_to_image/document_cache.py | 2 +- tests/unit/test_paper_analyzer.py | 8 +++---- tests/unit/test_paper_contract.py | 26 +++++++++++++++++++++++ 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 0421b6f..b8dbc5b 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -37,6 +37,8 @@ Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedul When a deadline is active, every local OCR page runs in an isolated worker process. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. Single-page RapidOCR retains two intra-op threads; multi-page waves use one thread per worker. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. +For a fully scanned long paper with no native section signals, the six-page fallback schedule covers pages 1-4 first, then a middle page and a page near 85% of the document. This biases scarce OCR time toward the abstract, overview figure, and early method details while retaining mid/late-document coverage. + `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 712bf1e..32a9dc0 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -760,9 +760,9 @@ def _prioritize_ocr_candidates( anchors = [ 1, min(2, page_count), - max(1, round(page_count * 0.25)), + min(3, page_count), + min(4, page_count), max(1, round(page_count * 0.50)), - max(1, round(page_count * 0.75)), max(1, round(page_count * 0.85)), ] for anchor in anchors: diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 778948b..cf3d2cb 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -30,6 +30,9 @@ class ConceptRule: context_pattern: str | None = None +EMBEDDING_SUM_CONTEXT = r"\btoken(?: embeddings?)?\s*,\s*(?:the\s+)?(?:segment|segmentation)(?: embeddings?)?\s*,?\s+and\s+(?:the\s+)?position embeddings?\b" + + CONCEPT_RULES = ( ConceptRule("input_image", "Input Image", "inputs", "input", r"\binput images?\b|\bsplit an image\b|\braw image\b|\btest images?\b|\bimages? are shown\b", ("image", "2d image")), ConceptRule("images", "Images", "inputs", "input modality", r"\b(?:batch of\s+)?\(?images?\s*[,)]", ("training images", "image batch"), True), @@ -98,11 +101,11 @@ class ConceptRule: ConceptRule("output_linear", "Linear", "modules", "output projection", r"\blearned linear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("linear layer", "output projection")), ConceptRule("output_softmax", "Softmax", "modules", "output normalization", r"\blinear transfor(?:mation|\-\s*mation) and softmax function to convert the decoder output\b", ("softmax layer",)), ConceptRule("output_probabilities", "Output Probabilities", "outputs", "output", r"\bpredicted next-token probabilities\b", ("predicted next-token probabilities", "next-token probabilities", "outputs")), - ConceptRule("input_tokens", "Input Sequence", "inputs", "input sequence", r"\brepresent the input\s+sequence\b|\bfor a given token, its input representation\b", ("input tokens", "token sequence"), context_pattern=r"\btoken embeddings?\b.*\b(?:segment|segmentation) embeddings?\b.*\bposition embeddings?\b"), - ConceptRule("token_embeddings", "Token Embeddings", "modules", "embedding", r"\btoken embeddings?\b", (), context_pattern=r"\b(?:segment|segmentation) embeddings?\b.*\bposition embeddings?\b"), - ConceptRule("segment_embeddings", "Segment Embeddings", "modules", "embedding", r"\b(?:segment|segmentation) embeddings?\b", ("segmentation embeddings",), context_pattern=r"\btoken embeddings?\b.*\bposition embeddings?\b"), - ConceptRule("bert_position_embeddings", "Position Embeddings", "modules", "embedding", r"\bposition embeddings?\b", (), context_pattern=r"\btoken embeddings?\b.*\b(?:segment|segmentation) embeddings?\b"), - ConceptRule("input_representation", "Input Representation", "modules", "representation", r"\binput representation\b|\binput embeddings are the sum of\b", ("bert input representation",), context_pattern=r"\btoken embeddings?\b.*\b(?:segment|segmentation) embeddings?\b.*\bposition embeddings?\b"), + ConceptRule("input_tokens", "Input Sequence", "inputs", "input sequence", r"\brepresent the input\s+sequence\b|\bfor a given token, its input representation\b", ("input tokens", "token sequence"), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("token_embeddings", "Token Embeddings", "modules", "embedding", rf"\btoken embeddings?\b|{EMBEDDING_SUM_CONTEXT}", (), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("segment_embeddings", "Segment Embeddings", "modules", "embedding", rf"\b(?:segment|segmentation) embeddings?\b|{EMBEDDING_SUM_CONTEXT}", ("segmentation embeddings",), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("bert_position_embeddings", "Position Embeddings", "modules", "embedding", r"\bposition embeddings?\b", (), context_pattern=EMBEDDING_SUM_CONTEXT), + ConceptRule("input_representation", "Input Representation", "modules", "representation", r"\binput representation\b|\binput embeddings are the sum of\b", ("bert input representation",), context_pattern=EMBEDDING_SUM_CONTEXT), ConceptRule("bidirectional_encoder", "Bidirectional Transformer Encoder", "modules", "encoder", r"\bmulti-layer bidirectional transformer en-?\s*coder\b|\bbidirectional transformer encoder\b", ("bert", "bert encoder")), ConceptRule("masked_lm", "Masked LM", "modules", "training objective", r"\bmasked (?:language model(?:ing)?|lm)\b", ("masked language model", "masked language modeling", "mlm objective"), context_pattern=r"\bnext sentence prediction\b|\bNSP\b"), ConceptRule("next_sentence", "Next Sentence Prediction", "modules", "training objective", r"\bnext sentence prediction\b|\bNSP\b", ("nsp",), context_pattern=r"\bmasked (?:language model(?:ing)?|lm)\b"), diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 041caf7..2ebb25f 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 21 +DOCUMENT_CACHE_VERSION = 22 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 716168a..187f25d 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -378,7 +378,7 @@ def test_ocr_priority_spreads_fully_scanned_long_document(self): selected, details = _prioritize_ocr_candidates(pages, list(range(1, 13)), 6) - self.assertEqual(selected, [1, 2, 3, 6, 9, 10]) + self.assertEqual(selected, [1, 2, 3, 4, 6, 10]) self.assertEqual([item["rank"] for item in details], [1, 2, 3, 4, 5, 6]) def test_rapidocr_worker_count_is_bounded_and_adapter_safe(self): @@ -440,9 +440,9 @@ def adapter(image_path, _lang): parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=6) - self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 6, 9, 10]) - self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 6, 9, 10]) - self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 6, 9, 10]) + self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 4, 6, 10]) + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 4, 6, 10]) + self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 4, 6, 10]) self.assertEqual(parsed["extraction_report"]["status"], "warning") self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") self.assertEqual(parsed["extraction_report"]["semantic_scope"], "sampled_pages_only") diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 764921d..34fc99d 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -596,6 +596,32 @@ def test_generic_completion_recovers_embedding_pretraining_and_finetuning(self): self.assertIn((ids["Token Embeddings"], ids["Input Representation"]), pairs) self.assertIn((ids["Bidirectional Transformer Encoder"], ids["Fine-tuning"]), pairs) + def test_generic_completion_recovers_elided_embedding_list_across_ocr_blocks(self): + caption = "Figure 1: Overall pre-training and fine-tuning procedures." + parsed = { + "page_count": 16, + "document_index": {"figures": [{"page": 3, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 3, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 0.99}, + {"id": "E0002", "page": 4, "kind": "paragraph", "text": "For a given token, its input representation is", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0003", "page": 4, "kind": "paragraph", "text": "constructed by summing the corresponding token,", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0004", "page": 4, "kind": "paragraph", "text": "segment, and position embeddings.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0005", "page": 4, "kind": "paragraph", "text": "The architecture is a multi-layer bidirectional Transformer encoder.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0006", "page": 4, "kind": "paragraph", "text": "We pre-train with Masked LM and Next Sentence Prediction before fine-tuning on downstream tasks.", "section_hint": "Method", "confidence": 0.98}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Sequence", "Token Embeddings", "Segment Embeddings", "Position Embeddings", "Input Representation", "Bidirectional Transformer Encoder"): + self.assertIn(label, ids) + self.assertIn((ids["Token Embeddings"], ids["Input Representation"]), pairs) + self.assertIn((ids["Segment Embeddings"], ids["Input Representation"]), pairs) + self.assertIn((ids["Position Embeddings"], ids["Input Representation"]), pairs) + def test_generic_completion_recovers_retrieval_conditioned_generation(self): caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." result_caption = "Figure 2: Posterior for each generated token with five retrieved documents." From b4a00e90c804532c6f38db668964f0bab9182e02 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 13:00:12 +0800 Subject: [PATCH 39/74] Ground architecture context by local evidence --- rfs/paper_to_image/contract_completion.py | 63 ++++++++++++++++++++--- tests/unit/test_paper_contract.py | 52 +++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index cf3d2cb..86e0f08 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -52,8 +52,8 @@ class ConceptRule: ConceptRule("backbone", "Backbone", "modules", "feature extractor", r"\bbackbones?\b", ("backbone architecture",), True), ConceptRule("image_encoder", "Image Encoder", "modules", "encoder", r"\bimage encoder\b"), ConceptRule("text_encoder", "Text Encoder", "modules", "encoder", r"\btext encoder\b"), - ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b", (), True), - ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b", (), True), + ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b|^encoder$", (), True), + ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b|^decoder$", (), True), ConceptRule("position_embedding", "Position Embedding", "modules", "conditioning", r"\bposition embeddings?\b", ("positional embedding",), True), ConceptRule("positional_encoding", "Positional Encoding", "modules", "conditioning", r"\bposition(?:al)? encod(?:ing|ings)\b"), ConceptRule("image_patches", "Image Patches", "modules", "intermediate", r"\bimage patches?\b|\bfixed-size patches\b", ("patches",), True), @@ -87,7 +87,7 @@ class ConceptRule: ConceptRule("classification", "Classification", "outputs", "output head", r"\bclassification head\b|\bpredicts? the class label\b", ("class label",)), ConceptRule("box_regression", "Bounding-box Regression", "outputs", "output head", r"\bbounding[ -]?box regression\b|\bbox regression\b", ("bounding box", "box branch")), ConceptRule("mask_branch", "Mask Branch", "outputs", "output head", r"\bmask branch\b"), - ConceptRule("refined_output", "Refined Output", "outputs", "output", r"\brefined output\b|\bquality of the output improves\b"), + ConceptRule("refined_output", "Refined Output", "outputs", "output", r"\brefined output\b|\brefines? (?:the )?(?:previously generated )?output\b|\bquality of the output improves\b"), ConceptRule("emergent_alignment", "Emergent Cross-modal Alignment", "outputs", "output", r"\bemergent (?:cross-modal )?alignments?\b|\bbinding (?:agent|property)\b", ("emergent alignment", "binding agent", "binding property")), ConceptRule("valid_mask", "Valid Segmentation Mask", "outputs", "output", r"\bvalid masks?\b|\bobject masks?\b", ("valid masks", "valid mask"), True), ConceptRule("zero_shot_classifier", "Zero-Shot Classifier", "outputs", "inference output", r"\bzero[ -]?shot (?:linear )?classifier\b|\bzero[ -]?shot prediction\b", ("zero-shot linear classifier", "zero-shot prediction")), @@ -249,7 +249,28 @@ def _evidence_for_figure(parsed: dict[str, Any], figure: dict[str, Any]) -> dict def _relevant_evidence(parsed: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: candidates = _overview_candidates(parsed) - selected = [item for item in (_evidence_for_figure(parsed, figure) for figure in candidates) if item] + caption_evidence = [item for item in (_evidence_for_figure(parsed, figure) for figure in candidates) if item] + selected_pages = {int(item.get("page") or 0) for item in caption_evidence if item.get("page")} + selected = list(caption_evidence) + for item in parsed.get("evidence", []): + if int(item.get("page") or 0) not in selected_pages or item in selected: + continue + section = str(item.get("section_hint") or "").casefold() + if any(term in section for term in ("reference", "acknowledg")): + continue + item_bbox = item.get("bbox") if isinstance(item.get("bbox"), list) and len(item.get("bbox")) == 4 else None + captions = [value for value in caption_evidence if value.get("page") == item.get("page")] + near_caption = False + for caption in captions: + caption_bbox = caption.get("bbox") if isinstance(caption.get("bbox"), list) and len(caption.get("bbox")) == 4 else None + if item_bbox is None or caption_bbox is None: + near_caption = True + break + if float(item_bbox[1]) <= float(caption_bbox[3]) + 72.0 and float(item_bbox[3]) >= float(caption_bbox[1]) - 540.0: + near_caption = True + break + if near_caption: + selected.append(item) selected_ids = {item["id"] for item in selected} page_count = max(1, int(parsed.get("page_count") or 1)) early_page_limit = max(8, int(page_count * 0.45)) @@ -294,9 +315,12 @@ def _find_matches(rule: ConceptRule, selected: list[dict[str, Any]], relevant: l if rule.requires_overview: return [] secondary = [item for item in relevant if pattern.search(str(item.get("text") or ""))] + secondary_window = _window_matches(pattern, relevant) + if rule.context_pattern and secondary_window: + return secondary_window if secondary: return secondary[:2] - return _window_matches(pattern, relevant) + return secondary_window def _find_existing(spec: dict[str, Any], rule: ConceptRule) -> tuple[str, dict[str, Any]] | None: @@ -412,10 +436,11 @@ def _heuristic_rule_scope( relevant: list[dict[str, Any]], declared_rule_keys: set[str], ) -> tuple[set[str], dict[str, list[dict[str, Any]]]]: + scoped_relevant = relevant matches = { rule.key: found for rule in CONCEPT_RULES - if (found := _find_matches(rule, selected, relevant)) + if (found := _find_matches(rule, selected, scoped_relevant)) } candidates = set(matches) if not candidates: @@ -426,8 +451,32 @@ def _heuristic_rule_scope( if rule.key in candidates and _find_matches(rule, selected, []) } adjacency = {key: set() for key in candidates} + + def locally_connected(left: str, right: str) -> bool: + left_items, right_items = matches.get(left, []), matches.get(right, []) + if not left_items or not right_items: + return True + for left_item in left_items: + for right_item in right_items: + left_page, right_page = int(left_item.get("page") or 0), int(right_item.get("page") or 0) + if not left_page or not right_page: + return True + page_gap = abs(left_page - right_page) + if 0 < page_gap <= 3: + return True + if page_gap != 0: + continue + left_bbox, right_bbox = left_item.get("bbox"), right_item.get("bbox") + if not (isinstance(left_bbox, list) and len(left_bbox) == 4 and isinstance(right_bbox, list) and len(right_bbox) == 4): + return True + left_center = (float(left_bbox[1]) + float(left_bbox[3])) / 2.0 + right_center = (float(right_bbox[1]) + float(right_bbox[3])) / 2.0 + if abs(left_center - right_center) <= 360.0: + return True + return False + for source_key, target_key, _ in RELATION_RULES: - if source_key in candidates and target_key in candidates: + if source_key in candidates and target_key in candidates and locally_connected(source_key, target_key): adjacency[source_key].add(target_key) adjacency[target_key].add(source_key) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 34fc99d..12cfc7a 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -568,6 +568,23 @@ def test_generic_completion_recovers_encoder_decoder_from_method_lines(self): self.assertIn((ids["Encoder Stack"], ids["Decoder Stack"]), pairs) self.assertIn((ids["Linear"], ids["Softmax"]), pairs) + def test_generic_completion_treats_refinement_action_as_refined_output(self): + caption = "Figure 1: Given an input, the system starts by generating an output, gets feedback, and then refines the previously generated output." + parsed = { + "page_count": 8, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn("Refine", ids) + self.assertIn("Refined Output", ids) + self.assertIn((ids["Refine"], ids["Refined Output"]), pairs) + def test_generic_completion_recovers_embedding_pretraining_and_finetuning(self): captions = [ "Figure 1: Overall pre-training and fine-tuning procedures. The same pre-trained model parameters are used to initialize models for different down-stream tasks.", @@ -622,6 +639,41 @@ def test_generic_completion_recovers_elided_embedding_list_across_ocr_blocks(sel self.assertIn((ids["Segment Embeddings"], ids["Input Representation"]), pairs) self.assertIn((ids["Position Embeddings"], ids["Input Representation"]), pairs) + def test_generic_completion_uses_selected_architecture_page_context_and_diagram_labels(self): + captions = [ + "Figure 1: Model overview directly predicts a set by combining a CNN with a transformer and bipartite matching.", + "Figure 10: Architecture of the proposed transformer's subsystem. See the detailed description on this page.", + ] + parsed = { + "page_count": 24, + "document_index": {"figures": [{"page": 2, "caption": captions[0]}, {"page": 20, "caption": captions[1]}]}, + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "bbox": [40, 300, 500, 320], "text": captions[0], "section_hint": "Figure Captions", "confidence": 0.98}, + {"id": "E0002", "page": 2, "kind": "paragraph", "bbox": [40, 220, 500, 240], "text": "The input image is processed for direct prediction with bipartite matching.", "section_hint": "Introduction", "confidence": 0.98}, + {"id": "E0003", "page": 20, "kind": "caption", "bbox": [40, 600, 500, 620], "text": captions[1], "section_hint": "Figure Captions", "confidence": 0.98}, + {"id": "E0004", "page": 20, "kind": "paragraph", "bbox": [40, 100, 500, 120], "text": "Image features from the CNN backbone are passed through the transformer encoder.", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0005", "page": 20, "kind": "paragraph", "bbox": [40, 150, 500, 170], "text": "The decoder receives object queries and encoder memory.", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0006", "page": 20, "kind": "paragraph", "bbox": [340, 280, 380, 300], "text": "FFN", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0007", "page": 20, "kind": "paragraph", "bbox": [140, 390, 190, 410], "text": "Encoder", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0008", "page": 20, "kind": "paragraph", "bbox": [310, 310, 360, 330], "text": "Decoder", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0009", "page": 20, "kind": "paragraph", "bbox": [40, 180, 500, 200], "text": "The final class labels and bounding boxes are predicted in parallel.", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0010", "page": 20, "kind": "paragraph", "bbox": [40, 710, 500, 730], "text": "For example, a separate text encoder uses class descriptions and contrastive learning to create text embeddings.", "section_hint": "Related Work", "confidence": 0.98}, + ], + } + plan = {"paper_summary": {"unknowns": []}, "figure_specification": {"modules": [], "inputs": [], "outputs": [], "relations": [], "innovations": [], "must_show": [], "terminology": {}}} + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Image", "CNN Backbone", "Transformer Encoder", "Object Queries", "Transformer Decoder", "Feed Forward Network", "Class Predictions", "Bounding Box Predictions", "Bipartite Matching"): + self.assertIn(label, ids) + self.assertIn((ids["CNN Backbone"], ids["Transformer Encoder"]), pairs) + self.assertIn((ids["Object Queries"], ids["Transformer Decoder"]), pairs) + self.assertIn((ids["Feed Forward Network"], ids["Class Predictions"]), pairs) + self.assertNotIn("Text Encoder", ids) + self.assertNotIn("Contrastive Learning", ids) + def test_generic_completion_recovers_retrieval_conditioned_generation(self): caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." result_caption = "Figure 2: Posterior for each generated token with five retrieved documents." From 45ab79c7fafa37a5fb10483bc9816ab5b4834972 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 13:21:39 +0800 Subject: [PATCH 40/74] Recover high-value scan evidence within deadlines --- docs/paper-to-image.md | 6 +++-- rfs/cli.py | 2 +- rfs/paper_to_image/analyzer.py | 29 +++++++++++++------- rfs/paper_to_image/contract_completion.py | 12 +++++++++ rfs/paper_to_image/document_cache.py | 2 +- tests/unit/test_paper_analyzer.py | 33 ++++++++++++++++++++--- tests/unit/test_paper_contract.py | 4 +-- 7 files changed, 69 insertions(+), 19 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index b8dbc5b..e694a00 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -22,7 +22,7 @@ rfs benchmark fast-suite --root benchmarks --out output/benchmarks/unseen --case rfs benchmark fast-suite --root benchmarks --out output/benchmarks/nlp --case-id 109_transformer_encoder_decoder --case-id 110_bert_pretrain_finetune --case-id 111_rag_retrieval_generation --planner-mode vlm --json ``` -For OCR performance diagnosis, `extraction_report.json` and `benchmark pdf-suite` now record page rendering, text detection, orientation classification, text recognition, and post-processing time separately. RapidOCR defaults are deliberately conservative: detector limit `512`, recognition batch `6`, and up to three page workers. Advanced local experiments can override these without changing repository defaults: +For OCR performance diagnosis, `extraction_report.json` and `benchmark pdf-suite` now record page rendering, text detection, orientation classification, text recognition, and post-processing time separately. RapidOCR defaults are deliberately conservative: detector limit `512`, recognition batch `6`, and an adaptive page-worker cap (4 on 8+ logical CPUs, 2 on 4-7, 1 below 4). Advanced local experiments can override these without changing repository defaults: ```powershell $env:RFS_RAPIDOCR_DET_LIMIT = "512" @@ -37,7 +37,9 @@ Deadline-limited or failed OCR schedules are explicitly marked with `ocr_schedul When a deadline is active, every local OCR page runs in an isolated worker process. Completed pages are preserved, while workers still running at the reserved cutoff are terminated and recorded with `timed_out=true`. Single-page RapidOCR retains two intra-op threads; multi-page waves use one thread per worker. This prevents a slow ONNX recognition call from holding the fast workflow beyond its parsing budget or leaving background OCR processes behind. -For a fully scanned long paper with no native section signals, the six-page fallback schedule covers pages 1-4 first, then a middle page and a page near 85% of the document. This biases scarce OCR time toward the abstract, overview figure, and early method details while retaining mid/late-document coverage. +For a fully scanned long paper with no native section signals, the six-page fallback schedule covers pages 1-4 first, then a page near 85% of the document and finally a middle page. Under a deadline, the first wave uses adaptive page concurrency; the two coverage-rescue pages run one at a time with two OCR threads whenever at least 45 seconds remain. This biases scarce OCR time toward the abstract, overview figure, early method details, and likely appendix/conclusion architecture evidence without weakening the hard cutoff. + +If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. The deterministic compiler can recover relations across adjacent PDF blocks, canonicalize evidence-backed VLM aliases, repair missing relation evidence, and ground or downgrade unsupported scalar claims. It also normalizes string entities plus `source_id`/`target_id`/`relation_type` variants, resolves a missing endpoint only when the relation label exactly identifies one declared entity, grounds short uppercase acronyms through exact token matches, and removes unresolved relations into explicit uncertainties. These repairs are paper-name agnostic and are covered by the NLP suite. diff --git a/rfs/cli.py b/rfs/cli.py index e09b380..0670549 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -82,7 +82,7 @@ def _doctor() -> dict: "rapidocr": { "available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, "runtime": "onnxruntime", - "worker_policy": "up to 3 page workers; deadline runs use killable worker processes; 2 intra-op threads for a single page", + "worker_policy": "up to 4 page workers on 8+ CPUs, 2 on 4-7 CPUs, 1 below 4; deadline runs use killable worker processes; 2 intra-op threads for a single page", "worker_override": os.getenv("RFS_OCR_WORKERS"), "thread_override": os.getenv("RFS_RAPIDOCR_THREADS"), "detector_limit_override": os.getenv("RFS_RAPIDOCR_DET_LIMIT"), diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 32a9dc0..ee3df8b 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -605,11 +605,13 @@ def _rapidocr_worker_count(engine: str, adapter: Callable | None, page_count: in import rapidocr_onnxruntime # noqa: F401 except Exception: return 1 + cpu_count = int(os.cpu_count() or 1) + default_workers = 4 if cpu_count >= 8 else 2 if cpu_count >= 4 else 1 try: - configured = int(os.getenv("RFS_OCR_WORKERS") or 3) + configured = int(os.getenv("RFS_OCR_WORKERS") or default_workers) except ValueError: - configured = 3 - return max(1, min(int(page_count), configured, int(os.cpu_count() or 1))) + configured = default_workers + return max(1, min(int(page_count), configured, cpu_count)) def _deadline_ocr_wave( @@ -762,8 +764,8 @@ def _prioritize_ocr_candidates( min(2, page_count), min(3, page_count), min(4, page_count), - max(1, round(page_count * 0.50)), max(1, round(page_count * 0.85)), + max(1, round(page_count * 0.50)), ] for anchor in anchors: if len(selected) >= max_pages: @@ -892,9 +894,14 @@ def consume_ocr_result(page_number: int, result: tuple[list[dict[str, Any]], str }) ocr_pages.append(page_number) - def deadline_allows_wave() -> bool: + def deadline_allows_wave(rescue_page: bool = False) -> bool: if deadline_at is not None: remaining = deadline_at - time.monotonic() + if rescue_page: + if remaining < 45.0: + warnings.append("OCR stopped to preserve deadline validation budget") + return False + return True completed_times = [float(item["elapsed_seconds"]) for item in ocr_page_durations if isinstance(item.get("elapsed_seconds"), (int, float))] estimated = (max(completed_times) * 1.25) if completed_times else 45.0 if remaining < estimated + 20: @@ -903,12 +910,16 @@ def deadline_allows_wave() -> bool: return True if deadline_at is not None and ocr_adapter is None: - for start in range(0, len(prioritized), ocr_worker_count): - if not deadline_allows_wave(): + start = 0 + while start < len(prioritized): + rescue_page = start > 0 + if not deadline_allows_wave(rescue_page=rescue_page): break - batch = prioritized[start:start + ocr_worker_count] + wave_size = 1 if rescue_page else ocr_worker_count + batch = prioritized[start:start + wave_size] for page_number, result, ocr_elapsed in _deadline_ocr_wave(path, batch, ocr_engine, ocr_lang, deadline_at): consume_ocr_result(page_number, result, ocr_elapsed) + start += wave_size elif ocr_worker_count > 1: def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str | None, dict[str, Any]], float]: ocr_started = time.monotonic() @@ -1296,7 +1307,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st short_ocr_block_ratio = round(len(short_ocr_blocks) / max(1, len(ocr_blocks)), 4) if ocr_blocks else None sampled_scan_ready = bool( partial_scan - and len(readable) >= min(6, len(pages)) + and len(readable) >= min(3, len(pages)) and coverage.get("abstract") and coverage.get("method") and mean_ocr_confidence is not None diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 86e0f08..5a21495 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -436,6 +436,7 @@ def _heuristic_rule_scope( relevant: list[dict[str, Any]], declared_rule_keys: set[str], ) -> tuple[set[str], dict[str, list[dict[str, Any]]]]: + selected_ids = {str(item.get("id")) for item in selected if item.get("id")} scoped_relevant = relevant matches = { rule.key: found @@ -473,6 +474,17 @@ def locally_connected(left: str, right: str) -> bool: right_center = (float(right_bbox[1]) + float(right_bbox[3])) / 2.0 if abs(left_center - right_center) <= 360.0: return True + left_selected = any(str(item.get("id")) in selected_ids for item in left_items) + right_selected = any(str(item.get("id")) in selected_ids for item in right_items) + + def method_grounded(items: list[dict[str, Any]]) -> bool: + blocked = ("introduction", "background", "related work", "reference", "acknowledg", "experiment", "result") + return any(not any(term in str(item.get("section_hint") or "").casefold() for term in blocked) for item in items) + + if left_selected and method_grounded(right_items): + return True + if right_selected and method_grounded(left_items): + return True return False for source_key, target_key, _ in RELATION_RULES: diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 2ebb25f..5d3ceee 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 22 +DOCUMENT_CACHE_VERSION = 23 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 187f25d..d40ab0b 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -378,7 +378,7 @@ def test_ocr_priority_spreads_fully_scanned_long_document(self): selected, details = _prioritize_ocr_candidates(pages, list(range(1, 13)), 6) - self.assertEqual(selected, [1, 2, 3, 4, 6, 10]) + self.assertEqual(selected, [1, 2, 3, 4, 10, 6]) self.assertEqual([item["rank"] for item in details], [1, 2, 3, 4, 5, 6]) def test_rapidocr_worker_count_is_bounded_and_adapter_safe(self): @@ -387,6 +387,11 @@ def test_rapidocr_worker_count_is_bounded_and_adapter_safe(self): self.assertEqual(_rapidocr_worker_count("rapidocr", lambda *_args: [], 6), 1) self.assertEqual(_rapidocr_worker_count("easyocr", None, 6), 1) + with patch.dict("os.environ", {"RFS_OCR_WORKERS": ""}, clear=False), patch("rfs.paper_to_image.analyzer.os.cpu_count", return_value=12): + self.assertEqual(_rapidocr_worker_count("rapidocr", None, 6), 4) + with patch.dict("os.environ", {"RFS_OCR_WORKERS": ""}, clear=False), patch("rfs.paper_to_image.analyzer.os.cpu_count", return_value=4): + self.assertEqual(_rapidocr_worker_count("rapidocr", None, 6), 2) + def test_invalid_rapidocr_environment_setting_falls_back(self): with patch.dict("os.environ", {"RFS_RAPIDOCR_THREADS": "invalid"}, clear=False): self.assertEqual(_positive_ocr_setting(None, "RFS_RAPIDOCR_THREADS", 1), 1) @@ -440,9 +445,9 @@ def adapter(image_path, _lang): parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=6) - self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 4, 6, 10]) - self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 4, 6, 10]) - self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 4, 6, 10]) + self.assertEqual(parsed["extraction_report"]["ocr_priority_pages"], [1, 2, 3, 4, 10, 6]) + self.assertEqual(parsed["extraction_report"]["ocr_pages"], [1, 2, 3, 4, 10, 6]) + self.assertEqual([item["page"] for item in parsed["extraction_report"]["ocr_priority"]], [1, 2, 3, 4, 10, 6]) self.assertEqual(parsed["extraction_report"]["status"], "warning") self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") self.assertEqual(parsed["extraction_report"]["semantic_scope"], "sampled_pages_only") @@ -450,6 +455,26 @@ def adapter(image_path, _lang): self.assertTrue(parsed["extraction_report"]["ocr_schedule_complete"]) self.assertTrue(parsed["extraction_report"]["ocr_run_complete"]) + def test_three_high_confidence_scan_pages_are_usable_partial_evidence(self): + path = self._multipage_pdf([[] for _ in range(12)]) + + def adapter(image_path, _lang): + page_number = int(Path(image_path).stem.rsplit("_", 1)[-1]) + return [{ + "text": f"Page {page_number} Abstract Method architecture evidence with enough reliable text for a partial engineering contract.", + "confidence": 0.97, + "quad": [[20, 20], [560, 20], [560, 70], [20, 70]], + }] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter, max_ocr_pages=3) + report = parsed["extraction_report"] + + self.assertEqual(report["ocr_pages"], [1, 2, 3]) + self.assertTrue(report["sampled_scan_ready"]) + self.assertEqual(report["status"], "warning") + self.assertEqual(report["semantic_scope"], "sampled_pages_only") + self.assertFalse(report["scientific_scope_complete"]) + def test_deadline_limited_ocr_is_not_cached_as_complete(self): path = self._multipage_pdf([[] for _ in range(3)]) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 12cfc7a..4cfe67f 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -641,7 +641,7 @@ def test_generic_completion_recovers_elided_embedding_list_across_ocr_blocks(sel def test_generic_completion_uses_selected_architecture_page_context_and_diagram_labels(self): captions = [ - "Figure 1: Model overview directly predicts a set by combining a CNN with a transformer and bipartite matching.", + "Figure 1: Direct set prediction combines a CNN with a transformer and bipartite matching.", "Figure 10: Architecture of the proposed transformer's subsystem. See the detailed description on this page.", ] parsed = { @@ -649,7 +649,7 @@ def test_generic_completion_uses_selected_architecture_page_context_and_diagram_ "document_index": {"figures": [{"page": 2, "caption": captions[0]}, {"page": 20, "caption": captions[1]}]}, "evidence": [ {"id": "E0001", "page": 2, "kind": "caption", "bbox": [40, 300, 500, 320], "text": captions[0], "section_hint": "Figure Captions", "confidence": 0.98}, - {"id": "E0002", "page": 2, "kind": "paragraph", "bbox": [40, 220, 500, 240], "text": "The input image is processed for direct prediction with bipartite matching.", "section_hint": "Introduction", "confidence": 0.98}, + {"id": "E0002", "page": 4, "kind": "paragraph", "bbox": [40, 220, 500, 240], "text": "The input image is processed for direct prediction with bipartite matching.", "section_hint": "Object Detection Method", "confidence": 0.98}, {"id": "E0003", "page": 20, "kind": "caption", "bbox": [40, 600, 500, 620], "text": captions[1], "section_hint": "Figure Captions", "confidence": 0.98}, {"id": "E0004", "page": 20, "kind": "paragraph", "bbox": [40, 100, 500, 120], "text": "Image features from the CNN backbone are passed through the transformer encoder.", "section_hint": "Detailed Architecture", "confidence": 0.98}, {"id": "E0005", "page": 20, "kind": "paragraph", "bbox": [40, 150, 500, 170], "text": "The decoder receives object queries and encoder memory.", "section_hint": "Detailed Architecture", "confidence": 0.98}, From 90325b58779561b16f440f7400d8095d1408ce22 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 13:40:28 +0800 Subject: [PATCH 41/74] Add reproducible scanned paper benchmarks --- benchmarks/README.md | 17 ++++++++++ rfs/cli.py | 5 +-- rfs/evaluation/benchmarking.py | 16 ++++++++- rfs/evaluation/scanned_pdf.py | 40 +++++++++++++++++++++++ rfs/paper_to_image/contract_completion.py | 2 +- tests/integration/test_paper_to_image.py | 5 +++ tests/unit/test_benchmarking.py | 27 +++++++++++++++ tests/unit/test_paper_contract.py | 2 +- 8 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 rfs/evaluation/scanned_pdf.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 7611eb9..9433360 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -43,6 +43,23 @@ Real-paper cases commit `source.json` and human-authored Ground Truth, while `be The committed unseen-generalization set spans vision, multimodal learning, and NLP: ViT, Mask R-CNN, Self-Refine, ImageBind, SAM, DETR, CLIP, NeRF, Transformer, BERT, and retrieval-augmented generation. The NLP cases specifically exercise short overview captions, method-text recovery, embedding summation, pre-training/fine-tuning separation, and retrieval-conditioned generation. Local image-only scan regressions remain under ignored `tmp/pdfs/` and `output/pdf/`; do not commit publisher PDFs or generated OCR artifacts. +Any locally available PDF benchmark can also be tested as an image-only degradation without committing a publisher scan: + +```powershell +rfs benchmark fast-suite ` + --root benchmarks ` + --out output/benchmarks/scanned ` + --case-id 106_detr_set_prediction ` + --case-id 110_bert_pretrain_finetune ` + --rasterize-dpi 144 ` + --ocr-engine rapidocr ` + --planner-mode heuristic ` + --deadline 180 ` + --json +``` + +The generated image-only PDFs stay under the benchmark output directory. Score them with the same human-authored entity, relation, and forbidden-content contracts as the native PDFs. + `benchmark fast-suite` writes `fast_suite_report.json` with per-case results and aggregate entity/relation recall, forbidden content, cache hit rates, provider success/retry counts, failure categories, and stage timings. ## Evaluation tiers diff --git a/rfs/cli.py b/rfs/cli.py index 0670549..0bdcd30 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -320,6 +320,7 @@ def build_parser() -> argparse.ArgumentParser: benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") + benchmark.add_argument("--rasterize-dpi", type=int, help="For benchmark fast/fast-suite, convert local PDF inputs to image-only scanned PDFs at this DPI before extraction.") benchmark.add_argument("--json", action="store_true", help="Emit JSON.") validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") @@ -576,11 +577,11 @@ def main(argv: list[str] | None = None) -> int: elif args.benchmark_action == "fast": if not args.case or not args.out: parser.error("benchmark fast requires --case and --out") - result = run_fast_benchmark_case(args.case, args.out, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine) + result = run_fast_benchmark_case(args.case, args.out, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine, rasterize_pdf_dpi=args.rasterize_dpi) elif args.benchmark_action == "fast-suite": if not args.out: parser.error("benchmark fast-suite requires --out") - result = run_fast_benchmark_suite(args.root, args.out, case_ids=args.case_id, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine) + result = run_fast_benchmark_suite(args.root, args.out, case_ids=args.case_id, deadline_seconds=args.deadline, planner_mode=args.planner_mode, planner_model=args.planner_model, ocr_engine=args.ocr_engine, rasterize_pdf_dpi=args.rasterize_dpi) elif args.benchmark_action == "pdf-suite": if not args.out: parser.error("benchmark pdf-suite requires --out") diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 321e452..67e5919 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -707,6 +707,7 @@ def run_fast_benchmark_case( planner_mode: str = "vlm", planner_model: str | None = None, ocr_engine: str = "off", + rasterize_pdf_dpi: int | None = None, ) -> dict[str, Any]: case_root = Path(case_dir).resolve() case = _load(case_root / "case.json") @@ -721,10 +722,16 @@ def run_fast_benchmark_case( if not validation.get("ok"): return {**validation, "passed": False} from ..paper_to_image import run_fast_framework_prompt + from .scanned_pdf import rasterize_pdf_as_scan preferences = case_root / str(case.get("preferences")) if case.get("preferences") else None + active_paper = paper_path + rasterized_input = None + if rasterize_pdf_dpi is not None and paper_path.suffix.casefold() == ".pdf": + rasterized_input = rasterize_pdf_as_scan(paper_path, Path(out) / "benchmark_inputs" / "paper_scanned.pdf", dpi=rasterize_pdf_dpi) + active_paper = rasterized_input run = run_fast_framework_prompt( - paper=paper_path, + paper=active_paper, out=out, deadline_seconds=deadline_seconds, planner_mode=planner_mode, @@ -738,6 +745,8 @@ def run_fast_benchmark_case( "summary": "Fast paper framework benchmark completed.", "ok": bool(run.get("ok")), "case_id": case.get("case_id"), + "rasterized_input": str(rasterized_input) if rasterized_input else None, + "rasterize_pdf_dpi": int(rasterize_pdf_dpi) if rasterized_input else None, "run": run, "benchmark": score, "passed_planning_thresholds": not any("plan_" in value for value in score.get("threshold_failures", [])), @@ -754,6 +763,7 @@ def run_fast_benchmark_suite( planner_mode: str = "vlm", planner_model: str | None = None, ocr_engine: str = "off", + rasterize_pdf_dpi: int | None = None, ) -> dict[str, Any]: root = Path(benchmarks_root).resolve() target = ensure_dir(out).resolve() @@ -772,6 +782,7 @@ def run_fast_benchmark_suite( planner_mode=planner_mode, planner_model=planner_model, ocr_engine=ocr_engine, + rasterize_pdf_dpi=rasterize_pdf_dpi, ) except Exception as exc: result = {"summary": "Fast benchmark case raised an exception.", "ok": False, "case_id": case_id, "error": str(exc)} @@ -788,6 +799,7 @@ def run_fast_benchmark_suite( "contract_source": run.get("contract_source"), "cache_hit": bool(run.get("cache_hit")), "document_cache_hit": bool(run.get("document_cache_hit")), + "rasterized_input": result.get("rasterized_input"), "elapsed_seconds": run.get("elapsed_seconds"), "stage_timings": run.get("stage_timings", {}), "extraction_quality": extraction_quality, @@ -839,6 +851,7 @@ def percentile(values: list[float], fraction: float) -> float: "cache_hit_rate": round(sum(bool(item.get("cache_hit")) for item in results) / max(1, len(results)), 4), "document_cache_hit_count": sum(bool(item.get("document_cache_hit")) for item in results), "document_cache_hit_rate": round(sum(bool(item.get("document_cache_hit")) for item in results) / max(1, len(results)), 4), + "rasterized_case_count": sum(bool(item.get("rasterized_input")) for item in results), "mean_plan_entity_recall": _mean(entity_scores), "mean_plan_relation_recall": _mean(relation_scores), "forbidden_content_total": sum(int(item.get("plan_forbidden_content_count") or 0) for item in results), @@ -881,6 +894,7 @@ def percentile(values: list[float], fraction: float) -> float: "planner_model": planner_model, "deadline_seconds": deadline_seconds, "ocr_engine": ocr_engine, + "rasterize_pdf_dpi": rasterize_pdf_dpi, "selected_case_ids": [item.get("case_id") for item in cases], "aggregate": aggregate, "cases": results, diff --git a/rfs/evaluation/scanned_pdf.py b/rfs/evaluation/scanned_pdf.py new file mode 100644 index 0000000..8f67aed --- /dev/null +++ b/rfs/evaluation/scanned_pdf.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + + +def rasterize_pdf_as_scan(source: str | Path, target: str | Path, dpi: int = 144, jpeg_quality: int = 78) -> Path: + """Create an image-only PDF fixture while preserving displayed page sizes.""" + import fitz + + source_path = Path(source).resolve() + target_path = Path(target).resolve() + if not source_path.exists(): + raise FileNotFoundError(f"PDF does not exist: {source_path}") + if source_path.suffix.casefold() != ".pdf": + raise ValueError(f"Rasterized scan input must be a PDF: {source_path}") + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.unlink(missing_ok=True) + document = fitz.open(str(source_path)) + scanned = fitz.open() + try: + for page in document: + pixmap = page.get_pixmap(dpi=max(72, min(300, int(dpi))), alpha=False) + image = pixmap.tobytes("jpeg", jpg_quality=max(40, min(95, int(jpeg_quality)))) + output_page = scanned.new_page(width=float(page.rect.width), height=float(page.rect.height)) + output_page.insert_image(output_page.rect, stream=image) + scanned.set_metadata({ + "title": "ResearchFigureStudio rasterized scan benchmark", + "author": "ResearchFigureStudio", + "subject": "Deterministic image-only PDF benchmark fixture", + "keywords": "ResearchFigureStudio,scan,benchmark", + "creator": "ResearchFigureStudio", + "producer": "ResearchFigureStudio", + "creationDate": "", + "modDate": "", + }) + scanned.save(str(target_path), garbage=4, deflate=True, no_new_id=True, preserve_metadata=False) + finally: + scanned.close() + document.close() + return target_path diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 5a21495..53caef8 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -53,7 +53,7 @@ class ConceptRule: ConceptRule("image_encoder", "Image Encoder", "modules", "encoder", r"\bimage encoder\b"), ConceptRule("text_encoder", "Text Encoder", "modules", "encoder", r"\btext encoder\b"), ConceptRule("transformer_encoder", "Transformer Encoder", "modules", "encoder", r"\btransformer encoder\b|^encoder$", (), True), - ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b|^decoder$", (), True), + ConceptRule("transformer_decoder", "Transformer Decoder", "modules", "decoder", r"\btransformer decoder\b|\b(?:the\s+)?decoder receives\b|^decoder$", (), True), ConceptRule("position_embedding", "Position Embedding", "modules", "conditioning", r"\bposition embeddings?\b", ("positional embedding",), True), ConceptRule("positional_encoding", "Positional Encoding", "modules", "conditioning", r"\bposition(?:al)? encod(?:ing|ings)\b"), ConceptRule("image_patches", "Image Patches", "modules", "intermediate", r"\bimage patches?\b|\bfixed-size patches\b", ("patches",), True), diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index bb4ce07..31bc434 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -128,6 +128,11 @@ def test_inspect_pdf_cli_defaults(self): self.assertEqual(args.deadline, 180) self.assertEqual(args.ocr_engine, "auto") + def test_scan_benchmark_cli_accepts_rasterize_dpi(self): + parser = build_parser() + args = parser.parse_args(["benchmark", "fast-suite", "--out", "output/scan", "--rasterize-dpi", "144"]) + self.assertEqual(args.rasterize_dpi, 144) + def test_doctor_reports_easyocr_model_readiness(self): report = _doctor() models = report["pdf_tools"]["easyocr"]["models"] diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 72d6652..2c84177 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -1,3 +1,4 @@ +import hashlib import json import tempfile import unittest @@ -16,12 +17,38 @@ validate_benchmark_case, ) from rfs.evaluation.pdf_extraction_benchmark import run_pdf_extraction_stress_suite +from rfs.evaluation.scanned_pdf import rasterize_pdf_as_scan +from rfs.paper_to_image.analyzer import parse_paper ROOT = Path(__file__).resolve().parents[2] class BenchmarkingTests(unittest.TestCase): + def test_rasterized_scan_fixture_preserves_pages_and_removes_native_text(self): + import fitz + + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source.pdf" + target = Path(temp) / "scanned.pdf" + second_target = Path(temp) / "scanned_again.pdf" + document = fitz.open() + for text in ("Abstract native text", "Method native text"): + page = document.new_page(width=600, height=800) + page.insert_text((50, 80), text, fontsize=14) + document.save(source) + document.close() + + rasterize_pdf_as_scan(source, target, dpi=110) + rasterize_pdf_as_scan(source, second_target, dpi=110) + parsed = parse_paper(target, ocr_engine="off") + first_hash = hashlib.sha256(target.read_bytes()).hexdigest() + second_hash = hashlib.sha256(second_target.read_bytes()).hexdigest() + + self.assertEqual(parsed["page_count"], 2) + self.assertEqual(parsed["extraction_report"]["pdf_type"], "scanned") + self.assertTrue(all(page["char_count"] == 0 for page in parsed["pages"])) + self.assertEqual(first_hash, second_hash) def test_planning_contract_treats_relation_labels_as_intermediate_artifacts(self): expected = { "entities": [ diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 4cfe67f..c68252f 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -655,7 +655,7 @@ def test_generic_completion_uses_selected_architecture_page_context_and_diagram_ {"id": "E0005", "page": 20, "kind": "paragraph", "bbox": [40, 150, 500, 170], "text": "The decoder receives object queries and encoder memory.", "section_hint": "Detailed Architecture", "confidence": 0.98}, {"id": "E0006", "page": 20, "kind": "paragraph", "bbox": [340, 280, 380, 300], "text": "FFN", "section_hint": "Detailed Architecture", "confidence": 0.98}, {"id": "E0007", "page": 20, "kind": "paragraph", "bbox": [140, 390, 190, 410], "text": "Encoder", "section_hint": "Detailed Architecture", "confidence": 0.98}, - {"id": "E0008", "page": 20, "kind": "paragraph", "bbox": [310, 310, 360, 330], "text": "Decoder", "section_hint": "Detailed Architecture", "confidence": 0.98}, + {"id": "E0008", "page": 20, "kind": "paragraph", "bbox": [310, 310, 360, 330], "text": "Add & Norm", "section_hint": "Detailed Architecture", "confidence": 0.98}, {"id": "E0009", "page": 20, "kind": "paragraph", "bbox": [40, 180, 500, 200], "text": "The final class labels and bounding boxes are predicted in parallel.", "section_hint": "Detailed Architecture", "confidence": 0.98}, {"id": "E0010", "page": 20, "kind": "paragraph", "bbox": [40, 710, 500, 730], "text": "For example, a separate text encoder uses class descriptions and contrastive learning to create text embeddings.", "section_hint": "Related Work", "confidence": 0.98}, ], From e36b162d746c34179a970a2450816a1e2351685b Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 14:54:43 +0800 Subject: [PATCH 42/74] Bound fast VLM planning to the workflow deadline --- docs/paper-to-image.md | 4 +++ rfs/paper_to_image/analyzer.py | 8 +++-- rfs/paper_to_image/contract_completion.py | 26 ++++++++++++++++ rfs/paper_to_image/planner.py | 6 ++-- rfs/paper_to_image/preparation.py | 25 +++++++++------ rfs/paper_to_image/review.py | 4 +-- rfs/vlm_client.py | 25 ++++++++++++--- tests/unit/test_paper_contract.py | 37 +++++++++++++++++++++++ tests/unit/test_vlm_client.py | 13 ++++++++ 9 files changed, 128 insertions(+), 20 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index e694a00..bd82565 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -39,6 +39,10 @@ When a deadline is active, every local OCR page runs in an isolated worker proce For a fully scanned long paper with no native section signals, the six-page fallback schedule covers pages 1-4 first, then a page near 85% of the document and finally a middle page. Under a deadline, the first wave uses adaptive page concurrency; the two coverage-rescue pages run one at a time with two OCR threads whenever at least 45 seconds remain. This biases scarce OCR time toward the abstract, overview figure, early method details, and likely appendix/conclusion architecture evidence without weakening the hard cutoff. +In the 180-second fast path with `planner_mode=vlm`, rescue pages require at least 90 seconds remaining so the workflow can preserve a model-call and validation budget. Heuristic planning and PDF inspection retain the 45-second rescue threshold. The effective value is recorded as `ocr_rescue_min_remaining_seconds`. + +Planner and review retries share one absolute provider deadline rather than receiving a fresh timeout per attempt. If the service is slow or unavailable, each request is clamped to the remaining budget, further retries stop at the cutoff, and any later semantic stage switches to the deterministic evidence-grounded fallback. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index ee3df8b..d32b33c 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -801,6 +801,7 @@ def _read_pdf_pages( ocr_lang: str, ocr_adapter: Callable | None, max_ocr_pages: int, + ocr_rescue_min_remaining: float, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: try: import fitz @@ -898,7 +899,7 @@ def deadline_allows_wave(rescue_page: bool = False) -> bool: if deadline_at is not None: remaining = deadline_at - time.monotonic() if rescue_page: - if remaining < 45.0: + if remaining < max(25.0, float(ocr_rescue_min_remaining)): warnings.append("OCR stopped to preserve deadline validation budget") return False return True @@ -970,6 +971,7 @@ def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str "ocr_schedule_complete": schedule_complete, "ocr_run_complete": run_complete, "ocr_worker_count": ocr_worker_count, + "ocr_rescue_min_remaining_seconds": float(ocr_rescue_min_remaining), "warnings": warnings, "poppler_available": bool(poppler_text), "elapsed_seconds": elapsed, @@ -1347,6 +1349,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "ocr_schedule_complete": bool(details.get("ocr_schedule_complete", True)), "ocr_run_complete": bool(details.get("ocr_run_complete", True)), "ocr_worker_count": int(details.get("ocr_worker_count") or 1), + "ocr_rescue_min_remaining_seconds": float(details.get("ocr_rescue_min_remaining_seconds") or 45.0), "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), "mean_ocr_confidence": mean_ocr_confidence, @@ -1378,13 +1381,14 @@ def parse_paper( ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, max_ocr_pages: int = 6, + ocr_rescue_min_remaining: float = 45.0, ) -> dict[str, Any]: source = Path(path).resolve() if not source.exists(): raise FileNotFoundError(f"Paper does not exist: {source}") source_hash = hashlib.sha256(source.read_bytes()).hexdigest() if source.suffix.lower() == ".pdf": - pages, details = _read_pdf_pages(source, max_chars, deadline_at, ocr_engine, ocr_lang, ocr_adapter, max_ocr_pages) + pages, details = _read_pdf_pages(source, max_chars, deadline_at, ocr_engine, ocr_lang, ocr_adapter, max_ocr_pages, ocr_rescue_min_remaining) loader = "structured-pdf" else: pages, loader = _read_non_pdf_pages(source, max_chars) diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 53caef8..508b61f 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -595,6 +595,32 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) found[rule.key] = item added_entities.append(item["id"]) + if conservative_expansion: + for _pass in range(3): + progress = False + connected_keys = set(found) + for rule in CONCEPT_RULES: + if rule.key in found or _find_existing(spec, rule): + continue + connected_to_found = any( + (source_key == rule.key and target_key in connected_keys) + or (target_key == rule.key and source_key in connected_keys) + for source_key, target_key, _ in RELATION_RULES + ) + if not connected_to_found: + continue + matches = _find_matches(rule, selected, []) or _find_matches(rule, [], relevant_non_background) + if not matches: + continue + evidence_ids = list(dict.fromkeys(str(item.get("id")) for item in matches if item.get("id"))) + item = {"id": _stable_id(rule.field.rstrip("s"), rule.label), "name": rule.label, "role": rule.role, "evidence_ids": evidence_ids} + spec.setdefault(rule.field, []).append(item) + found[rule.key] = item + added_entities.append(item["id"]) + progress = True + if not progress: + break + for rule in CONCEPT_RULES: if rule.key not in found: existing = _find_existing(spec, rule) diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 6a0b443..156d63f 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -462,6 +462,7 @@ def plan_fast_paper_contract( timeout_seconds: int = 45, retries: int = 2, evidence_max_chars: int = 36000, + deadline_at: float | None = None, ) -> tuple[dict, dict]: prompt = _fast_planner_prompt(parsed, preferences, evidence_max_chars=evidence_max_chars) metadata: dict[str, Any] = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} @@ -475,6 +476,7 @@ def plan_fast_paper_contract( timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"], + deadline_at=deadline_at, ) metadata["model"] = resolved return _compile_fast_plan(raw, preferences), metadata @@ -485,13 +487,13 @@ def plan_fast_paper_contract( return normalize_plan(_heuristic_plan(parsed, preferences), preferences), metadata -def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1, evidence_max_chars: int = 58000) -> tuple[dict, dict]: +def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: str | None = None, reference_images: list[str] | None = None, paper_review: dict | None = None, timeout_seconds: int = 240, retries: int = 1, evidence_max_chars: int = 58000, deadline_at: float | None = None) -> tuple[dict, dict]: prompt = _planner_prompt(parsed, preferences, paper_review=paper_review, evidence_max_chars=evidence_max_chars) metadata = {"requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_MODEL", "RFS_PAPER_PLANNER_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"]) + raw = call_vlm_json(prompt, reference_images or [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"], deadline_at=deadline_at) metadata["model"] = resolved return normalize_plan(raw, preferences), metadata except Exception as exc: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index aa5191a..5c09e65 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -755,10 +755,12 @@ def prepare_paper_figure_contract( deadline = max(30, int(deadline_seconds)) deadline_at = started + deadline + provider_deadline_at = deadline_at - 15.0 + ocr_rescue_min_remaining = 90.0 if fast_mode and planner_mode == "vlm" and deadline <= 240 else 45.0 parsed = None if ocr_adapter else read_document_cache(archived_paper, ocr_engine=ocr_engine, ocr_lang=ocr_lang) document_cache_hit = parsed is not None if parsed is None: - parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter) + parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, ocr_rescue_min_remaining=ocr_rescue_min_remaining) if not ocr_adapter: write_document_cache(archived_paper, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) document_preparation_seconds = round(time.monotonic() - started, 3) @@ -786,7 +788,7 @@ def prepare_paper_figure_contract( selected_domain = detect_domain_profile(parsed, explicit=domain_profile) write_json(root / "domain_profile.json", selected_domain) if fast_mode: - remaining = max(10, int(deadline_at - time.monotonic() - 15)) + remaining = max(0, int(provider_deadline_at - time.monotonic())) effective_mode = planner_mode if remaining >= 25 else "heuristic" cache_path = _fast_cache_path(parsed, str(planner_model or ""), preferences) if cache_path.exists() and effective_mode == "vlm" and _semantic_cache_safe(parsed): @@ -801,29 +803,32 @@ def prepare_paper_figure_contract( timeout_seconds=min(35, remaining), retries=2, evidence_max_chars=36000, + deadline_at=provider_deadline_at, ) if planner_metadata.get("mode") == "vlm": paper_review = _paper_review_from_plan(plan, selected_domain) review_metadata = {"summary": "Paper review derived from the successful fast VLM plan.", "requested_mode": planner_mode, "mode": "derived_from_vlm_plan", "model": planner_metadata.get("model"), "warning": None, "prompt": ""} else: - remaining = max(10, int(deadline_at - time.monotonic() - 15)) + remaining = max(0, int(provider_deadline_at - time.monotonic())) + review_mode = planner_mode if remaining >= 25 else "heuristic" paper_review, review_metadata = build_paper_review( parsed, selected_domain, - mode=effective_mode, + mode=review_mode, model=planner_model, - timeout_seconds=min(35, remaining), + timeout_seconds=max(1, min(35, remaining)), retries=1, evidence_max_chars=36000, + deadline_at=provider_deadline_at, ) if review_metadata.get("mode") == "vlm": plan = build_review_grounded_plan(parsed, preferences, paper_review) planner_metadata = {**planner_metadata, "mode": "review_grounded_vlm", "model": review_metadata.get("model"), "warning": planner_metadata.get("warning")} else: - remaining = max(10, int(deadline_at - time.monotonic() - 15)) + remaining = max(0, int(provider_deadline_at - time.monotonic())) effective_mode = planner_mode if remaining >= 30 else "heuristic" - paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=effective_mode, model=planner_model, timeout_seconds=min(35, remaining), retries=1) - remaining = max(10, int(deadline_at - time.monotonic() - 15)) + paper_review, review_metadata = build_paper_review(parsed, selected_domain, mode=effective_mode, model=planner_model, timeout_seconds=max(1, min(35, remaining)), retries=1, deadline_at=provider_deadline_at) + remaining = max(0, int(provider_deadline_at - time.monotonic())) planning_mode = effective_mode if remaining >= 25 else "heuristic" plan, planner_metadata = plan_paper_image( parsed, @@ -832,8 +837,9 @@ def prepare_paper_figure_contract( model=planner_model, reference_images=archived_positive + archived_negative, paper_review=paper_review, - timeout_seconds=min(35, remaining), + timeout_seconds=max(1, min(35, remaining)), retries=1, + deadline_at=provider_deadline_at, ) fallback_plan = build_review_grounded_plan(parsed, preferences, paper_review) if review_metadata.get("mode") == "vlm": @@ -934,6 +940,7 @@ def prepare_paper_figure_contract( "ocr_schedule_complete": parsed["extraction_report"].get("ocr_schedule_complete", True), "ocr_run_complete": parsed["extraction_report"].get("ocr_run_complete", True), "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), + "ocr_rescue_min_remaining_seconds": parsed["extraction_report"].get("ocr_rescue_min_remaining_seconds", 45.0), "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), }, diff --git a/rfs/paper_to_image/review.py b/rfs/paper_to_image/review.py index 3aef6bb..2033f59 100644 --- a/rfs/paper_to_image/review.py +++ b/rfs/paper_to_image/review.py @@ -328,13 +328,13 @@ def validate_review_coverage(review: dict, parsed: dict, domain_profile: dict, s } -def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1, evidence_max_chars: int = 65000) -> tuple[dict, dict]: +def build_paper_review(parsed: dict, domain_profile: dict, mode: str = "vlm", model: str | None = None, timeout_seconds: int = 300, retries: int = 1, evidence_max_chars: int = 65000, deadline_at: float | None = None) -> tuple[dict, dict]: prompt = _review_prompt(parsed, domain_profile, evidence_max_chars=evidence_max_chars) metadata = {"summary": "Paper-review execution metadata.", "requested_mode": mode, "mode": mode, "model": None, "warning": None, "prompt": prompt, "provider": {}} if mode == "vlm" and vlm_credentials_available(): resolved = resolve_vlm_model("RFS_PAPER_REVIEW_MODEL", "RFS_PAPER_TO_IMAGE_MODEL", explicit_model=model) try: - raw = call_vlm_json(prompt, [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"]) + raw = call_vlm_json(prompt, [], model=resolved, timeout=max(10, int(timeout_seconds)), retries=max(0, int(retries)), call_metadata=metadata["provider"], deadline_at=deadline_at) metadata["model"] = resolved review = normalize_review(raw, domain_profile) _expand_evidence(review, parsed) diff --git a/rfs/vlm_client.py b/rfs/vlm_client.py index 5c5d792..8cc1cf1 100644 --- a/rfs/vlm_client.py +++ b/rfs/vlm_client.py @@ -65,6 +65,7 @@ def call_vlm_json( timeout: int = 180, retries: int = 1, call_metadata: dict[str, Any] | None = None, + deadline_at: float | None = None, ) -> dict: api_base = os.getenv("API_BASE", "").rstrip("/") api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") @@ -97,15 +98,25 @@ def call_vlm_json( "elapsed_seconds": 0.0, "failure_categories": [], }) + attempts_made = 0 for attempt in range(attempts): + request_timeout = max(0.1, float(timeout)) + if deadline_at is not None: + remaining = float(deadline_at) - time.monotonic() + if remaining <= 0.1: + last_error = TimeoutError("VLM deadline budget exhausted") + failures.append({"attempt": attempts_made, "category": "timeout"}) + break + request_timeout = min(request_timeout, remaining) + attempts_made += 1 if call_metadata is not None: - call_metadata["attempts"] = attempt + 1 + call_metadata["attempts"] = attempts_made try: response = requests.post( f"{api_base}/chat/completions", headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), - timeout=timeout, + timeout=request_timeout, ) response.raise_for_status() content_text = response.json()["choices"][0]["message"]["content"] @@ -120,14 +131,18 @@ def call_vlm_json( return result except Exception as exc: last_error = exc - failures.append({"attempt": attempt + 1, "category": _error_category(exc)}) + failures.append({"attempt": attempts_made, "category": _error_category(exc)}) if attempt < attempts - 1: - time.sleep(min(2.0, 0.75 * (2 ** attempt))) + delay = min(2.0, 0.75 * (2 ** attempt)) + if deadline_at is not None and float(deadline_at) - time.monotonic() <= delay + 0.1: + break + time.sleep(delay) if call_metadata is not None: call_metadata.update({ "success": False, - "retries_used": max(0, attempts - 1), + "retries_used": max(0, attempts_made - 1), "elapsed_seconds": round(time.monotonic() - started, 3), "failure_categories": list(dict.fromkeys(item["category"] for item in failures)), + "deadline_reached": bool(deadline_at is not None and time.monotonic() >= float(deadline_at)), }) raise last_error or RuntimeError("VLM call failed") diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index c68252f..1fb9640 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -820,6 +820,43 @@ def test_generic_completion_repairs_vlm_ids_labels_and_relation_grounding(self): self.assertTrue(all(item.get("evidence_ids") for item in spec["relations"] if item.get("source") in {"embedding_layer", "transformer_encoder"} and item.get("target") in {"transformer_encoder", "mlm_head"})) self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0006", "E0007"]) + def test_rich_contract_expands_newly_grounded_intermediate_to_evidence_neighbors(self): + caption = "Figure 1: Overall pre-training and fine-tuning procedures." + parsed = { + "page_count": 16, + "document_index": {"figures": [{"page": 3, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 3, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 0.99}, + {"id": "E0002", "page": 4, "kind": "paragraph", "text": "For a given token, its input representation is", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0003", "page": 4, "kind": "paragraph", "text": "constructed by summing the corresponding token,", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0004", "page": 4, "kind": "paragraph", "text": "segment, and position embeddings.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0005", "page": 4, "kind": "paragraph", "text": "The architecture is a multi-layer bidirectional Transformer encoder.", "section_hint": "Method", "confidence": 0.98}, + {"id": "E0006", "page": 4, "kind": "paragraph", "text": "We train with Masked LM and Next Sentence Prediction before fine-tuning on downstream tasks.", "section_hint": "Method", "confidence": 0.98}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [{"id": "sentence_pair", "name": "Sentence Pair", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "bert", "name": "Bidirectional Transformer Encoder", "evidence_ids": ["E0005"]}, + {"id": "mlm", "name": "Masked LM", "evidence_ids": ["E0006"]}, + {"id": "nsp", "name": "Next Sentence Prediction", "evidence_ids": ["E0006"]}, + ], + "outputs": [{"id": "tasks", "name": "Downstream Tasks", "evidence_ids": ["E0006"]}], + "relations": [{"source": "bert", "target": "mlm", "type": "training_objective", "evidence_ids": ["E0006"]}], + "innovations": [], "must_show": [], "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + ids = {_item.get("name"): _item.get("id") for field in ("inputs", "modules", "outputs") for _item in spec[field]} + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + for label in ("Input Sequence", "Token Embeddings", "Segment Embeddings", "Position Embeddings", "Input Representation"): + self.assertIn(label, ids) + self.assertIn((ids["Input Representation"], ids["Bidirectional Transformer Encoder"]), pairs) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_vlm_client.py b/tests/unit/test_vlm_client.py index bfa75e4..beef1b0 100644 --- a/tests/unit/test_vlm_client.py +++ b/tests/unit/test_vlm_client.py @@ -29,6 +29,19 @@ def test_retry_metadata_records_recovery_without_secrets(self, _post, _sleep): self.assertEqual(metadata["failure_categories"], ["tls"]) self.assertNotIn("secret-token", str(metadata)) + @patch("rfs.vlm_client.requests.post") + def test_expired_deadline_skips_provider_call(self, post): + metadata = {} + with patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-token"}, clear=False): + with self.assertRaisesRegex(TimeoutError, "deadline budget exhausted"): + call_vlm_json("Return JSON", [], model="test-model", timeout=30, retries=2, call_metadata=metadata, deadline_at=0.0) + + post.assert_not_called() + self.assertEqual(metadata["attempts"], 0) + self.assertEqual(metadata["retries_used"], 0) + self.assertEqual(metadata["failure_categories"], ["timeout"]) + self.assertTrue(metadata["deadline_reached"]) + if __name__ == "__main__": unittest.main() From 88ccbc62a2e4de94e86e06374e9aee2424004fcb Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 15:07:42 +0800 Subject: [PATCH 43/74] Filter repeated PDF margins and preserve rotated reading order --- docs/paper-to-image.md | 4 ++ rfs/evaluation/benchmarking.py | 1 + rfs/evaluation/pdf_extraction_benchmark.py | 32 +++++++++ rfs/paper_to_image/analyzer.py | 80 ++++++++++++++++++++-- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 1 + tests/unit/test_benchmarking.py | 6 +- 7 files changed, 119 insertions(+), 7 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index bd82565..ea9a700 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -43,6 +43,10 @@ In the 180-second fast path with `planner_mode=vlm`, rescue pages require at lea Planner and review retries share one absolute provider deadline rather than receiving a fresh timeout per attempt. If the service is slow or unavailable, each request is clamped to the remaining budget, further retries stop at the cutoff, and any later semantic stage switches to the deterministic evidence-grounded fallback. +Born-digital and OCR-recovered pages also pass through a conservative repeated-margin filter. Short header or footer lines are removed only when the same digit-normalized pattern appears in the same margin on at least three pages and at least one quarter of the document. The report records `repeated_margin_noise_removed_count` so benchmark runs can detect publisher-header and page-number contamination without hiding the cleanup. + +For pages carrying a PDF rotation flag, native blocks are ordered in the unrotated media-box coordinate system and only then transformed into displayed page coordinates. This preserves Abstract-to-Method-to-Figure reading order on 90/180/270-degree pages while keeping overlay coordinates aligned with rendered output. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 67e5919..913784f 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -876,6 +876,7 @@ def percentile(values: list[float], fraction: float) -> float: "ocr_completed_page_total": sum(int(value) for value in extraction_metric("ocr_completed_count")), "ocr_incomplete_run_count": sum(not bool(item.get("extraction_quality", {}).get("ocr_run_complete", True)) for item in results), "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), + "repeated_margin_noise_removed_total": sum(int(value) for value in extraction_metric("repeated_margin_noise_removed_count")), "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), "provider_call_count": len(provider_calls), diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index e9041d4..b7e3ab6 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -88,6 +88,25 @@ def _mixed_scan_fixture(target: Path) -> Path: return _save_document(document, target) +def _repeated_margin_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + sections = [ + ("Abstract", "A document encoder grounds paper evidence before framework generation."), + ("2 Method", "The document encoder sends grounded entities to a relation decoder and final output."), + ("3 Experiments", "Experiments measure entity recall, relation recall, and extraction latency."), + ("4 Conclusion", "The system preserves evidence citations and editable scientific labels."), + ] + for page_number, (heading, body) in enumerate(sections, 1): + page = document.new_page(width=600, height=800) + page.insert_text((45, 25), "Proceedings of the Example Conference 2026", fontsize=8) + page.insert_text((45, 785), f"Anonymous Paper 1234 | Page {page_number}", fontsize=8) + page.insert_text((45, 70), heading, fontname="hebo", fontsize=12) + page.insert_textbox((45, 95, 555, 220), body, fontsize=10) + return _save_document(document, target) + + def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: if "page_002" not in image_path.name: return [] @@ -180,6 +199,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> bold = _bold_heading_fixture(fixtures / "unnumbered_bold_sections.pdf") rotated = _rotated_fixture(fixtures / "rotated_native_page.pdf") mixed = _mixed_scan_fixture(fixtures / "mixed_scan.pdf") + repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") results = [ _run_case( @@ -211,6 +231,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> lambda parsed: [ _check("rotation", parsed["extraction_report"].get("rotated_pages") == [1], parsed["extraction_report"].get("rotated_pages"), [1]), _check("display_coordinates", all(0 <= block["bbox"][0] <= block["bbox"][2] <= parsed["pages"][0]["width"] and 0 <= block["bbox"][1] <= block["bbox"][3] <= parsed["pages"][0]["height"] for block in parsed["pages"][0]["blocks"]), parsed["pages"][0]["blocks"], "all blocks inside displayed page"), + _check("semantic_reading_order", parsed["pages"][0]["text"].index("Abstract") < parsed["pages"][0]["text"].index("2 Method") < parsed["pages"][0]["text"].index("Figure 1:"), parsed["pages"][0]["text"], "Abstract < Method < Figure 1"), _check("caption", parsed["extraction_report"].get("figure_caption_count", 0) >= 1, parsed["extraction_report"].get("figure_caption_count"), ">=1"), ], ), @@ -228,6 +249,17 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> ocr_adapter=_fixture_ocr_adapter, preview_page=1, ), + _run_case( + "repeated_margin_noise", + repeated_margin, + root, + lambda parsed: [ + _check("noise_removed", parsed["extraction_report"].get("repeated_margin_noise_removed_count") == 8, parsed["extraction_report"].get("repeated_margin_noise_removed_count"), 8), + _check("header_absent", all("Proceedings of the Example Conference" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no repeated header evidence"), + _check("footer_absent", all("Anonymous Paper 1234" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no repeated footer evidence"), + _check("scientific_content_preserved", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder evidence"), + ], + ), ] if ocr_engine != "off": diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index d32b33c..a854bdb 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -543,6 +543,73 @@ def _filter_ocr_margin_noise(records: list[dict[str, Any]], image_width: float) return kept, removed +def _repeated_margin_signature(text: str) -> str: + value = _clean(text).casefold() + value = re.sub(r"\d+", "#", value) + value = re.sub(r"\s+", " ", value).strip(" |.-_") + return value + + +def _remove_repeated_margin_noise(pages: list[dict[str, Any]]) -> int: + if len(pages) < 3: + return 0 + occurrences: dict[tuple[str, str], set[int]] = {} + candidates: dict[tuple[int, str], tuple[str, str]] = {} + for page in pages: + page_number = int(page.get("page") or 0) + height = float(page.get("height") or 0.0) + if height <= 0: + continue + for block in page.get("blocks", []): + bbox = block.get("bbox") + text = str(block.get("text") or "").strip() + if not isinstance(bbox, (list, tuple)) or len(bbox) != 4 or not text or len(text) > 140 or len(_lexical_units(text)) > 12: + continue + top_region = float(bbox[3]) <= height * 0.12 + bottom_region = float(bbox[1]) >= height * 0.88 + if not top_region and not bottom_region: + continue + signature = _repeated_margin_signature(text) + if not signature: + continue + key = ("top" if top_region else "bottom", signature) + occurrences.setdefault(key, set()).add(page_number) + candidates[(page_number, str(block.get("id") or ""))] = key + + threshold = max(3, (len(pages) + 3) // 4) + repeated = {key for key, page_numbers in occurrences.items() if len(page_numbers) >= threshold} + if not repeated: + return 0 + + removed = 0 + for page in pages: + page_number = int(page.get("page") or 0) + kept = [] + page_removed = 0 + for block in page.get("blocks", []): + key = candidates.get((page_number, str(block.get("id") or ""))) + if key in repeated: + page_removed += 1 + continue + kept.append(block) + if not page_removed: + continue + for order, block in enumerate(kept, 1): + block["reading_order"] = order + page_text = "\n\n".join(str(item.get("text") or "") for item in kept) + page.update({ + "blocks": kept, + "text": page_text, + "char_count": len(page_text), + "replacement_character_rate": _replacement_rate(page_text), + "mojibake_rate": _mojibake_rate(page_text), + "column_count": max((int(item.get("column", 0)) for item in kept), default=0) + 1, + "repeated_margin_noise_removed": page_removed, + }) + removed += page_removed + return removed + + def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, str | None, dict[str, Any]]: try: with tempfile.TemporaryDirectory() as temp: @@ -818,13 +885,15 @@ def _read_pdf_pages( ocr_candidates: list[int] = [] try: for index, page in enumerate(document, 1): - raw_blocks = _rotate_blocks_to_page(_pymupdf_line_blocks(page), page) - ordered, order_confidence = _reading_order(raw_blocks, float(page.rect.width)) + native_width = float(page.mediabox.width) if int(page.rotation or 0) else float(page.rect.width) + raw_blocks = _pymupdf_line_blocks(page) + ordered, order_confidence = _reading_order(raw_blocks, native_width) if len(_normalized_compare_text(" ".join(item["text"] for item in ordered))) < 80: - fallback_blocks = _rotate_blocks_to_page(_pdfplumber_blocks(path, index), page) - fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, float(page.rect.width)) + fallback_blocks = _pdfplumber_blocks(path, index) + fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, native_width) if len(_normalized_compare_text(" ".join(item["text"] for item in fallback_ordered))) > len(_normalized_compare_text(" ".join(item["text"] for item in ordered))): ordered, order_confidence = fallback_ordered, min(fallback_confidence, 0.9) + ordered = _rotate_blocks_to_page(ordered, page) for block_index, block in enumerate(ordered, 1): block["id"] = f"P{index:03d}_B{block_index:03d}" block["kind"] = _block_kind(block["text"], (block["bbox"][2] - block["bbox"][0]) / max(1.0, float(page.rect.width))) @@ -951,6 +1020,7 @@ def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str ocr_started = time.monotonic() result = _ocr_page(document[page_number - 1], page_number, ocr_engine, ocr_lang, ocr_adapter, rapidocr_threads=rapidocr_threads) consume_ocr_result(page_number, result, time.monotonic() - ocr_started) + repeated_margin_noise_removed = _remove_repeated_margin_noise(pages) document.close() elapsed = round(time.monotonic() - started, 3) attempted_pages = [int(item["page"]) for item in ocr_page_durations if item.get("page")] @@ -972,6 +1042,7 @@ def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str "ocr_run_complete": run_complete, "ocr_worker_count": ocr_worker_count, "ocr_rescue_min_remaining_seconds": float(ocr_rescue_min_remaining), + "repeated_margin_noise_removed_count": repeated_margin_noise_removed, "warnings": warnings, "poppler_available": bool(poppler_text), "elapsed_seconds": elapsed, @@ -1350,6 +1421,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "ocr_run_complete": bool(details.get("ocr_run_complete", True)), "ocr_worker_count": int(details.get("ocr_worker_count") or 1), "ocr_rescue_min_remaining_seconds": float(details.get("ocr_rescue_min_remaining_seconds") or 45.0), + "repeated_margin_noise_removed_count": int(details.get("repeated_margin_noise_removed_count") or 0), "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), "mean_ocr_confidence": mean_ocr_confidence, diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 5d3ceee..f11875a 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 23 +DOCUMENT_CACHE_VERSION = 24 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 5c09e65..214ef45 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -941,6 +941,7 @@ def prepare_paper_figure_contract( "ocr_run_complete": parsed["extraction_report"].get("ocr_run_complete", True), "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), "ocr_rescue_min_remaining_seconds": parsed["extraction_report"].get("ocr_rescue_min_remaining_seconds", 45.0), + "repeated_margin_noise_removed_count": parsed["extraction_report"].get("repeated_margin_noise_removed_count", 0), "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), }, diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 2c84177..22b6431 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -172,6 +172,7 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("merged_heading_line_total", result["aggregate"]) self.assertIn("figure_caption_total", result["aggregate"]) self.assertIn("max_ocr_worker_count", result["aggregate"]) + self.assertIn("repeated_margin_noise_removed_total", result["aggregate"]) self.assertIn("ocr_spacing_repair_total", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) @@ -190,9 +191,10 @@ def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr( result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") self.assertTrue(result["ok"]) - self.assertEqual(result["aggregate"]["case_count"], 4) - self.assertEqual(result["aggregate"]["passed_case_count"], 4) + self.assertEqual(result["aggregate"]["case_count"], 5) + self.assertEqual(result["aggregate"]["passed_case_count"], 5) self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "repeated_margin_noise.pdf").exists()) self.assertTrue((Path(tmp) / "mixed_scan_fixture_adapter" / "document_model.json").exists()) self.assertTrue((Path(tmp) / "pdf_extraction_stress_report.json").exists()) From 68538fc8a4ef3c1d731c66d666de52564f8e0306 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 15:44:53 +0800 Subject: [PATCH 44/74] Preserve multilingual labels and repair PDF hyphenation --- docs/paper-to-image.md | 4 ++ rfs/evaluation/benchmarking.py | 1 + rfs/evaluation/pdf_extraction_benchmark.py | 54 +++++++++++++++ rfs/paper_to_image/analyzer.py | 59 +++++++++++++++- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/planner.py | 44 +++++++++++- rfs/paper_to_image/preparation.py | 34 +++++++++- tests/unit/test_benchmarking.py | 7 +- tests/unit/test_paper_contract.py | 79 ++++++++++++++++++++++ 9 files changed, 275 insertions(+), 9 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index ea9a700..d62ced1 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -47,6 +47,10 @@ Born-digital and OCR-recovered pages also pass through a conservative repeated-m For pages carrying a PDF rotation flag, native blocks are ordered in the unrotated media-box coordinate system and only then transformed into displayed page coordinates. This preserves Abstract-to-Method-to-Figure reading order on 90/180/270-degree pages while keeping overlay coordinates aligned with rendered output. +Fast VLM planning preserves the paper's original writing system for every visible scientific label. Entity names and relation labels must either occur verbatim in their cited evidence or remain in the same source script; cross-script translations such as `文档编码器` to `Document encoder` fail planning validation and cannot be marked production-ready. Explanatory prose may follow the requested output language, but editable overlay labels do not. + +Native PDF lines split by publisher hyphenation are joined inside the same source text block before evidence IDs are created. Known scientific words such as `trans-` + `former` and `de-` + `coder` lose the layout-only hyphen, while unknown long compound parts retain a literal hyphen. The extraction report records `native_hyphenation_repair_count`. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 913784f..630e183 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -877,6 +877,7 @@ def percentile(values: list[float], fraction: float) -> float: "ocr_incomplete_run_count": sum(not bool(item.get("extraction_quality", {}).get("ocr_run_complete", True)) for item in results), "max_ocr_worker_count": max((int(value) for value in extraction_metric("ocr_worker_count")), default=1), "repeated_margin_noise_removed_total": sum(int(value) for value in extraction_metric("repeated_margin_noise_removed_count")), + "native_hyphenation_repair_total": sum(int(value) for value in extraction_metric("native_hyphenation_repair_count")), "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), "provider_call_count": len(provider_calls), diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index b7e3ab6..15712ee 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -107,6 +107,37 @@ def _repeated_margin_fixture(target: Path) -> Path: return _save_document(document, target) +def _native_chinese_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + font = fitz.Font(fontname="china-s") + page.insert_font(fontname="CJK", fontbuffer=font.buffer) + page.insert_text((45, 50), "\u6458\u8981", fontname="CJK", fontsize=13) + page.insert_textbox((45, 75, 555, 145), "\u672c\u6587\u63d0\u51fa\u8bba\u6587\u7f16\u7801\u5668\u548c\u5173\u7cfb\u89e3\u7801\u5668,\u751f\u6210\u53ef\u7f16\u8f91\u6846\u67b6\u56fe\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 185), "2 \u65b9\u6cd5", fontname="CJK", fontsize=13) + page.insert_textbox((45, 210, 555, 300), "\u8f93\u5165\u8bba\u6587\u9996\u5148\u8fdb\u5165\u6587\u6863\u7f16\u7801\u5668\u3002\u5173\u7cfb\u89e3\u7801\u5668\u8fde\u63a5\u5b9e\u4f53,\u6700\u540e\u8f93\u51fa\u53ef\u7f16\u8f91\u7684\u6f14\u793a\u6587\u7a3f\u56fe\u3002", fontname="CJK", fontsize=10) + page.insert_textbox((45, 340, 555, 400), "\u56fe 1:\u8bba\u6587\u7f16\u7801\u5668\u3001\u5173\u7cfb\u89e3\u7801\u5668\u4e0e\u53ef\u7f16\u8f91\u8f93\u51fa\u7684\u7cfb\u7edf\u6846\u67b6\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 455), "3 \u5b9e\u9a8c", fontname="CJK", fontsize=13) + page.insert_textbox((45, 480, 555, 545), "\u5b9e\u9a8c\u8bc4\u4f30\u5b9e\u4f53\u53ec\u56de\u7387\u3001\u5173\u7cfb\u53ec\u56de\u7387\u548c\u5904\u7406\u65f6\u95f4\u3002", fontname="CJK", fontsize=10) + page.insert_text((45, 600), "4 \u7ed3\u8bba", fontname="CJK", fontsize=13) + page.insert_textbox((45, 625, 555, 690), "\u8be5\u65b9\u6cd5\u80fd\u591f\u4fdd\u6301\u8bba\u6587\u672f\u8bed\u3001\u8bc1\u636e\u9875\u7801\u548c\u6a21\u5757\u5173\u7cfb\u3002", fontname="CJK", fontsize=10) + return _save_document(document, target) + + +def _hyphenated_native_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 75, 555, 145), "We introduce a trans-\nformer encoder for evidence-grounded figure generation.", fontsize=10) + page.insert_text((45, 185), "2 Method", fontname="hebo", fontsize=12) + page.insert_textbox((45, 210, 555, 300), "The trans-\nformer encoder passes the input representation to a relation de-\ncoder and produces an editable output.", fontsize=10) + return _save_document(document, target) + + def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: if "page_002" not in image_path.name: return [] @@ -200,6 +231,8 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> rotated = _rotated_fixture(fixtures / "rotated_native_page.pdf") mixed = _mixed_scan_fixture(fixtures / "mixed_scan.pdf") repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") + chinese = _native_chinese_fixture(fixtures / "native_chinese.pdf") + hyphenated = _hyphenated_native_fixture(fixtures / "hyphenated_native.pdf") results = [ _run_case( @@ -260,6 +293,27 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> _check("scientific_content_preserved", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder evidence"), ], ), + _run_case( + "native_chinese", + chinese, + root, + lambda parsed: [ + _check("unicode_preserved", "\u6587\u6863\u7f16\u7801\u5668" in parsed["pages"][0]["text"], parsed["pages"][0]["text"], "contains document encoder in Chinese"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Chinese abstract/method/experiments/conclusion"), + _check("figure_caption", any("\u8bba\u6587\u7f16\u7801\u5668" in str(item.get("caption") or "") for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "Chinese Figure 1 caption"), + _check("no_mojibake", parsed["extraction_report"].get("mojibake_rate") == 0.0, parsed["extraction_report"].get("mojibake_rate"), 0.0), + ], + ), + _run_case( + "hyphenated_native", + hyphenated, + root, + lambda parsed: [ + _check("transformer_repaired", sum("transformer encoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])) >= 2, [item.get("text") for item in parsed.get("evidence", [])], "two transformer encoder mentions"), + _check("decoder_repaired", any("relation decoder" in item.get("text", "").casefold() for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "relation decoder"), + _check("repair_count", parsed["extraction_report"].get("native_hyphenation_repair_count") == 3, parsed["extraction_report"].get("native_hyphenation_repair_count"), 3), + ], + ), ] if ocr_engine != "off": diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index a854bdb..2bfb972 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -107,6 +107,15 @@ def _lexical_units(text: str) -> list[str]: return units +def _cjk_character_count(text: str) -> int: + return sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in str(text or "") + ) + + def _token_overlap_agreement(left: str, right: str) -> float | None: left_tokens = _lexical_units(left) right_tokens = _lexical_units(right) @@ -326,6 +335,45 @@ def _pymupdf_line_blocks(page: Any) -> list[dict[str, Any]]: return records +def _merge_native_hyphenated_lines(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]: + merged: list[dict[str, Any]] = [] + repairs = 0 + for record in records: + current = dict(record) + if not merged: + merged.append(current) + continue + previous = merged[-1] + previous_text = str(previous.get("text") or "") + current_text = str(current.get("text") or "") + match_left = re.search(r"([A-Za-z]{2,})-$", previous_text) + match_right = re.match(r"([A-Za-z]{2,})", current_text) + same_parent = previous.get("parent_block") is not None and previous.get("parent_block") == current.get("parent_block") + if not same_parent or not match_left or not match_right: + merged.append(current) + continue + combined_word = f"{match_left.group(1)}{match_right.group(1)}".casefold() + separator = "" if combined_word in OCR_PROTECTED_WORDS or len(match_left.group(1)) <= 3 else "-" + joined_text = f"{previous_text[:-1]}{separator}{current_text.lstrip()}" + previous_bbox = previous.get("bbox") or [0, 0, 0, 0] + current_bbox = current.get("bbox") or previous_bbox + previous.update({ + "text": _clean(joined_text), + "bbox": [ + round(min(float(previous_bbox[0]), float(current_bbox[0])), 3), + round(min(float(previous_bbox[1]), float(current_bbox[1])), 3), + round(max(float(previous_bbox[2]), float(current_bbox[2])), 3), + round(max(float(previous_bbox[3]), float(current_bbox[3])), 3), + ], + "confidence": min(float(previous.get("confidence") or 1.0), float(current.get("confidence") or 1.0)), + "font_size": max(float(previous.get("font_size") or 0.0), float(current.get("font_size") or 0.0)) or None, + "font_bold": bool(previous.get("font_bold")) and bool(current.get("font_bold")), + "native_hyphenation_repairs": int(previous.get("native_hyphenation_repairs") or 0) + 1, + }) + repairs += 1 + return merged, repairs + + def _poppler_pages(path: Path, timeout: int = 30) -> tuple[list[str], str | None]: executable = shutil.which("pdftotext") if not executable: @@ -563,7 +611,7 @@ def _remove_repeated_margin_noise(pages: list[dict[str, Any]]) -> int: for block in page.get("blocks", []): bbox = block.get("bbox") text = str(block.get("text") or "").strip() - if not isinstance(bbox, (list, tuple)) or len(bbox) != 4 or not text or len(text) > 140 or len(_lexical_units(text)) > 12: + if not isinstance(bbox, (list, tuple)) or len(bbox) != 4 or not text or len(text) > 140 or len(_lexical_units(text)) > 12 or _cjk_character_count(text) > 36: continue top_region = float(bbox[3]) <= height * 0.12 bottom_region = float(bbox[1]) >= height * 0.88 @@ -886,13 +934,14 @@ def _read_pdf_pages( try: for index, page in enumerate(document, 1): native_width = float(page.mediabox.width) if int(page.rotation or 0) else float(page.rect.width) - raw_blocks = _pymupdf_line_blocks(page) + raw_blocks, hyphenation_repairs = _merge_native_hyphenated_lines(_pymupdf_line_blocks(page)) ordered, order_confidence = _reading_order(raw_blocks, native_width) if len(_normalized_compare_text(" ".join(item["text"] for item in ordered))) < 80: - fallback_blocks = _pdfplumber_blocks(path, index) + fallback_blocks, fallback_hyphenation_repairs = _merge_native_hyphenated_lines(_pdfplumber_blocks(path, index)) fallback_ordered, fallback_confidence = _reading_order(fallback_blocks, native_width) if len(_normalized_compare_text(" ".join(item["text"] for item in fallback_ordered))) > len(_normalized_compare_text(" ".join(item["text"] for item in ordered))): ordered, order_confidence = fallback_ordered, min(fallback_confidence, 0.9) + hyphenation_repairs = fallback_hyphenation_repairs ordered = _rotate_blocks_to_page(ordered, page) for block_index, block in enumerate(ordered, 1): block["id"] = f"P{index:03d}_B{block_index:03d}" @@ -920,6 +969,7 @@ def _read_pdf_pages( "poppler_agreement": agreement, "poppler_order_agreement": order_agreement, "reading_order_confidence": order_confidence, + "native_hyphenation_repair_count": hyphenation_repairs, "used_ocr": False, }) used_chars += len(page_text) @@ -961,6 +1011,7 @@ def consume_ocr_result(page_number: int, result: tuple[list[dict[str, Any]], str "used_ocr": True, "ocr_engine": used_engine, "ocr_margin_noise_removed": int(ocr_diagnostics.get("margin_noise_removed") or 0), + "native_hyphenation_repair_count": 0, }) ocr_pages.append(page_number) @@ -1043,6 +1094,7 @@ def isolated_ocr(page_number: int) -> tuple[tuple[list[dict[str, Any]], str, str "ocr_worker_count": ocr_worker_count, "ocr_rescue_min_remaining_seconds": float(ocr_rescue_min_remaining), "repeated_margin_noise_removed_count": repeated_margin_noise_removed, + "native_hyphenation_repair_count": sum(int(page.get("native_hyphenation_repair_count") or 0) for page in pages), "warnings": warnings, "poppler_available": bool(poppler_text), "elapsed_seconds": elapsed, @@ -1422,6 +1474,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "ocr_worker_count": int(details.get("ocr_worker_count") or 1), "ocr_rescue_min_remaining_seconds": float(details.get("ocr_rescue_min_remaining_seconds") or 45.0), "repeated_margin_noise_removed_count": int(details.get("repeated_margin_noise_removed_count") or 0), + "native_hyphenation_repair_count": int(details.get("native_hyphenation_repair_count") or 0), "ocr_margin_noise_removed_count": sum(int(item.get("margin_noise_removed") or 0) for item in details.get("ocr_page_durations", [])), "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), "mean_ocr_confidence": mean_ocr_confidence, diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index f11875a..db832ac 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 24 +DOCUMENT_CACHE_VERSION = 25 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 156d63f..1696295 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -2,6 +2,7 @@ import json import re +import unicodedata from typing import Any from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available @@ -172,6 +173,7 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in 1. Select the most information-rich method/architecture/framework figure caption from the candidates below. Figure 1 is not automatically best; prefer a caption that explicitly names components and directed operations. 2. Extract every separately named input, component, intermediate representation, conditioning signal, training-only objective, inference-only step, and output needed to reproduce that overview. 3. Use exact short paper terms as visible names. Do not replace "image encoder" with a prose description such as "network that extracts image features". + A visible name must be copied verbatim from one of its cited evidence blocks. Preserve the paper's original language and writing system; never translate a Chinese, Japanese, Korean, or other non-English term because the user preference says English. If no concise source phrase exists, use the shortest verbatim source-language phrase. Relation labels must follow the same rule or be left blank. 4. Split compound outputs and branches into separate entities. If a component predicts class and box, create separate class and box outputs. If two encoders produce two embeddings, create both embeddings. 5. Preserve direction. Every relation endpoint must be a declared entity id. Include input boundary edges and final output edges. 6. Separate training and inference. Matching, losses, supervision, optimization, and feedback belong in training_flow or training-only entities; test-time classification, decoding, retrieval, rendering, or deployment belong in inference_flow. @@ -225,7 +227,7 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in Extraction scope (do not claim coverage beyond this scope): {json.dumps({key: parsed.get('extraction_report', {}).get(key) for key in ('pdf_type', 'semantic_scope', 'readable_page_ratio', 'ocr_pages', 'warnings')}, ensure_ascii=False, indent=2)} -User preferences relevant to wording only: +User preferences relevant to explanatory wording only (never translate scientific labels away from the paper's source language): {json.dumps({key: preferences.get(key) for key in ('language', 'must_show', 'must_not_show')}, ensure_ascii=False, indent=2)} Prioritized paper evidence: @@ -570,10 +572,40 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = def validate_plan_grounding(plan: dict, parsed: dict) -> dict: - valid_evidence = {item["id"] for item in parsed.get("evidence", [])} + evidence_items = [item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id")] + valid_evidence = {item["id"] for item in evidence_items} + evidence_text = {str(item["id"]): str(item.get("text") or "") for item in evidence_items} errors: list[str] = [] warnings: list[str] = [] spec = plan.get("figure_specification", {}) + + def normalized_term(value: str) -> str: + return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", value).casefold()).strip() + + def script_counts(value: str) -> tuple[int, int]: + cjk = sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in value + ) + latin = sum(("a" <= char.casefold() <= "z") for char in value) + return cjk, latin + + def validate_visible_label(location: str, label: str, evidence_ids: list[str], *, fallback_text: str = "") -> None: + value = str(label or "").strip() + if not value: + return + source = " ".join(evidence_text.get(str(evidence_id), "") for evidence_id in evidence_ids).strip() or fallback_text + if not source or normalized_term(value) in normalized_term(source): + return + label_cjk, label_latin = script_counts(value) + source_cjk, source_latin = script_counts(source) + changed_from_cjk = source_cjk >= 2 and label_cjk == 0 and label_latin >= 2 + changed_from_latin = source_latin >= 4 and source_cjk == 0 and label_cjk >= 2 and label_latin == 0 + if changed_from_cjk or changed_from_latin: + errors.append(f"{location} visible label changes the source writing system and is not verbatim evidence: {value}") + modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] if not modules: errors.append("figure_specification.modules is empty") @@ -595,6 +627,7 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: invalid = [item for item in evidence_ids if item not in valid_evidence] if invalid: errors.append(f"module {module_id or index + 1} references unknown evidence ids: {invalid}") + validate_visible_label(f"module {module_id or index + 1}", str(module.get("name") or module.get("visible_label") or ""), evidence_ids) endpoint_ids = set(module_ids) for field in ("inputs", "outputs", "innovations"): @@ -610,6 +643,7 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: invalid = [value for value in evidence_ids if value not in valid_evidence] if invalid: errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") + validate_visible_label(f"{field}[{index}]", str(item.get("name") or item.get("visible_label") or item.get("text") or ""), evidence_ids) relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] for index, relation in enumerate(relations): @@ -626,6 +660,12 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: invalid = [item for item in evidence_ids if item not in valid_evidence] if invalid: errors.append(f"relation {source} -> {target} references unknown evidence ids: {invalid}") + validate_visible_label(f"relation {source} -> {target}", str(relation.get("label") or ""), evidence_ids) + + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + all_evidence_text = " ".join(evidence_text.values()) + for source_term, visible_label in terminology.items(): + validate_visible_label(f"terminology[{source_term}]", str(visible_label or ""), [], fallback_text=all_evidence_text) for field in ("must_show", "innovations"): items = spec.get(field) if isinstance(spec.get(field), list) else [] diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 214ef45..c44a819 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -120,6 +120,36 @@ def _normalized_label(value: str) -> str: return "".join(char for char in value.casefold() if char.isalnum()) +def _script_counts(value: str) -> tuple[int, int]: + cjk = sum( + "\u3400" <= char <= "\u9fff" + or "\u3040" <= char <= "\u30ff" + or "\uac00" <= char <= "\ud7af" + for char in str(value or "") + ) + latin = sum(("a" <= char.casefold() <= "z") for char in str(value or "")) + return cjk, latin + + +def _repair_cross_script_terminology(spec: dict[str, Any], parsed: dict[str, Any]) -> None: + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + if not terminology: + return + corpus = " ".join(str(item.get("text") or "") for item in parsed.get("evidence", []) if isinstance(item, dict)) + normalized_corpus = re.sub(r"\s+", " ", corpus.casefold()) + repaired: dict[str, str] = {} + for source_term, visible_label in terminology.items(): + source = str(source_term or "").strip() + visible = str(visible_label or "").strip() + source_present = bool(source and re.sub(r"\s+", " ", source.casefold()) in normalized_corpus) + visible_present = bool(visible and re.sub(r"\s+", " ", visible.casefold()) in normalized_corpus) + source_cjk, source_latin = _script_counts(source) + visible_cjk, visible_latin = _script_counts(visible) + changed_script = (source_cjk >= 2 and visible_cjk == 0 and visible_latin >= 2) or (source_latin >= 4 and source_cjk == 0 and visible_cjk >= 2 and visible_latin == 0) + repaired[source] = source if source_present and not visible_present and changed_script else visible + spec["terminology"] = repaired + + def _stable_id(prefix: str, value: str, index: int) -> str: slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", value.casefold()).split() if part)[:48] return f"{prefix}_{slug or index}" @@ -538,6 +568,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: allowed_topologies = {"linear", "branch", "feedback", "multimodal", "dense_multiframe", "unknown"} if str(spec.get("topology") or "") not in allowed_topologies: spec["topology"] = _infer_topology(spec) + _repair_cross_script_terminology(spec, parsed) labels = [] terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} labels.extend(str(value).strip() for value in terminology.values() if str(value).strip()) @@ -703,7 +734,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 27, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 28, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -942,6 +973,7 @@ def prepare_paper_figure_contract( "ocr_worker_count": parsed["extraction_report"].get("ocr_worker_count", 1), "ocr_rescue_min_remaining_seconds": parsed["extraction_report"].get("ocr_rescue_min_remaining_seconds", 45.0), "repeated_margin_noise_removed_count": parsed["extraction_report"].get("repeated_margin_noise_removed_count", 0), + "native_hyphenation_repair_count": parsed["extraction_report"].get("native_hyphenation_repair_count", 0), "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), }, diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 22b6431..21e0e87 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -173,6 +173,7 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("figure_caption_total", result["aggregate"]) self.assertIn("max_ocr_worker_count", result["aggregate"]) self.assertIn("repeated_margin_noise_removed_total", result["aggregate"]) + self.assertIn("native_hyphenation_repair_total", result["aggregate"]) self.assertIn("ocr_spacing_repair_total", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) @@ -191,10 +192,12 @@ def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr( result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") self.assertTrue(result["ok"]) - self.assertEqual(result["aggregate"]["case_count"], 5) - self.assertEqual(result["aggregate"]["passed_case_count"], 5) + self.assertEqual(result["aggregate"]["case_count"], 7) + self.assertEqual(result["aggregate"]["passed_case_count"], 7) self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "repeated_margin_noise.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "native_chinese.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "hyphenated_native.pdf").exists()) self.assertTrue((Path(tmp) / "mixed_scan_fixture_adapter" / "document_model.json").exists()) self.assertTrue((Path(tmp) / "pdf_extraction_stress_report.json").exists()) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 1fb9640..2f22fa6 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -74,6 +74,85 @@ def test_contract_normalizes_scalar_claim_and_grounds_it_from_evidence(self): self.assertEqual(spec["research_problem"]["evidence_ids"], ["E0001"]) self.assertTrue(validation["ok"]) + def test_validation_rejects_cross_script_translation_of_visible_labels(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "输入论文首先进入文档编码器,关系解码器连接实体,最后输出可编辑框架图。", "confidence": 1.0}, + ] + } + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "Input document", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "Document encoder", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "可编辑框架图", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "label": "进入", "evidence_ids": ["E0001"]}, + {"source": "encoder", "target": "output", "type": "prediction", "label": "produces", "evidence_ids": ["E0001"]}, + ], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertFalse(validation["ok"]) + self.assertTrue(any("Input document" in item for item in validation["errors"])) + self.assertTrue(any("Document encoder" in item for item in validation["errors"])) + self.assertTrue(any("produces" in item for item in validation["errors"])) + + def test_validation_accepts_verbatim_non_english_visible_labels(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "输入论文首先进入文档编码器,关系解码器连接实体,最后输出可编辑框架图。", "confidence": 1.0}, + ] + } + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "输入论文", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "文档编码器", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "可编辑框架图", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "label": "进入", "evidence_ids": ["E0001"]}, + {"source": "encoder", "target": "output", "type": "prediction", "label": "", "evidence_ids": ["E0001"]}, + ], + "terminology": {"文档编码器": "文档编码器"}, + } + } + + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_contract_repairs_cross_script_terminology_to_verbatim_source_key(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "输入论文首先进入文档编码器,关系解码器连接实体,最后输出可编辑框架图。", "confidence": 1.0}, + ] + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "输入论文", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "文档编码器", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "可编辑框架图", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [{"source": "input", "target": "encoder", "type": "data_flow", "label": "进入", "evidence_ids": ["E0001"]}], + "terminology": {"文档编码器": "Document encoder"}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["terminology"]["文档编码器"], "文档编码器") + self.assertIn("文档编码器", spec["required_labels"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + def test_contract_grounds_declared_multword_entity_by_exact_paper_term(self): parsed = { "page_count": 6, From bc7ea61062c95fbf5be99fd69c19d871c5a40f15 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 16:05:03 +0800 Subject: [PATCH 45/74] Recover numbered CJK sections from OCR --- docs/paper-to-image.md | 2 ++ rfs/paper_to_image/analyzer.py | 3 +++ rfs/paper_to_image/document_cache.py | 2 +- tests/unit/test_paper_analyzer.py | 31 ++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index d62ced1..28153b1 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -51,6 +51,8 @@ Fast VLM planning preserves the paper's original writing system for every visibl Native PDF lines split by publisher hyphenation are joined inside the same source text block before evidence IDs are created. Known scientific words such as `trans-` + `former` and `de-` + `coder` lose the layout-only hyphen, while unknown long compound parts retain a literal hyphen. The extraction report records `native_hyphenation_repair_count`. +OCR normalization also repairs numbered CJK section boundaries such as `2方法`, `3实验`, and `4结论` before heading classification. This keeps multilingual scan coverage and the actual section index consistent instead of merely finding section keywords inside undifferentiated page text. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 2bfb972..8a8ab5e 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -63,6 +63,9 @@ def _repair_ocr_spacing(text: str, splitter: Callable[[str], list[str]] | None = section_terms = "abstract|introduction|background|method|methods|approach|experiments|results|conclusion|discussion|references|appendix" value, count = re.subn(rf"(? bool: diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index d40ab0b..edfaf67 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -219,6 +219,37 @@ def test_ocr_spacing_repair_preserves_cjk_acronyms_and_known_compounds(self): self.assertEqual(repaired, source) + def test_ocr_spacing_repair_restores_numbered_chinese_section_heading(self): + repaired, count = _repair_ocr_spacing("2\u65b9\u6cd5", splitter=lambda token: [token]) + + self.assertEqual(repaired, "2 \u65b9\u6cd5") + self.assertEqual(count, 1) + + def test_scanned_chinese_numbered_sections_form_real_section_boundaries(self): + path = self._multipage_pdf([[]]) + + def adapter(_image_path, _lang): + lines = [ + ("\u6458\u8981", 30), + ("\u672c\u6587\u63d0\u51fa\u6587\u6863\u7f16\u7801\u5668\u548c\u5173\u7cfb\u89e3\u7801\u5668,\u751f\u6210\u53ef\u7f16\u8f91\u6846\u67b6\u56fe\u3002", 90), + ("2\u65b9\u6cd5", 190), + ("\u8f93\u5165\u8bba\u6587\u8fdb\u5165\u6587\u6863\u7f16\u7801\u5668,\u7136\u540e\u7531\u5173\u7cfb\u89e3\u7801\u5668\u751f\u6210\u8f93\u51fa\u3002", 250), + ("3\u5b9e\u9a8c", 390), + ("\u5b9e\u9a8c\u8bc4\u4f30\u5b9e\u4f53\u53ec\u56de\u7387\u548c\u5173\u7cfb\u53ec\u56de\u7387\u3002", 450), + ("4\u7ed3\u8bba", 590), + ("\u8be5\u65b9\u6cd5\u4fdd\u6301\u8bc1\u636e\u9875\u7801\u548c\u6a21\u5757\u5173\u7cfb\u3002", 650), + ] + return [ + {"text": text, "confidence": 0.99, "quad": [[40, top], [900, top], [900, top + 45], [40, top + 45]]} + for text, top in lines + ] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter) + + self.assertEqual(parsed["headings"], ["\u6458\u8981", "\u65b9\u6cd5", "\u5b9e\u9a8c", "\u7ed3\u8bba"]) + self.assertEqual(parsed["extraction_report"]["section_count"], 4) + self.assertGreaterEqual(parsed["extraction_report"]["ocr_spacing_repair_count"], 3) + def test_low_text_page_uses_local_ocr_adapter(self): path = self._pdf(lambda _page: None) From 020414e9ddb815cb41157832450cbce1f958a7da Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 16:49:21 +0800 Subject: [PATCH 46/74] Enforce evidence for every visible contract entity --- docs/paper-to-image.md | 2 + rfs/paper_to_image/planner.py | 2 +- rfs/paper_to_image/preparation.py | 37 ++++++++++++++- tests/unit/test_paper_contract.py | 78 +++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 28153b1..3299dcb 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -53,6 +53,8 @@ Native PDF lines split by publisher hyphenation are joined inside the same sourc OCR normalization also repairs numbered CJK section boundaries such as `2方法`, `3实验`, and `4结论` before heading classification. This keeps multilingual scan coverage and the actual section index consistent instead of merely finding section keywords inside undifferentiated page text. +Production validation treats visible inputs, outputs, and innovations without evidence IDs as hard errors. Before validation, exact entity labels may be grounded deterministically to matching evidence; innovation labels are grounded only when the same evidence also contains an explicit novelty cue such as `we propose`, `we introduce`, or `本文提出`. Otherwise the result remains engineering-only instead of silently presenting an unsupported contribution. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 1696295..ff83535 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -639,7 +639,7 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * endpoint_ids.add(endpoint_id) evidence_ids = item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [] if not evidence_ids: - warnings.append(f"{field}[{index}] has no evidence_ids") + errors.append(f"{field}[{index}] has no evidence_ids") invalid = [value for value in evidence_ids if value not in valid_evidence] if invalid: errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index c44a819..6445bb1 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -150,6 +150,40 @@ def _repair_cross_script_terminology(spec: dict[str, Any], parsed: dict[str, Any spec["terminology"] = repaired +def _ground_exact_visible_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id") and item.get("text")] + novelty_pattern = re.compile(r"\b(?:we\s+(?:introduce|propose|present|develop)|novel|new\s+(?:method|model|system|framework)|contribution)\b|\u672c\u6587\u63d0\u51fa|\u6211\u4eec\u63d0\u51fa|\u9996\u6b21|\u521b\u65b0", re.IGNORECASE) + overview_pattern = re.compile(r"overview|framework|architecture|pipeline|procedure|pre-training|fine-tuning|system|model|\u6846\u67b6|\u67b6\u6784|\u7cfb\u7edf", re.IGNORECASE) + overview_pages = { + int(item.get("page") or 0) + for item in parsed.get("document_index", {}).get("figures", []) + if isinstance(item, dict) and overview_pattern.search(str(item.get("caption") or "")) + } + grounded: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for index, item in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): + if not isinstance(item, dict) or item.get("evidence_ids"): + continue + label = _item_label(item) + compact = _normalized_label(label) + if not compact: + continue + matches = [] + for record in evidence: + record_compact = _normalized_label(str(record.get("text") or "")) + short_diagram_label = len(compact) < 3 and int(record.get("page") or 0) in overview_pages and record_compact == compact + if not short_diagram_label and (len(compact) < 3 or compact not in record_compact): + continue + if field == "innovations" and not novelty_pattern.search(str(record.get("text") or "")): + continue + matches.append(str(record["id"])) + if not matches: + continue + item["evidence_ids"] = list(dict.fromkeys(matches))[:3] + grounded.append(f"{field}[{index}]") + return grounded + + def _stable_id(prefix: str, value: str, index: int) -> str: slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", value.casefold()).split() if part)[:48] return f"{prefix}_{slug or index}" @@ -568,6 +602,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: allowed_topologies = {"linear", "branch", "feedback", "multimodal", "dense_multiframe", "unknown"} if str(spec.get("topology") or "") not in allowed_topologies: spec["topology"] = _infer_topology(spec) + completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) _repair_cross_script_terminology(spec, parsed) labels = [] terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} @@ -734,7 +769,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 28, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 29, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 2f22fa6..97a90df 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -153,6 +153,84 @@ def test_contract_repairs_cross_script_terminology_to_verbatim_source_key(self): self.assertIn("文档编码器", spec["required_labels"]) self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + def test_contract_grounds_exact_innovation_only_from_novelty_evidence(self): + parsed = { + "evidence": [ + {"id": "E0001", "text": "本文提出论文编码器和关系解码器,生成可编辑框架图。", "section_hint": "摘要", "confidence": 1.0}, + {"id": "E0002", "text": "A baseline also contains a relation decoder.", "section_hint": "Related Work", "confidence": 1.0}, + ] + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [{"id": "decoder", "name": "关系解码器", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [{"id": "innovation_decoder", "name": "关系解码器", "evidence_ids": []}], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["innovations"][0]["evidence_ids"], ["E0001"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_validation_rejects_visible_input_without_evidence(self): + parsed = {"evidence": [{"id": "E0001", "text": "The encoder produces an output.", "confidence": 1.0}]} + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "Input Image", "evidence_ids": []}], + "modules": [{"id": "encoder", "name": "encoder", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertFalse(validation["ok"]) + self.assertIn("inputs[0] has no evidence_ids", validation["errors"]) + + def test_contract_grounds_short_diagram_labels_only_on_overview_figure_page(self): + parsed = { + "document_index": {"figures": [{"page": 3, "caption": "Figure 1: Overall pre-training and fine-tuning procedures."}]}, + "evidence": [ + {"id": "E0001", "page": 3, "text": "T 1", "confidence": 1.0}, + {"id": "E0002", "page": 3, "text": "C", "confidence": 1.0}, + {"id": "E0003", "page": 8, "text": "C", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [{"id": "encoder", "name": "Encoder", "evidence_ids": ["E0001"]}], + "outputs": [ + {"id": "token_output", "name": "T 1", "evidence_ids": []}, + {"id": "class_output", "name": "C", "evidence_ids": []}, + ], + "innovations": [], + "relations": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["outputs"][0]["evidence_ids"], ["E0001"]) + self.assertEqual(spec["outputs"][1]["evidence_ids"], ["E0002"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + def test_contract_grounds_declared_multword_entity_by_exact_paper_term(self): parsed = { "page_count": 6, From 1add7f260bca32877fb0bde44ef96f7d4c2ab426 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 16:57:44 +0800 Subject: [PATCH 47/74] Handle rotated margins and repeated section titles --- docs/paper-to-image.md | 4 ++ rfs/evaluation/pdf_extraction_benchmark.py | 27 ++++++++++++++ rfs/paper_to_image/analyzer.py | 43 ++++++++++++++++++---- rfs/paper_to_image/document_cache.py | 2 +- tests/unit/test_benchmarking.py | 5 ++- 5 files changed, 70 insertions(+), 11 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 3299dcb..8c4653b 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -55,6 +55,10 @@ OCR normalization also repairs numbered CJK section boundaries such as `2方法` Production validation treats visible inputs, outputs, and innovations without evidence IDs as hard errors. Before validation, exact entity labels may be grounded deterministically to matching evidence; innovation labels are grounded only when the same evidence also contains an explicit novelty cue such as `we propose`, `we introduce`, or `本文提出`. Otherwise the result remains engineering-only instead of silently presenting an unsupported contribution. +Repeated-margin filtering interprets page edges in the PDF's semantic orientation. On a 90-degree page, the original header is detected at the displayed right edge and the original footer at the left edge; corresponding mappings are applied for 180 and 270 degrees. Mixed-orientation papers therefore use one header/footer signature space without leaking rotated publisher furniture into evidence. + +Section headings are deduplicated only within their concrete page/block occurrence, not globally by title text. Repeated `Methods`, `Experiments`, or appendix headings therefore create new evidence boundaries after publisher headers have been removed. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index 15712ee..67d20a9 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -138,6 +138,20 @@ def _hyphenated_native_fixture(target: Path) -> Path: return _save_document(document, target) +def _rotated_repeated_margin_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + for page_number in range(1, 5): + page = document.new_page(width=600, height=800) + page.insert_text((45, 25), "Rotated Conference Header 2026", fontsize=8) + page.insert_text((45, 785), f"Rotated Paper 42 | Page {page_number}", fontsize=8) + page.insert_text((45, 75), f"{page_number} Method Stage", fontname="hebo", fontsize=12) + page.insert_textbox((45, 105, 555, 220), f"Stage {page_number} preserves rotated scientific evidence and semantic reading order.", fontsize=10) + page.set_rotation(90) + return _save_document(document, target) + + def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: if "page_002" not in image_path.name: return [] @@ -233,6 +247,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") chinese = _native_chinese_fixture(fixtures / "native_chinese.pdf") hyphenated = _hyphenated_native_fixture(fixtures / "hyphenated_native.pdf") + rotated_margin = _rotated_repeated_margin_fixture(fixtures / "rotated_repeated_margin.pdf") results = [ _run_case( @@ -314,6 +329,18 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> _check("repair_count", parsed["extraction_report"].get("native_hyphenation_repair_count") == 3, parsed["extraction_report"].get("native_hyphenation_repair_count"), 3), ], ), + _run_case( + "rotated_repeated_margin", + rotated_margin, + root, + lambda parsed: [ + _check("all_pages_rotated", parsed["extraction_report"].get("rotated_pages") == [1, 2, 3, 4], parsed["extraction_report"].get("rotated_pages"), [1, 2, 3, 4]), + _check("rotated_noise_removed", parsed["extraction_report"].get("repeated_margin_noise_removed_count") == 8, parsed["extraction_report"].get("repeated_margin_noise_removed_count"), 8), + _check("repeated_section_boundaries", parsed["extraction_report"].get("section_count") == 4, parsed["extraction_report"].get("section_count"), 4), + _check("header_absent", all("Rotated Conference Header" not in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no rotated header evidence"), + _check("body_preserved", all(any(f"Stage {page_number} preserves" in item.get("text", "") for item in parsed.get("evidence", [])) for page_number in range(1, 5)), [item.get("text") for item in parsed.get("evidence", [])], "all rotated stage evidence"), + ], + ), ] if ocr_engine != "off": diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 8a8ab5e..cc29df5 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -601,6 +601,35 @@ def _repeated_margin_signature(text: str) -> str: return value +def _semantic_margin_region(page: dict[str, Any], bbox: list[float] | tuple[float, ...]) -> str | None: + width = float(page.get("width") or 0.0) + height = float(page.get("height") or 0.0) + if width <= 0 or height <= 0: + return None + rotation = int(page.get("rotation") or 0) % 360 + if rotation == 90: + if float(bbox[0]) >= width * 0.88: + return "header" + if float(bbox[2]) <= width * 0.12: + return "footer" + elif rotation == 180: + if float(bbox[1]) >= height * 0.88: + return "header" + if float(bbox[3]) <= height * 0.12: + return "footer" + elif rotation == 270: + if float(bbox[2]) <= width * 0.12: + return "header" + if float(bbox[0]) >= width * 0.88: + return "footer" + else: + if float(bbox[3]) <= height * 0.12: + return "header" + if float(bbox[1]) >= height * 0.88: + return "footer" + return None + + def _remove_repeated_margin_noise(pages: list[dict[str, Any]]) -> int: if len(pages) < 3: return 0 @@ -608,22 +637,20 @@ def _remove_repeated_margin_noise(pages: list[dict[str, Any]]) -> int: candidates: dict[tuple[int, str], tuple[str, str]] = {} for page in pages: page_number = int(page.get("page") or 0) - height = float(page.get("height") or 0.0) - if height <= 0: - continue for block in page.get("blocks", []): bbox = block.get("bbox") text = str(block.get("text") or "").strip() + if str(block.get("kind") or "") == "heading" or bool(block.get("font_bold")): + continue if not isinstance(bbox, (list, tuple)) or len(bbox) != 4 or not text or len(text) > 140 or len(_lexical_units(text)) > 12 or _cjk_character_count(text) > 36: continue - top_region = float(bbox[3]) <= height * 0.12 - bottom_region = float(bbox[1]) >= height * 0.88 - if not top_region and not bottom_region: + region = _semantic_margin_region(page, bbox) + if not region: continue signature = _repeated_margin_signature(text) if not signature: continue - key = ("top" if top_region else "bottom", signature) + key = (region, signature) occurrences.setdefault(key, set()).add(page_number) candidates[(page_number, str(block.get("id") or ""))] = key @@ -1179,7 +1206,7 @@ def _heading_candidates(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: if not (explicit or numbered or known or typographic): continue normalized = numbered.group(2).strip() if numbered else line.strip().rstrip(".:") - key = normalized.casefold() + key = (normalized.casefold(), int(page.get("page") or 0), str(block.get("id") or ""), line_index) if normalized and key not in seen: seen.add(key) headings.append({ diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 18c250a..c765c7b 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 26 +DOCUMENT_CACHE_VERSION = 27 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 21e0e87..e9e6bce 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -192,12 +192,13 @@ def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr( result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") self.assertTrue(result["ok"]) - self.assertEqual(result["aggregate"]["case_count"], 7) - self.assertEqual(result["aggregate"]["passed_case_count"], 7) + self.assertEqual(result["aggregate"]["case_count"], 8) + self.assertEqual(result["aggregate"]["passed_case_count"], 8) self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "repeated_margin_noise.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "native_chinese.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "hyphenated_native.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "rotated_repeated_margin.pdf").exists()) self.assertTrue((Path(tmp) / "mixed_scan_fixture_adapter" / "document_model.json").exists()) self.assertTrue((Path(tmp) / "pdf_extraction_stress_report.json").exists()) From 4dbe172de0be8759a6ba3f0814275a2efb1b2621 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 17:03:33 +0800 Subject: [PATCH 48/74] Fail cleanly on encrypted and damaged PDFs --- docs/paper-to-image.md | 2 ++ rfs/paper_to_image/analyzer.py | 15 ++++++++++++++ tests/unit/test_paper_analyzer.py | 33 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 8c4653b..434305c 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -59,6 +59,8 @@ Repeated-margin filtering interprets page edges in the PDF's semantic orientatio Section headings are deduplicated only within their concrete page/block occurrence, not globally by title text. Repeated `Methods`, `Experiments`, or appendix headings therefore create new evidence boundaries after publisher headers have been removed. +PDF inputs receive a lightweight container preflight before pypdf, Poppler, or PyMuPDF extraction. Missing `%PDF-` headers and missing tail `%%EOF` markers return stable `ValueError` messages immediately; encrypted documents retain the explicit password-required error. This keeps `--json` failures concise and prevents damaged files from entering either cache. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index cc29df5..417bb67 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -437,6 +437,20 @@ def _pdf_metadata(path: Path) -> dict[str, Any]: return {"page_count": None, "encrypted": None, "metadata": {}, "warning": "pypdf unavailable"} +def _validate_pdf_container(path: Path) -> None: + size = path.stat().st_size + if size < 8: + raise ValueError("PDF is empty or too small to contain a valid document") + with path.open("rb") as handle: + header = handle.read(min(1024, size)) + handle.seek(max(0, size - 4096)) + tail = handle.read() + if b"%PDF-" not in header: + raise ValueError("File does not contain a valid PDF header") + if b"%%EOF" not in tail: + raise ValueError("PDF appears truncated because the EOF marker is missing") + + def _ocr_records(image_path: Path, engine: str, lang: str, adapter: Callable | None, rapidocr_threads: int = 1) -> tuple[list[dict[str, Any]], str, dict[str, Any]]: if adapter: return list(adapter(image_path, lang) or []), "adapter", {} @@ -1543,6 +1557,7 @@ def parse_paper( raise FileNotFoundError(f"Paper does not exist: {source}") source_hash = hashlib.sha256(source.read_bytes()).hexdigest() if source.suffix.lower() == ".pdf": + _validate_pdf_container(source) pages, details = _read_pdf_pages(source, max_chars, deadline_at, ocr_engine, ocr_lang, ocr_adapter, max_ocr_pages, ocr_rescue_min_remaining) loader = "structured-pdf" else: diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index edfaf67..ac38b65 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -250,6 +250,39 @@ def adapter(_image_path, _lang): self.assertEqual(parsed["extraction_report"]["section_count"], 4) self.assertGreaterEqual(parsed["extraction_report"]["ocr_spacing_repair_count"], 3) + def test_truncated_pdf_fails_before_parser_noise(self): + path = self._multipage_pdf([["Abstract", "A valid document before truncation."]]) + payload = path.read_bytes() + path.write_bytes(payload[: max(16, len(payload) // 2)]) + + with self.assertRaisesRegex(ValueError, "EOF marker is missing"): + parse_paper(path, ocr_engine="off") + + def test_non_pdf_payload_with_pdf_extension_fails_preflight(self): + root = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True)) + path = root / "not_really.pdf" + path.write_bytes(b"this is not a PDF document") + + with self.assertRaisesRegex(ValueError, "valid PDF header"): + parse_paper(path, ocr_engine="off") + + def test_encrypted_pdf_returns_explicit_password_error(self): + from pypdf import PdfReader, PdfWriter + + source = self._multipage_pdf([["Abstract", "Encrypted scientific content."]]) + target = source.parent / "encrypted.pdf" + reader = PdfReader(str(source)) + writer = PdfWriter() + for page in reader.pages: + writer.add_page(page) + writer.encrypt("secret") + with target.open("wb") as handle: + writer.write(handle) + + with self.assertRaisesRegex(ValueError, "encrypted"): + parse_paper(target, ocr_engine="off") + def test_low_text_page_uses_local_ocr_adapter(self): path = self._pdf(lambda _page: None) From be480ecce1ca074e21e66742c5aef4ffa74bcd27 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 17:10:28 +0800 Subject: [PATCH 49/74] Benchmark real OCR on scanned and skewed columns --- docs/paper-to-image.md | 2 + rfs/evaluation/pdf_extraction_benchmark.py | 47 ++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 434305c..0ab9e3e 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -61,6 +61,8 @@ Section headings are deduplicated only within their concrete page/block occurren PDF inputs receive a lightweight container preflight before pypdf, Poppler, or PyMuPDF extraction. Missing `%PDF-` headers and missing tail `%%EOF` markers return stable `ValueError` messages immediately; encrypted documents retain the explicit password-required error. This keeps `--json` failures concise and prevents damaged files from entering either cache. +When `benchmark pdf-suite` runs with a real OCR engine, it now includes fully rasterized two-column and 2-degree skewed two-column pages in addition to the mixed native/scan fixture. These cases enforce column reconstruction, left-column-first reading order, section recovery, caption recovery, and OCR confidence using the actual runtime engine. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index 67d20a9..bf90211 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import math import time from pathlib import Path @@ -7,6 +8,7 @@ from ..paper_to_image.analyzer import parse_paper from ..utils import ensure_dir, write_json +from .scanned_pdf import rasterize_pdf_as_scan def _save_document(document: Any, target: Path) -> Path: @@ -152,6 +154,25 @@ def _rotated_repeated_margin_fixture(target: Path) -> Path: return _save_document(document, target) +def _skewed_scan_fixture(source: Path, target: Path, angle: float = 2.0) -> Path: + import fitz + from PIL import Image + + source_document = fitz.open(str(source)) + try: + pixmap = source_document[0].get_pixmap(dpi=144, alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))).convert("RGB") + finally: + source_document.close() + rotated = image.rotate(float(angle), resample=Image.Resampling.BICUBIC, expand=False, fillcolor="white") + payload = io.BytesIO() + rotated.save(payload, format="PNG") + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_image(page.rect, stream=payload.getvalue()) + return _save_document(document, target) + + def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: if "page_002" not in image_path.name: return [] @@ -344,6 +365,8 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> ] if ocr_engine != "off": + scanned_two_column = rasterize_pdf_as_scan(native, fixtures / "scanned_two_column.pdf", dpi=144) + skewed_two_column = _skewed_scan_fixture(native, fixtures / "skewed_two_column.pdf") results.append(_run_case( "mixed_scan_runtime_ocr", mixed, @@ -358,6 +381,30 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> ocr_engine=ocr_engine, preview_page=1, )) + results.append(_run_case( + "scanned_two_column_runtime_ocr", + scanned_two_column, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("headings", all(value in parsed.get("headings", []) for value in ("Abstract", "Method", "Conclusion")), parsed.get("headings", []), "Abstract/Method/Conclusion"), + _check("caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "skewed_two_column_runtime_ocr", + skewed_two_column, + root, + lambda parsed: [ + _check("two_columns", parsed["extraction_report"].get("max_column_count") == 2, parsed["extraction_report"].get("max_column_count"), 2), + _check("reading_order", all(parsed["pages"][0]["text"].index(left) < parsed["pages"][0]["text"].index(right) for left, right in (("LEFT_TOP", "LEFT_BOTTOM"), ("LEFT_BOTTOM", "RIGHT_TOP"), ("RIGHT_TOP", "RIGHT_BOTTOM"))), parsed["pages"][0]["text"], "LEFT_TOP < LEFT_BOTTOM < RIGHT_TOP < RIGHT_BOTTOM"), + _check("mean_confidence", float(parsed["extraction_report"].get("mean_ocr_confidence") or 0.0) >= 0.9, parsed["extraction_report"].get("mean_ocr_confidence"), ">=0.9"), + _check("caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + ], + ocr_engine=ocr_engine, + )) elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] ocr_stage_totals = { From 2e96d1b37102300513d4cd906b3d48522cc7dced Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 18:06:32 +0800 Subject: [PATCH 50/74] Add OCR plausibility gates and structured table evidence --- docs/paper-to-image.md | 6 + rfs/evaluation/benchmarking.py | 2 + rfs/evaluation/pdf_extraction_benchmark.py | 84 ++++++++++++ rfs/paper_to_image/analyzer.py | 141 ++++++++++++++++++++- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/preparation.py | 3 + tests/unit/test_benchmarking.py | 7 +- tests/unit/test_paper_analyzer.py | 24 ++++ 8 files changed, 264 insertions(+), 5 deletions(-) diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 0ab9e3e..c088fc9 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -63,6 +63,12 @@ PDF inputs receive a lightweight container preflight before pypdf, Poppler, or P When `benchmark pdf-suite` runs with a real OCR engine, it now includes fully rasterized two-column and 2-degree skewed two-column pages in addition to the mixed native/scan fixture. These cases enforce column reconstruction, left-column-first reading order, section recovery, caption recovery, and OCR confidence using the actual runtime engine. +English OCR pages with at least two explicit English section anchors also receive a lexical plausibility check backed by the bundled word-frequency model. A known-word ratio below 0.72 raises a warning and below 0.50 fails extraction even when the OCR engine reports high confidence. The gate is not applied to CJK or unrecognized non-English Latin documents, preventing an English dictionary from rejecting unrelated languages. + +Table captions now seed coordinate-based table regions. Header cells define column anchors, later cells are assigned by row and nearest column, and the document index stores `columns`, `rows`, `cells`, and a full table bounding box. Individual table cells are excluded from narrative evidence and replaced by one row-major structured table evidence record, preventing OCR detection order from turning `Model | Depth | Accuracy` into misleading prose. + +Table reconstruction is intentionally conservative: a candidate needs a plausible labeled header and at least two data rows. Pure numeric rows and lowercase prose fragments such as `are | obtained` are rejected without changing the original block kinds, avoiding false table structure on captions, equations, and surrounding sentences. + If a deadline ends after at least three high-confidence scan pages have recovered both Abstract and Method-like evidence, the workflow continues with an engineering-grade partial contract instead of returning `extraction_failed`. It remains explicitly marked `sampled_pages_only` and `scientific_scope_complete=false`, is not eligible for production status, and cannot enter the semantic cache. `fast_suite_report.json` records planning recall, forbidden content, document/contract cache hits, provider attempts and retries, failure categories, parser/semantic/total timings, readable-page ratio, evidence-page coverage, evidence character counts, maximum detected column count, multi-column page totals, OCR candidate/scheduled/completed totals, maximum OCR concurrency, and removed OCR margin-noise totals. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 630e183..84c9b66 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -870,6 +870,7 @@ def percentile(values: list[float], fraction: float) -> float: "merged_heading_line_total": sum(int(value) for value in extraction_metric("merged_heading_line_count")), "figure_caption_total": sum(int(value) for value in extraction_metric("figure_caption_count")), "table_caption_total": sum(int(value) for value in extraction_metric("table_caption_count")), + "formula_total": sum(int(value) for value in extraction_metric("formula_count")), "ocr_candidate_page_total": sum(int(value) for value in extraction_metric("ocr_candidate_count")), "ocr_scheduled_page_total": sum(int(value) for value in extraction_metric("ocr_scheduled_count")), "ocr_attempted_page_total": sum(int(value) for value in extraction_metric("ocr_attempted_count")), @@ -880,6 +881,7 @@ def percentile(values: list[float], fraction: float) -> float: "native_hyphenation_repair_total": sum(int(value) for value in extraction_metric("native_hyphenation_repair_count")), "ocr_margin_noise_removed_total": sum(int(value) for value in extraction_metric("ocr_margin_noise_removed_count")), "ocr_spacing_repair_total": sum(int(value) for value in extraction_metric("ocr_spacing_repair_count")), + "mean_ocr_latin_known_word_ratio": _mean(extraction_metric("ocr_latin_known_word_ratio")), "provider_call_count": len(provider_calls), "provider_success_count": sum(bool(item.get("provider", {}).get("success")) for item in provider_calls), "provider_success_rate": round(sum(bool(item.get("provider", {}).get("success")) for item in provider_calls) / len(provider_calls), 4) if provider_calls else None, diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index bf90211..7c03965 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -140,6 +140,31 @@ def _hyphenated_native_fixture(target: Path) -> Path: return _save_document(document, target) +def _formula_table_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 45), "Abstract", fontname="hebo", fontsize=12) + page.insert_textbox((45, 65, 555, 115), "We introduce a calibrated relation encoder and evaluate it with a dense ablation table.", fontsize=10) + page.insert_text((45, 145), "2 Method", fontname="hebo", fontsize=12) + page.insert_text((80, 185), "Q = X W_q K = X W_k V = X W_v", fontname="cour", fontsize=10) + page.insert_text((80, 215), "A = softmax(Q K^T / sqrt(d)) V", fontname="cour", fontsize=10) + page.insert_textbox((45, 245, 555, 295), "The relation encoder sends the calibrated representation to the prediction head.", fontsize=10) + page.insert_text((45, 335), "3 Experiments", fontname="hebo", fontsize=12) + page.insert_textbox((45, 360, 555, 390), "Table 1: Ablation results for encoder depth and relation accuracy.", fontsize=10) + rows = [("Model", "Depth", "Accuracy"), ("Base", "6", "81.2"), ("Large", "12", "84.7"), ("Ours", "18", "87.9")] + top = 420 + for row in rows: + for left, value in zip((65, 260, 390), row): + page.insert_text((left, top), value, fontsize=10) + top += 28 + page.insert_textbox((45, 555, 555, 600), "The ablation shows that calibrated relation encoding improves accuracy.", fontsize=10) + page.insert_text((45, 650), "4 Conclusion", fontname="hebo", fontsize=12) + page.insert_textbox((45, 675, 555, 720), "The method preserves directed evidence relations and calibrated outputs.", fontsize=10) + return _save_document(document, target) + + def _rotated_repeated_margin_fixture(target: Path) -> Path: import fitz @@ -173,6 +198,25 @@ def _skewed_scan_fixture(source: Path, target: Path, angle: float = 2.0) -> Path return _save_document(document, target) +def _degraded_scan_fixture(source: Path, target: Path) -> Path: + import fitz + from PIL import Image, ImageFilter + + source_document = fitz.open(str(source)) + try: + pixmap = source_document[0].get_pixmap(dpi=120, alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))).convert("L") + finally: + source_document.close() + degraded = image.resize((420, 560), Image.Resampling.LANCZOS).filter(ImageFilter.GaussianBlur(0.3)).convert("RGB") + payload = io.BytesIO() + degraded.save(payload, format="JPEG", quality=45, optimize=True) + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_image(page.rect, stream=payload.getvalue()) + return _save_document(document, target) + + def _fixture_ocr_adapter(image_path: Path, _lang: str) -> list[dict[str, Any]]: if "page_002" not in image_path.name: return [] @@ -268,6 +312,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") chinese = _native_chinese_fixture(fixtures / "native_chinese.pdf") hyphenated = _hyphenated_native_fixture(fixtures / "hyphenated_native.pdf") + formula_table = _formula_table_fixture(fixtures / "formula_table_dense.pdf") rotated_margin = _rotated_repeated_margin_fixture(fixtures / "rotated_repeated_margin.pdf") results = [ @@ -350,6 +395,19 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> _check("repair_count", parsed["extraction_report"].get("native_hyphenation_repair_count") == 3, parsed["extraction_report"].get("native_hyphenation_repair_count"), 3), ], ), + _run_case( + "formula_table_dense", + formula_table, + root, + lambda parsed: [ + _check("formulas", parsed["extraction_report"].get("formula_count") == 2, parsed["extraction_report"].get("formula_count"), 2), + _check("table_caption", parsed["extraction_report"].get("table_caption_count") == 1, parsed["extraction_report"].get("table_caption_count"), 1), + _check("table_columns", parsed["document_index"]["tables"][0].get("columns") == ["Model", "Depth", "Accuracy"], parsed["document_index"]["tables"][0].get("columns"), ["Model", "Depth", "Accuracy"]), + _check("table_rows", any(row.get("values") == ["Large", "12", "84.7"] for row in parsed["document_index"]["tables"][0].get("rows", [])), parsed["document_index"]["tables"][0].get("rows"), ["Large", "12", "84.7"]), + _check("structured_table_evidence", any(item.get("kind") == "table" and "Row: Large | 12 | 84.7" in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", []) if item.get("kind") == "table"], "structured Large row"), + _check("no_orphan_cells", not any(item.get("text") == "84.7" for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", [])], "no isolated table cells"), + ], + ), _run_case( "rotated_repeated_margin", rotated_margin, @@ -367,6 +425,8 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> if ocr_engine != "off": scanned_two_column = rasterize_pdf_as_scan(native, fixtures / "scanned_two_column.pdf", dpi=144) skewed_two_column = _skewed_scan_fixture(native, fixtures / "skewed_two_column.pdf") + degraded_two_column = _degraded_scan_fixture(native, fixtures / "degraded_two_column.pdf") + scanned_formula_table = rasterize_pdf_as_scan(formula_table, fixtures / "scanned_formula_table.pdf", dpi=144) results.append(_run_case( "mixed_scan_runtime_ocr", mixed, @@ -405,6 +465,30 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> ], ocr_engine=ocr_engine, )) + results.append(_run_case( + "degraded_two_column_quality_gate", + degraded_two_column, + root, + lambda parsed: [ + _check("quality_gate_applied", parsed["extraction_report"].get("ocr_latin_lexical_gate_applied") is True, parsed["extraction_report"].get("ocr_latin_lexical_gate_applied"), True), + _check("low_known_word_ratio", float(parsed["extraction_report"].get("ocr_latin_known_word_ratio") or 1.0) < 0.5, parsed["extraction_report"].get("ocr_latin_known_word_ratio"), "<0.5"), + _check("extraction_failed", parsed["extraction_report"].get("status") == "fail", parsed["extraction_report"].get("status"), "fail"), + _check("no_false_cache_scope", parsed["extraction_report"].get("sampled_scan_ready") is False, parsed["extraction_report"].get("sampled_scan_ready"), False), + ], + ocr_engine=ocr_engine, + )) + results.append(_run_case( + "scanned_formula_table_runtime_ocr", + scanned_formula_table, + root, + lambda parsed: [ + _check("formula_recovered", parsed["extraction_report"].get("formula_count", 0) >= 1, parsed["extraction_report"].get("formula_count"), ">=1"), + _check("table_columns", parsed["document_index"]["tables"][0].get("columns") == ["Model", "Depth", "Accuracy"], parsed["document_index"]["tables"][0].get("columns"), ["Model", "Depth", "Accuracy"]), + _check("large_row", any(row.get("values") == ["Large", "12", "84.7"] for row in parsed["document_index"]["tables"][0].get("rows", [])), parsed["document_index"]["tables"][0].get("rows"), ["Large", "12", "84.7"]), + _check("structured_evidence", any(item.get("kind") == "table" and "Row: Large | 12 | 84.7" in item.get("text", "") for item in parsed.get("evidence", [])), [item.get("text") for item in parsed.get("evidence", []) if item.get("kind") == "table"], "structured Large row"), + ], + ocr_engine=ocr_engine, + )) elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] ocr_stage_totals = { diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 417bb67..35af2bc 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -1292,6 +1292,88 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) tables.append({"id": f"table_{len(tables) + 1:03d}", **record}) elif block.get("kind") == "formula": formulas.append({"id": f"formula_{len(formulas) + 1:03d}", "page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "text": block.get("text", "")}) + page_map = {int(page["page"]): page for page in pages} + for table in tables: + page = page_map.get(int(table.get("page") or 0)) + if not page: + continue + blocks = page.get("blocks", []) + caption_index = next((index for index, block in enumerate(blocks) if str(block.get("id") or "") == str(table.get("block_id") or "")), None) + if caption_index is None: + continue + caption_bbox = table.get("bbox") or [0, 0, 0, 0] + candidates = [] + for block in blocks[caption_index + 1:]: + bbox = block.get("bbox") + text = _clean(block.get("text", "")) + if not bbox or not text: + continue + if str(block.get("kind") or "") in {"heading", "caption", "table", "formula"}: + break + if float(bbox[1]) > float(caption_bbox[3]) + min(300.0, float(page.get("height") or 800.0) * 0.35): + break + if len(text) > 60: + break + candidates.append(block) + if len(candidates) < 4: + continue + heights = sorted(max(1.0, float(block["bbox"][3]) - float(block["bbox"][1])) for block in candidates) + tolerance = max(5.0, heights[len(heights) // 2] * 0.75) + row_groups: list[list[dict[str, Any]]] = [] + for block in sorted(candidates, key=lambda item: ((float(item["bbox"][1]) + float(item["bbox"][3])) / 2.0, float(item["bbox"][0]))): + center = (float(block["bbox"][1]) + float(block["bbox"][3])) / 2.0 + if not row_groups: + row_groups.append([block]) + continue + previous_center = sum((float(item["bbox"][1]) + float(item["bbox"][3])) / 2.0 for item in row_groups[-1]) / len(row_groups[-1]) + if abs(center - previous_center) <= tolerance: + row_groups[-1].append(block) + else: + row_groups.append([block]) + if len(row_groups) < 2 or len(row_groups[0]) < 2: + continue + header = sorted(row_groups[0], key=lambda item: float(item["bbox"][0])) + anchors = [(float(item["bbox"][0]) + float(item["bbox"][2])) / 2.0 for item in header] + columns = [_clean(item.get("text", "")) for item in header] + rows = [] + cell_records = [] + for row_index, group in enumerate(row_groups): + values: list[str | None] = [None] * len(columns) + block_ids: list[str | None] = [None] * len(columns) + for block in sorted(group, key=lambda item: float(item["bbox"][0])): + center = (float(block["bbox"][0]) + float(block["bbox"][2])) / 2.0 + column_index = min(range(len(anchors)), key=lambda index: abs(center - anchors[index])) + value = _clean(block.get("text", "")) + values[column_index] = f"{values[column_index]} {value}".strip() if values[column_index] else value + block_ids[column_index] = str(block.get("id") or "") + cell_records.append({"row": row_index, "column": column_index, "text": value, "block_id": block.get("id"), "bbox": block.get("bbox")}) + if row_index: + rows.append({"values": values, "block_ids": block_ids}) + header_signal = any( + _cjk_character_count(value) > 0 + or (value and value[0].isalpha() and value[0].isupper()) + or value.isupper() + or bool(re.search(r"(?:[A-Za-z].*\d|\d.*[A-Za-z])", value)) + for value in columns + ) + if len(rows) < 2 or not header_signal: + continue + cell_block_ids = {str(record.get("block_id") or "") for record in cell_records} + for block in candidates: + if str(block.get("id") or "") in cell_block_ids: + block["kind"] = "table_cell" + all_boxes = [caption_bbox] + [block.get("bbox") for block in candidates if block.get("bbox")] + table.update({ + "columns": columns, + "rows": rows, + "cells": cell_records, + "bbox": [ + min(float(box[0]) for box in all_boxes), + min(float(box[1]) for box in all_boxes), + max(float(box[2]) for box in all_boxes), + max(float(box[3]) for box in all_boxes), + ], + }) return {"summary": "Page-aware section, formula, table-caption, and figure-caption index.", "sections": sections, "formulas": formulas[:120], "tables": tables[:80], "figures": figures[:80]} @@ -1304,6 +1386,27 @@ def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any] } +def _ocr_latin_lexical_quality(pages: list[dict[str, Any]]) -> dict[str, Any]: + text = " ".join(str(page.get("text") or "") for page in pages if page.get("used_ocr")) + anchors = re.findall(r"\b(?:abstract|introduction|method|methods|experiments|results|conclusion|discussion)\b", text, re.IGNORECASE) + tokens = [token for token in re.findall(r"[A-Za-z]{3,}", text) if not token.isupper()] + if len(tokens) < 20 or len(anchors) < 2: + return {"applied": False, "token_count": len(tokens), "known_word_ratio": None, "english_anchor_count": len(anchors)} + try: + import wordninja + + known_words = wordninja.DEFAULT_LANGUAGE_MODEL._wordcost + except Exception: + return {"applied": False, "token_count": len(tokens), "known_word_ratio": None, "english_anchor_count": len(anchors)} + known = sum(token.casefold() in known_words or token.casefold() in OCR_PROTECTED_WORDS for token in tokens) + return { + "applied": True, + "token_count": len(tokens), + "known_word_ratio": round(known / max(1, len(tokens)), 4), + "english_anchor_count": len(anchors), + } + + def _evidence_id(page: int, text: str, bbox: Any, kind: str) -> str: payload = json.dumps({"page": page, "text": _clean(text), "bbox": bbox, "kind": kind}, ensure_ascii=False, sort_keys=True).encode("utf-8") return f"E_P{int(page):03d}_{hashlib.sha1(payload).hexdigest()[:12]}" @@ -1331,11 +1434,36 @@ def _build_evidence(pages: list[dict[str, Any]], sections: list[dict[str, Any]], }) used += len(text) + for table in document_index.get("tables", []): + lines = [_clean(table.get("caption", ""))] + columns = [str(value or "") for value in table.get("columns", [])] + if columns: + lines.append("Columns: " + " | ".join(columns)) + for row in table.get("rows", []): + values = [str(value or "") for value in row.get("values", [])] + lines.append("Row: " + " | ".join(values)) + text = _clean("\n".join(value for value in lines if value)) + if not text or used + len(text) > max_chars: + continue + evidence.append({ + "id": _evidence_id(int(table.get("page") or 0), text, table.get("bbox"), "table"), + "page": table.get("page"), + "block_id": table.get("block_id"), + "bbox": table.get("bbox"), + "section_hint": "Table Captions", + "kind": "table", + "source": "structured-table", + "confidence": 1.0, + "text": text, + "char_count": len(text), + }) + used += len(text) + block_candidates: list[dict[str, Any]] = [] for page in pages: for block in page.get("blocks", []): text = _clean(block.get("text", "")) - if not text or block.get("kind") == "caption": + if not text or block.get("kind") in {"caption", "table", "table_cell"}: continue block_id = str(block.get("id") or "") section = next( @@ -1474,6 +1602,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st mean_ocr_confidence = round(sum(ocr_confidences) / max(1, len(ocr_confidences)), 4) if ocr_blocks else None short_ocr_blocks = [block for block in ocr_blocks if len(_normalized_compare_text(str(block.get("text") or ""))) < 3] short_ocr_block_ratio = round(len(short_ocr_blocks) / max(1, len(ocr_blocks)), 4) if ocr_blocks else None + lexical_quality = _ocr_latin_lexical_quality(pages) sampled_scan_ready = bool( partial_scan and len(readable) >= min(3, len(pages)) @@ -1490,10 +1619,13 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st warnings.append("mean OCR confidence is below 0.50") if short_ocr_block_ratio is not None and short_ocr_block_ratio > 0.3: warnings.append("OCR output is highly fragmented") + if lexical_quality.get("applied") and float(lexical_quality.get("known_word_ratio") or 0.0) < 0.72: + warnings.append("OCR English lexical plausibility is below 0.72") if not coverage.get("abstract") or not coverage.get("method"): warnings.append("abstract or method-like section was not confidently located") agreements = [page["poppler_agreement"] for page in pages if isinstance(page.get("poppler_agreement"), (int, float))] - report_status = "fail" if (readable_ratio < 0.6 and not sampled_scan_ready) or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) else "warning" if warnings else "pass" + lexical_failure = bool(lexical_quality.get("applied") and float(lexical_quality.get("known_word_ratio") or 0.0) < 0.5) + report_status = "fail" if (readable_ratio < 0.6 and not sampled_scan_ready) or (mean_ocr_confidence is not None and mean_ocr_confidence < 0.35) or lexical_failure else "warning" if warnings else "pass" return { "summary": "PDF extraction quality and fallback report.", "status": report_status, @@ -1523,6 +1655,10 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "ocr_spacing_repair_count": sum(int(item.get("spacing_repairs") or 0) for item in details.get("ocr_page_durations", [])), "mean_ocr_confidence": mean_ocr_confidence, "short_ocr_block_ratio": short_ocr_block_ratio, + "ocr_latin_lexical_gate_applied": bool(lexical_quality.get("applied")), + "ocr_latin_token_count": int(lexical_quality.get("token_count") or 0), + "ocr_latin_known_word_ratio": lexical_quality.get("known_word_ratio"), + "ocr_english_anchor_count": int(lexical_quality.get("english_anchor_count") or 0), "replacement_character_rate": round(sum(page.get("replacement_character_rate", 0.0) for page in pages) / max(1, len(pages)), 6), "mojibake_rate": round(sum(page.get("mojibake_rate", 0.0) for page in pages) / max(1, len(pages)), 6), "cross_extractor_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, @@ -1533,6 +1669,7 @@ def _extraction_report(source: Path, pages: list[dict[str, Any]], index: dict[st "section_coverage": coverage, "figure_caption_count": len(index.get("figures", [])), "table_caption_count": len(index.get("tables", [])), + "formula_count": len(index.get("formulas", [])), "section_count": len(index.get("sections", [])), "typographic_heading_count": sum((int(item.get("page") or 0), str(item.get("block_id") or "")) in bold_block_ids for item in index.get("sections", [])), "merged_heading_line_count": sum(len(item.get("continuation_block_ids", [])) for item in index.get("sections", [])), diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index c765c7b..1e4d6f4 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 27 +DOCUMENT_CACHE_VERSION = 29 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 6445bb1..445a6ae 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -997,6 +997,7 @@ def prepare_paper_figure_contract( "merged_heading_line_count": parsed["extraction_report"].get("merged_heading_line_count", 0), "figure_caption_count": parsed["extraction_report"].get("figure_caption_count", 0), "table_caption_count": parsed["extraction_report"].get("table_caption_count", 0), + "formula_count": parsed["extraction_report"].get("formula_count", 0), "section_coverage": parsed["extraction_report"].get("section_coverage", {}), "missing_priority_sections": [name for name, present in parsed["extraction_report"].get("section_coverage", {}).items() if not present], "ocr_candidate_count": len(parsed["extraction_report"].get("ocr_candidate_pages", [])), @@ -1011,6 +1012,8 @@ def prepare_paper_figure_contract( "native_hyphenation_repair_count": parsed["extraction_report"].get("native_hyphenation_repair_count", 0), "ocr_margin_noise_removed_count": parsed["extraction_report"].get("ocr_margin_noise_removed_count", 0), "ocr_spacing_repair_count": parsed["extraction_report"].get("ocr_spacing_repair_count", 0), + "ocr_latin_lexical_gate_applied": parsed["extraction_report"].get("ocr_latin_lexical_gate_applied", False), + "ocr_latin_known_word_ratio": parsed["extraction_report"].get("ocr_latin_known_word_ratio"), }, "provider": provider_summary, "contract_completion": { diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index e9e6bce..b46552f 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -171,9 +171,11 @@ def test_fast_suite_aggregates_reliability_and_timing_metrics(self): self.assertIn("typographic_heading_total", result["aggregate"]) self.assertIn("merged_heading_line_total", result["aggregate"]) self.assertIn("figure_caption_total", result["aggregate"]) + self.assertIn("formula_total", result["aggregate"]) self.assertIn("max_ocr_worker_count", result["aggregate"]) self.assertIn("repeated_margin_noise_removed_total", result["aggregate"]) self.assertIn("native_hyphenation_repair_total", result["aggregate"]) + self.assertIn("mean_ocr_latin_known_word_ratio", result["aggregate"]) self.assertIn("ocr_spacing_repair_total", result["aggregate"]) self.assertTrue((Path(tmp) / "fast_suite_report.json").exists()) @@ -192,13 +194,14 @@ def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr( result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") self.assertTrue(result["ok"]) - self.assertEqual(result["aggregate"]["case_count"], 8) - self.assertEqual(result["aggregate"]["passed_case_count"], 8) + self.assertEqual(result["aggregate"]["case_count"], 9) + self.assertEqual(result["aggregate"]["passed_case_count"], 9) self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "repeated_margin_noise.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "native_chinese.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "hyphenated_native.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "rotated_repeated_margin.pdf").exists()) + self.assertTrue((Path(tmp) / "fixtures" / "formula_table_dense.pdf").exists()) self.assertTrue((Path(tmp) / "mixed_scan_fixture_adapter" / "document_model.json").exists()) self.assertTrue((Path(tmp) / "pdf_extraction_stress_report.json").exists()) diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index ac38b65..f09f042 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -283,6 +283,30 @@ def test_encrypted_pdf_returns_explicit_password_error(self): with self.assertRaisesRegex(ValueError, "encrypted"): parse_paper(target, ocr_engine="off") + def test_high_confidence_garbled_english_ocr_fails_lexical_gate(self): + path = self._multipage_pdf([[]]) + + def adapter(_image_path, _lang): + lines = [ + "Abstract", + "2 Method", + "LEFT TOP encoder evidenoe begins themetiod ecndevidn letang clumn", + "Figur oveview TOerevl edne beong complene lett comn", + "RIGHT BOTTOM outputvid finihes rnading order clumn evidn", + "3 Conclusion", + ] + return [ + {"text": text, "confidence": 0.92, "quad": [[40, 30 + index * 80], [900, 30 + index * 80], [900, 75 + index * 80], [40, 75 + index * 80]]} + for index, text in enumerate(lines) + ] + + parsed = parse_paper(path, ocr_engine="easyocr", ocr_adapter=adapter) + report = parsed["extraction_report"] + + self.assertTrue(report["ocr_latin_lexical_gate_applied"]) + self.assertLess(report["ocr_latin_known_word_ratio"], 0.5) + self.assertEqual(report["status"], "fail") + def test_low_text_page_uses_local_ocr_adapter(self): path = self._pdf(lambda _page: None) From 46bb2804125c1a48ee9cfd47fb476cfec1e7343a Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Thu, 23 Jul 2026 19:30:25 +0800 Subject: [PATCH 51/74] Improve PDF extraction fidelity and contract grounding --- README.md | 4 +- docs/paper-to-image.md | 10 +- pyproject.toml | 4 +- rfs/cli.py | 20 +++- rfs/evaluation/pdf_extraction_benchmark.py | 43 ++++++++ rfs/paper_to_image/analyzer.py | 62 ++++++++--- rfs/paper_to_image/document_cache.py | 12 ++- rfs/paper_to_image/inspection.py | 9 +- rfs/paper_to_image/planner.py | 39 ++++++- rfs/paper_to_image/preparation.py | 90 ++++++++++++++-- tests/integration/test_paper_to_image.py | 14 ++- tests/unit/test_benchmarking.py | 4 +- tests/unit/test_paper_analyzer.py | 41 ++++++- tests/unit/test_paper_contract.py | 118 +++++++++++++++++++++ 14 files changed, 428 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 58b7944..a23a4ff 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. -Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Chinese, Japanese, and Korean headings plus Chinese `图/表` captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, and OCR-spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently and filters short vertical margin artifacts; the OCR extra includes `wordninja` for conservative English spacing recovery such as `2Method` to `2 Method`, while protecting CJK text, acronyms, camel-case model names, and common scientific compounds. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. +Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean headings plus localized Figure/Table captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, OCR render DPI, and spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently at 84 DPI and filters short vertical margin artifacts. The base installation includes `wordninja` for conservative English spacing recovery, while requiring multiple English anchors so non-English Latin words are not split accidentally. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: @@ -121,7 +121,7 @@ Run the generated multi-layout PDF extraction stress suite independently of pape rfs benchmark pdf-suite --out output\benchmarks\pdf-extraction --ocr-engine auto --json ``` -It generates fixed two-column, unnumbered-bold-section, rotated-page, and mixed native/scanned PDFs. The deterministic four-case tier always runs; `--ocr-engine auto` adds a fifth real local-OCR probe. Each case writes its fixture, rendered preview, document model, extraction report, assertions, and timing so reading-order or OCR regressions are reproducible without committing third-party PDFs. +It generates 10 deterministic parser cases covering columns, typography, rotation, repeated margins, CJK and Spanish text, hyphenation, formulas, and structured tables. A real OCR engine expands the suite to 16 cases with mixed, skewed, degraded, formula/table, and Spanish scans. Each case writes its fixture, rendered preview, document model, extraction report, assertions, and timing so reading-order or OCR regressions are reproducible without committing third-party PDFs. Use `paper-to-image` when the required endpoint is a generated raster framework figure rather than an editable PowerPoint file. The production route performs a diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index c088fc9..96a0bba 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -10,7 +10,7 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --j The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. -Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Quality gates count all Unicode letters and numbers. English, Chinese, Japanese, and Korean section aliases are recognized, Chinese `图/表` captions are indexed, and CJK cross-extractor agreement uses character bigrams so per-character spacing differences do not resemble corruption. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. +Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Quality gates count all Unicode letters and numbers. English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean section aliases are recognized; localized `Figure/Table` prefixes and Chinese `图/表` captions are indexed. Caption prefixes must include a real numeric, Roman, or appendix identifier, so ordinary prose such as `figura editable` is not misclassified. CJK cross-extractor agreement uses character bigrams so per-character spacing differences do not resemble corruption. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. If a long fully scanned paper cannot reach the 60% full-document readability gate within the deadline, the workflow may continue only when at least six high-confidence sampled pages cover both Abstract and Method. The result records `semantic_scope: sampled_pages_only`, includes an explicit uncertainty about unprocessed pages, and can never be production-ready. Missing Abstract/Method coverage or weak OCR still stops semantic planning. EasyOCR downloads are opt-in through `RFS_OCR_ALLOW_DOWNLOAD=1`; missing models fail quickly and still write a complete extraction report. @@ -49,9 +49,13 @@ For pages carrying a PDF rotation flag, native blocks are ordered in the unrotat Fast VLM planning preserves the paper's original writing system for every visible scientific label. Entity names and relation labels must either occur verbatim in their cited evidence or remain in the same source script; cross-script translations such as `文档编码器` to `Document encoder` fail planning validation and cannot be marked production-ready. Explanatory prose may follow the requested output language, but editable overlay labels do not. +For Spanish, French, German, and Portuguese papers, validation also detects same-script English translation. An English-looking visible label that does not occur verbatim anywhere in the paper evidence fails production validation even though both strings use Latin characters. This closes the gap where `codificador de documentos` could otherwise be silently changed to `Document Encoder`. + +Relation labels are optional: if a model supplies a connector label that is not verbatim in the relation's cited evidence, normalization clears the label while preserving the grounded direction and type. Overlay compilation also emits an exact scientific label only once when the same evidence-backed concept is both a module and an innovation callout. + Native PDF lines split by publisher hyphenation are joined inside the same source text block before evidence IDs are created. Known scientific words such as `trans-` + `former` and `de-` + `coder` lose the layout-only hyphen, while unknown long compound parts retain a literal hyphen. The extraction report records `native_hyphenation_repair_count`. -OCR normalization also repairs numbered CJK section boundaries such as `2方法`, `3实验`, and `4结论` before heading classification. This keeps multilingual scan coverage and the actual section index consistent instead of merely finding section keywords inside undifferentiated page text. +OCR normalization also repairs fused numbered section boundaries such as `2方法`, `2Metodo`, `3Experimentos`, and `4Conclusion` before heading classification. The same rule covers common French, German, and Portuguese headings. English word segmentation now requires at least two scientific anchor words, preventing a single English-looking suffix from splitting valid non-English words such as Spanish `procesamiento`. Production validation treats visible inputs, outputs, and innovations without evidence IDs as hard errors. Before validation, exact entity labels may be grounded deterministically to matching evidence; innovation labels are grounded only when the same evidence also contains an explicit novelty cue such as `we propose`, `we introduce`, or `本文提出`. Otherwise the result remains engineering-only instead of silently presenting an unsupported contribution. @@ -63,6 +67,8 @@ PDF inputs receive a lightweight container preflight before pypdf, Poppler, or P When `benchmark pdf-suite` runs with a real OCR engine, it now includes fully rasterized two-column and 2-degree skewed two-column pages in addition to the mixed native/scan fixture. These cases enforce column reconstruction, left-column-first reading order, section recovery, caption recovery, and OCR confidence using the actual runtime engine. +RapidOCR pages render at 84 DPI by default. This retains the detector limit and recognition batch used by the fast path while recovering long Latin-script lines that could disappear at 72 DPI. The generated stress suite includes native and rasterized Spanish papers and records `render_dpi` for each OCR page; the current suite contains 10 parser-only cases and 16 cases with runtime OCR. + English OCR pages with at least two explicit English section anchors also receive a lexical plausibility check backed by the bundled word-frequency model. A known-word ratio below 0.72 raises a warning and below 0.50 fails extraction even when the OCR engine reports high confidence. The gate is not applied to CJK or unrecognized non-English Latin documents, preventing an English dictionary from rejecting unrelated languages. Table captions now seed coordinate-based table regions. Header cells define column anchors, later cells are assigned by row and nearest column, and the document index stores `columns`, `rows`, `cells`, and a full table bounding box. Individual table cells are excluded from narrative evidence and replaced by one row-major structured table evidence record, preventing OCR detection order from turning `Model | Depth | Accuracy` into misleading prose. diff --git a/pyproject.toml b/pyproject.toml index 1c7f2a3..4b8768a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pdfplumber", "requests", "opencv-python-headless", + "wordninja", "pywin32; platform_system == 'Windows'" ] @@ -22,8 +23,7 @@ dependencies = [ ocr = [ "rapidocr-onnxruntime", "paddleocr", - "easyocr", - "wordninja" + "easyocr" ] [project.scripts] diff --git a/rfs/cli.py b/rfs/cli.py index 0bdcd30..935c05e 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -5,6 +5,7 @@ import json import os import shutil +import subprocess import sys from pathlib import Path @@ -29,6 +30,21 @@ def _json_print(data: dict) -> None: print(json.dumps(data, indent=2, ensure_ascii=False)) +def _probe_executable(name: str, version_arg: str = "-v") -> dict: + path = shutil.which(name) + if not path: + return {"available": False, "path": None, "error": "not found on PATH"} + try: + completed = subprocess.run([path, version_arg], capture_output=True, text=True, timeout=5, check=False) + except Exception as exc: + return {"available": False, "path": path, "error": str(exc)} + output = "\n".join(part.strip() for part in (completed.stdout, completed.stderr) if part and part.strip()) + result = {"available": completed.returncode == 0, "path": path, "returncode": completed.returncode} + if completed.returncode != 0: + result["error"] = output.splitlines()[0] if output else "version probe failed" + return result + + def _doctor() -> dict: deps = {} for name, module in [("Pillow", "PIL"), ("python-pptx", "pptx"), ("PyMuPDF", "fitz"), ("pypdf", "pypdf"), ("pdfplumber", "pdfplumber"), ("requests", "requests"), ("opencv-python-headless", "cv2")]: @@ -76,8 +92,8 @@ def _doctor() -> dict: "files": easyocr_files, } optional_pdf = { - "pdftotext": {"available": bool(shutil.which("pdftotext")), "path": shutil.which("pdftotext")}, - "pdftoppm": {"available": bool(shutil.which("pdftoppm")), "path": shutil.which("pdftoppm")}, + "pdftotext": _probe_executable("pdftotext"), + "pdftoppm": _probe_executable("pdftoppm"), "easyocr": {"available": importlib.util.find_spec("easyocr") is not None, "models": easyocr_models}, "rapidocr": { "available": importlib.util.find_spec("rapidocr_onnxruntime") is not None, diff --git a/rfs/evaluation/pdf_extraction_benchmark.py b/rfs/evaluation/pdf_extraction_benchmark.py index 7c03965..667c0e4 100644 --- a/rfs/evaluation/pdf_extraction_benchmark.py +++ b/rfs/evaluation/pdf_extraction_benchmark.py @@ -128,6 +128,23 @@ def _native_chinese_fixture(target: Path) -> Path: return _save_document(document, target) +def _native_spanish_fixture(target: Path) -> Path: + import fitz + + document = fitz.open() + page = document.new_page(width=600, height=800) + page.insert_text((45, 50), "Resumen", fontname="hebo", fontsize=13) + page.insert_textbox((45, 75, 555, 145), "Proponemos un codificador de documentos que transforma el artículo en evidencia estructurada y genera una figura editable.", fontsize=10) + page.insert_text((45, 185), "2 Método", fontname="hebo", fontsize=13) + page.insert_textbox((45, 210, 555, 295), "El documento de entrada pasa al codificador de documentos. El decodificador de relaciones conecta las entidades y produce una salida editable.", fontsize=10) + page.insert_textbox((45, 335, 555, 395), "Figura 1: Codificador de documentos, decodificador de relaciones y salida editable.", fontsize=10) + page.insert_text((45, 455), "3 Experimentos", fontname="hebo", fontsize=13) + page.insert_textbox((45, 480, 555, 545), "Evaluamos la recuperación de entidades, las relaciones correctas y el tiempo de procesamiento.", fontsize=10) + page.insert_text((45, 600), "4 Conclusión", fontname="hebo", fontsize=13) + page.insert_textbox((45, 625, 555, 690), "El método conserva la terminología científica y las referencias de evidencia.", fontsize=10) + return _save_document(document, target) + + def _hyphenated_native_fixture(target: Path) -> Path: import fitz @@ -311,6 +328,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> mixed = _mixed_scan_fixture(fixtures / "mixed_scan.pdf") repeated_margin = _repeated_margin_fixture(fixtures / "repeated_margin_noise.pdf") chinese = _native_chinese_fixture(fixtures / "native_chinese.pdf") + spanish = _native_spanish_fixture(fixtures / "native_spanish.pdf") hyphenated = _hyphenated_native_fixture(fixtures / "hyphenated_native.pdf") formula_table = _formula_table_fixture(fixtures / "formula_table_dense.pdf") rotated_margin = _rotated_repeated_margin_fixture(fixtures / "rotated_repeated_margin.pdf") @@ -385,6 +403,17 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> _check("no_mojibake", parsed["extraction_report"].get("mojibake_rate") == 0.0, parsed["extraction_report"].get("mojibake_rate"), 0.0), ], ), + _run_case( + "native_spanish", + spanish, + root, + lambda parsed: [ + _check("unicode_preserved", "artículo" in parsed["pages"][0]["text"] and "Conclusión" in parsed["pages"][0]["text"], parsed["pages"][0]["text"], "Spanish accents preserved"), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Spanish abstract/method/experiments/conclusion"), + _check("figure_caption", any("codificador de documentos" in str(item.get("caption") or "").casefold() for item in parsed.get("document_index", {}).get("figures", [])), parsed.get("document_index", {}).get("figures", []), "Spanish Figura 1 caption"), + _check("source_language_preserved", "document encoder" not in parsed["pages"][0]["text"].casefold() and "codificador de documentos" in parsed["pages"][0]["text"].casefold(), parsed["pages"][0]["text"], "Spanish terminology without English translation"), + ], + ), _run_case( "hyphenated_native", hyphenated, @@ -427,6 +456,7 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> skewed_two_column = _skewed_scan_fixture(native, fixtures / "skewed_two_column.pdf") degraded_two_column = _degraded_scan_fixture(native, fixtures / "degraded_two_column.pdf") scanned_formula_table = rasterize_pdf_as_scan(formula_table, fixtures / "scanned_formula_table.pdf", dpi=144) + scanned_spanish = rasterize_pdf_as_scan(spanish, fixtures / "scanned_spanish.pdf", dpi=144) results.append(_run_case( "mixed_scan_runtime_ocr", mixed, @@ -489,6 +519,19 @@ def run_pdf_extraction_stress_suite(out: str | Path, ocr_engine: str = "off") -> ], ocr_engine=ocr_engine, )) + results.append(_run_case( + "scanned_spanish_runtime_ocr", + scanned_spanish, + root, + lambda parsed: [ + _check("ocr_completed", parsed["extraction_report"].get("ocr_pages") == [1], parsed["extraction_report"].get("ocr_pages"), [1]), + _check("priority_sections", all(parsed["extraction_report"].get("section_coverage", {}).get(name) for name in ("abstract", "method", "experiments", "conclusion")), parsed["extraction_report"].get("section_coverage"), "Spanish abstract/method/experiments/conclusion"), + _check("figure_caption", parsed["extraction_report"].get("figure_caption_count") == 1, parsed["extraction_report"].get("figure_caption_count"), 1), + _check("source_language_preserved", "document encoder" not in parsed["pages"][0]["text"].casefold() and "codificador de documentos" in parsed["pages"][0]["text"].casefold(), parsed["pages"][0]["text"], "Spanish terminology without English translation"), + _check("quality_gate_passed", parsed["extraction_report"].get("status") != "fail", parsed["extraction_report"].get("status"), "pass or warning"), + ], + ocr_engine=ocr_engine, + )) elapsed = [float(item.get("elapsed_seconds") or 0.0) for item in results] ocr_stage_totals = { diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 35af2bc..f337c0b 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -19,11 +19,11 @@ MOJIBAKE_MARKERS = ("\ufffd", "Ã", "Â", "â€", "锟", "鈥", "銆", "鏅") SECTION_ALIASES = { - "abstract": ("abstract", "摘要", "概要", "초록"), - "introduction": ("introduction", "background", "引言", "介绍", "背景", "はじめに", "序論", "서론", "배경"), - "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework", "方法", "方法论", "架构", "系统概述", "框架", "手法", "方法論", "アーキテクチャ", "방법", "방법론", "아키텍처"), - "experiments": ("experiment", "experiments", "evaluation", "results", "实验", "评估", "结果", "実験", "評価", "結果", "실험", "평가", "결과"), - "conclusion": ("conclusion", "conclusions", "discussion", "结论", "讨论", "总结", "結論", "考察", "まとめ", "결론", "논의", "요약"), + "abstract": ("abstract", "resumen", "résumé", "zusammenfassung", "resumo", "摘要", "概要", "초록"), + "introduction": ("introduction", "background", "introducción", "introduccion", "introdução", "introducao", "einleitung", "contexte", "contexto", "antecedentes", "引言", "介绍", "背景", "はじめに", "序論", "서론", "배경"), + "method": ("method", "methods", "methodology", "approach", "architecture", "system overview", "framework", "método", "metodo", "métodos", "metodos", "metodología", "metodologia", "méthode", "methode", "méthodes", "methodes", "méthodologie", "methodologie", "methoden", "methodik", "enfoque", "approche", "ansatz", "abordagem", "方法", "方法论", "架构", "系统概述", "框架", "手法", "方法論", "アーキテクチャ", "방법", "방법론", "아키텍처"), + "experiments": ("experiment", "experiments", "evaluation", "results", "experimento", "experimentos", "evaluación", "evaluacion", "resultados", "expérience", "experience", "expériences", "experiences", "évaluation", "experimente", "auswertung", "ergebnisse", "avaliação", "avaliacao", "实验", "评估", "结果", "実験", "評価", "結果", "실험", "평가", "결과"), + "conclusion": ("conclusion", "conclusions", "discussion", "conclusión", "conclusiones", "discusión", "discusion", "conclusão", "conclusao", "conclusões", "conclusoes", "discussão", "discussao", "schlussfolgerung", "schlussfolgerungen", "fazit", "diskussion", "结论", "讨论", "总结", "結論", "考察", "まとめ", "결론", "논의", "요약"), } OCR_PROTECTED_WORDS = { @@ -38,6 +38,23 @@ "and", "decoder", "encoder", "evidence", "for", "from", "generation", "into", "method", "model", "of", "output", "retriever", "the", "to", "with", } +OCR_RENDER_DPI = 84 +LATIN_CAPTION_PREFIX = re.compile( + r"^(?Pfigure|fig\.|table|figura|abbildung|tabla|tableau|tabelle)\s*" + r"(?P[a-z]?\d+(?:[.\-]\d+)*|[ivxlcdm]+|[a-z])\b", + re.IGNORECASE, +) + + +def _caption_prefix_kind(text: str) -> str | None: + value = _clean(text) + match = LATIN_CAPTION_PREFIX.match(value) + if match: + prefix = match.group("prefix").casefold() + return "table" if prefix in {"table", "tabla", "tableau", "tabelle"} else "caption" + if re.match(r"^(图|圖|表)\s*[a-z0-9一二三四五六七八九十]+", value, re.IGNORECASE): + return "caption" if value.startswith(("图", "圖")) else "table" + return None def _clean(text: str) -> str: @@ -58,15 +75,22 @@ def _clean(text: str) -> str: def _repair_ocr_spacing(text: str, splitter: Callable[[str], list[str]] | None = None) -> tuple[str, int]: value = _clean(text) repairs = 0 - value, count = re.subn(r"\b(figure|fig\.|table)(?=\d)", r"\1 ", value, flags=re.IGNORECASE) + value, count = re.subn(r"\b(figure|fig\.|table|figura|abbildung|tabla|tableau|tabelle)(?=\d)", r"\1 ", value, flags=re.IGNORECASE) repairs += count - section_terms = "abstract|introduction|background|method|methods|approach|experiments|results|conclusion|discussion|references|appendix" + section_terms = ( + "abstract|introduction|background|method|methods|approach|experiments|results|conclusion|discussion|references|appendix|" + "resumen|introducción|introduccion|método|metodo|métodos|metodos|metodología|metodologia|enfoque|" + "experimento|experimentos|evaluación|evaluacion|resultados|conclusión|conclusiones|discusión|discusion|" + "résumé|méthode|méthodes|expérience|expériences|évaluation|résultats|" + "zusammenfassung|einleitung|methoden|methodik|experimente|ergebnisse|schlussfolgerung|fazit|" + "resumo|introdução|introducao|avaliação|avaliacao|conclusão|conclusao" + ) value, count = re.subn(rf"(? str: return token pieces = [str(piece) for piece in splitter(token) if str(piece)] lowered = [piece.casefold() for piece in pieces] - if len(pieces) < 2 or any(len(piece) < 2 for piece in pieces) or not any(piece in OCR_SPLIT_ANCHORS for piece in lowered): + anchor_count = sum(piece in OCR_SPLIT_ANCHORS for piece in lowered) + if len(pieces) < 2 or any(len(piece) < 2 for piece in pieces) or anchor_count < 2: return token repairs += len(pieces) - 1 return " ".join(pieces) @@ -160,8 +185,9 @@ def _looks_like_typographic_heading(text: str) -> bool: def _block_kind(text: str, width_ratio: float = 0.0) -> str: value = _clean(text) - if re.match(r"^(figure|fig\.|table|图|圖|表)\s*[a-z0-9一二三四五六七八九十]+", value, re.IGNORECASE): - return "caption" if value.casefold().startswith(("figure", "fig.", "图", "圖")) else "table" + caption_kind = _caption_prefix_kind(value) + if caption_kind: + return caption_kind section_aliases = tuple(alias.casefold() for aliases in SECTION_ALIASES.values() for alias in aliases) if len(value) <= 100 and any(value.casefold() == alias or value.casefold().startswith((f"{alias} ", f"{alias}:", f"{alias}:")) for alias in section_aliases): return "heading" @@ -528,7 +554,10 @@ def _group_ocr_words(records: list[dict[str, Any]], image_width: float) -> list[ column = 2 if width >= image_width * 0.62 else 0 if center_x <= image_width / 2.0 else 1 previous = previous_by_column.get(column) gap = item["bbox"][1] - previous["bbox"][3] if previous else float("inf") - starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|figure|fig\.|table)\b", item["text"], re.IGNORECASE)) + starts_new = bool( + re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+)\b", item["text"], re.IGNORECASE) + or _caption_prefix_kind(item["text"]) + ) if previous is None or gap > median_height * 1.8 or starts_new: parent += 1 item["parent_block"] = parent @@ -564,8 +593,8 @@ def _assign_ocr_parent_blocks(lines: list[dict[str, Any]], image_width: float) - (float(previous["bbox"][3]) - float(previous["bbox"][1])) if previous else 0.0, ) text = str(item.get("text") or "") - starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+|figure|fig\.|table)\b", text, re.IGNORECASE)) - starts_caption = bool(re.match(r"^(?:figure|fig\.|table)\b", text, re.IGNORECASE)) + starts_new = bool(re.match(r"^(?:abstract|\d+(?:\.\d+)*\s+)\b", text, re.IGNORECASE) or _caption_prefix_kind(text)) + starts_caption = bool(_caption_prefix_kind(text)) continuing_caption = bool(previous and previous.get("_caption_group") and gap <= local_height * 1.15 and not starts_new) paragraph_break = bool(previous and not continuing_caption and str(previous.get("text") or "").rstrip().endswith((".", ":")) and gap > local_height * 0.45) if previous is None or gap > local_height * 1.15 or starts_new or paragraph_break: @@ -706,7 +735,7 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call try: with tempfile.TemporaryDirectory() as temp: target = Path(temp) / f"page_{page_number:03d}.png" - dpi = 72 if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 + dpi = OCR_RENDER_DPI if adapter is None and engine in {"auto", "easyocr", "rapidocr"} else 160 render_started = time.monotonic() pixmap = page.get_pixmap(dpi=dpi, alpha=False) pixmap.save(str(target)) @@ -749,6 +778,7 @@ def _ocr_page(page: Any, page_number: int, engine: str, lang: str, adapter: Call "margin_noise_removed": margin_noise_removed, "spacing_repairs": spacing_repairs, "render_seconds": round(render_seconds, 4), + "render_dpi": dpi, "postprocess_seconds": round(time.monotonic() - postprocess_started, 4), **engine_diagnostics, } @@ -1282,7 +1312,7 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) parent = block.get("parent_block") continuation = [] for following in page_blocks[block_index + 1:]: - if parent is None or following.get("parent_block") != parent or re.match(r"^(figure|fig\.|table|图|圖|表)\s*[\d一二三四五六七八九十]", str(following.get("text") or ""), re.IGNORECASE): + if parent is None or following.get("parent_block") != parent or _caption_prefix_kind(str(following.get("text") or "")): break continuation.append(str(following.get("text") or "")) if continuation: diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 1e4d6f4..723536c 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 29 +DOCUMENT_CACHE_VERSION = 31 def document_model_cacheable(parsed: dict[str, Any]) -> bool: @@ -55,7 +55,11 @@ def read_document_cache( if not path.exists(): return None cached = read_json(path) - if not isinstance(cached, dict) or not document_model_cacheable(cached): + if ( + not isinstance(cached, dict) + or cached.get("document_cache_version") != DOCUMENT_CACHE_VERSION + or not document_model_cacheable(cached) + ): return None active = Path(source).resolve() cached["source_path"] = str(active) @@ -77,5 +81,7 @@ def write_document_cache( if not document_model_cacheable(parsed): return None path = document_cache_path(source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, max_chars=max_chars, max_ocr_pages=max_ocr_pages) - write_json(path, parsed) + payload = dict(parsed) + payload["document_cache_version"] = DOCUMENT_CACHE_VERSION + write_json(path, payload) return path diff --git a/rfs/paper_to_image/inspection.py b/rfs/paper_to_image/inspection.py index 13712ba..57f7aa2 100644 --- a/rfs/paper_to_image/inspection.py +++ b/rfs/paper_to_image/inspection.py @@ -8,7 +8,7 @@ from ..utils import ensure_dir, read_json, write_json, write_text from .analyzer import paper_markdown, parse_paper -from .document_cache import read_document_cache, write_document_cache +from .document_cache import DOCUMENT_CACHE_VERSION, read_document_cache, write_document_cache def _source_hash(path: Path) -> str: @@ -39,7 +39,11 @@ def inspect_paper( model_path = root / "document_model.json" if model_path.exists(): cached = read_json(model_path) - if isinstance(cached, dict) and cached.get("source_sha256") == digest: + if ( + isinstance(cached, dict) + and cached.get("source_sha256") == digest + and cached.get("document_cache_version") == DOCUMENT_CACHE_VERSION + ): report = dict(cached.get("extraction_report") or {}) elapsed = round(time.monotonic() - started, 3) return { @@ -65,6 +69,7 @@ def inspect_paper( ) if not ocr_adapter: write_document_cache(active_source, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + parsed["document_cache_version"] = DOCUMENT_CACHE_VERSION write_json(model_path, parsed) write_json(root / "extraction_report.json", parsed["extraction_report"]) write_json(root / "section_index.json", parsed["document_index"]) diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index ff83535..6e98f3c 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -575,6 +575,7 @@ def validate_plan_grounding(plan: dict, parsed: dict) -> dict: evidence_items = [item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id")] valid_evidence = {item["id"] for item in evidence_items} evidence_text = {str(item["id"]): str(item.get("text") or "") for item in evidence_items} + all_evidence_text = " ".join(evidence_text.values()) errors: list[str] = [] warnings: list[str] = [] spec = plan.get("figure_specification", {}) @@ -592,6 +593,37 @@ def script_counts(value: str) -> tuple[int, int]: latin = sum(("a" <= char.casefold() <= "z") for char in value) return cjk, latin + def latin_language(value: str) -> str | None: + folded = "".join(char for char in unicodedata.normalize("NFKD", value.casefold()) if not unicodedata.combining(char)) + words = re.findall(r"[a-z]{2,}", folded) + profiles = { + "english": {"the", "and", "that", "with", "from", "this", "we", "our", "for", "into", "method", "results"}, + "spanish": {"el", "la", "los", "las", "un", "una", "que", "con", "para", "por", "del", "este", "esta", "metodo", "resultados", "relaciones"}, + "french": {"le", "la", "les", "un", "une", "des", "que", "avec", "pour", "dans", "cette", "methode", "resultats", "relations"}, + "german": {"der", "die", "das", "den", "dem", "ein", "eine", "und", "mit", "fur", "von", "diese", "methode", "ergebnisse"}, + "portuguese": {"o", "os", "as", "um", "uma", "que", "com", "para", "por", "este", "esta", "metodo", "resultados", "relacoes"}, + } + scores = {name: sum(word in markers for word in words) for name, markers in profiles.items()} + language, score = max(scores.items(), key=lambda item: item[1]) + if language == "english" or score < 5 or score < scores["english"] + 2: + return None + return language + + detected_latin_language = latin_language(all_evidence_text) + + def english_label_ratio(value: str) -> float | None: + tokens = re.findall(r"[A-Za-z]{3,}", value) + if not tokens: + return None + try: + import wordninja + + known_words = wordninja.DEFAULT_LANGUAGE_MODEL._wordcost + except Exception: + return None + known = sum(token.casefold() in known_words or token.casefold() in {"encoder", "decoder", "embedding", "transformer"} for token in tokens) + return known / len(tokens) + def validate_visible_label(location: str, label: str, evidence_ids: list[str], *, fallback_text: str = "") -> None: value = str(label or "").strip() if not value: @@ -605,6 +637,11 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * changed_from_latin = source_latin >= 4 and source_cjk == 0 and label_cjk >= 2 and label_latin == 0 if changed_from_cjk or changed_from_latin: errors.append(f"{location} visible label changes the source writing system and is not verbatim evidence: {value}") + return + if detected_latin_language and normalized_term(value) not in normalized_term(all_evidence_text): + ratio = english_label_ratio(value) + if ratio is not None and ratio >= 0.8: + errors.append(f"{location} visible label appears translated from {detected_latin_language} and is not verbatim evidence: {value}") modules = spec.get("modules") if isinstance(spec.get("modules"), list) else [] if not modules: @@ -663,7 +700,6 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * validate_visible_label(f"relation {source} -> {target}", str(relation.get("label") or ""), evidence_ids) terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} - all_evidence_text = " ".join(evidence_text.values()) for source_term, visible_label in terminology.items(): validate_visible_label(f"terminology[{source_term}]", str(visible_label or ""), [], fallback_text=all_evidence_text) @@ -699,4 +735,5 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * "evidence_count": len(valid_evidence), "errors": errors, "warnings": warnings, + "detected_source_language": detected_latin_language or "unknown_or_english", } diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 445a6ae..3591f2e 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -321,6 +321,37 @@ def _normalize_contract_entities(raw_items: Any, field: str) -> list[dict[str, A return normalized +def _deduplicate_contract_entities(spec: dict[str, Any]) -> dict[str, list[str]]: + replacements: dict[str, str] = {} + removed: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + deduplicated: list[dict[str, Any]] = [] + local_by_label: dict[str, dict[str, Any]] = {} + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + label = _normalized_label(_item_label(item)) + existing = local_by_label.get(label) if label else None + if existing: + existing["evidence_ids"] = list(dict.fromkeys(list(existing.get("evidence_ids", [])) + list(item.get("evidence_ids", [])))) + replacements[str(item.get("id") or "")] = str(existing.get("id") or "") + removed.append(f"{field}:{item.get('id')}->{existing.get('id')}") + continue + deduplicated.append(item) + if label: + local_by_label[label] = item + spec[field] = deduplicated + + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + for key in ("source", "source_id", "target", "target_id"): + value = str(relation.get(key) or "") + if value in replacements: + relation[key] = replacements[value] + return {"removed": removed, "replacements": [f"{source}->{target}" for source, target in replacements.items() if source and target]} + + def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: aliases: dict[str, set[str]] = {} endpoint_ids: set[str] = set() @@ -400,6 +431,7 @@ def _repair_contract_relation_endpoints(spec: dict[str, Any]) -> dict[str, list[ by_key: dict[tuple[str, str, str], dict[str, Any]] = {} removed_duplicates: list[str] = [] removed_unresolved: list[str] = [] + removed_self_relations: list[str] = [] for relation in relations: source = str(relation.get("source") or "") target = str(relation.get("target") or "") @@ -407,6 +439,9 @@ def _repair_contract_relation_endpoints(spec: dict[str, Any]) -> dict[str, list[ if source not in endpoints or target not in endpoints: removed_unresolved.append(f"{source}->{target}") continue + if source == target: + removed_self_relations.append(f"{source}->{target}:{relation_type}") + continue key = (source, target, relation_type) existing = by_key.get(key) if existing: @@ -422,7 +457,29 @@ def _repair_contract_relation_endpoints(spec: dict[str, Any]) -> dict[str, list[ "repaired": list(dict.fromkeys(repaired)), "removed_duplicates": list(dict.fromkeys(removed_duplicates)), "removed_unresolved": list(dict.fromkeys(removed_unresolved)), + "removed_self_relations": list(dict.fromkeys(removed_self_relations)), + } + + +def _clear_unverbatim_relation_labels(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") } + removed: list[str] = [] + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + label = str(relation.get("label") or "").strip() + if not label: + continue + cited_text = " ".join(evidence_by_id.get(str(value), "") for value in relation.get("evidence_ids", [])) + if _normalized_label(label) and _normalized_label(label) in _normalized_label(cited_text): + continue + removed.append(f"{relation.get('source')}->{relation.get('target')}:{label}") + relation["label"] = "" + return removed def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} @@ -438,8 +495,11 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d spec["modules"] = summary_modules if isinstance(summary_modules, list) else [] for field in ("inputs", "modules", "outputs", "innovations"): spec[field] = _normalize_contract_entities(spec.get(field), field) + entity_deduplication = _deduplicate_contract_entities(spec) spec["relations"] = _normalize_contract_relations(spec) completion_report = augment_contract_from_evidence(spec, parsed) + completion_report["removed_duplicate_entities"] = entity_deduplication["removed"] + completion_report["duplicate_entity_replacements"] = entity_deduplication["replacements"] plan["contract_completion_report"] = completion_report if len(completion_report.get("added_entities", [])) >= 3: fallback_ids = { @@ -461,6 +521,7 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d completion_report["relation_endpoint_repairs"] = endpoint_report["repaired"] completion_report["removed_duplicate_relations"] = endpoint_report["removed_duplicates"] completion_report["removed_unresolved_relations"] = endpoint_report["removed_unresolved"] + completion_report["removed_self_relations"] = endpoint_report["removed_self_relations"] if endpoint_report["repaired"]: secondary_completion = augment_contract_from_evidence(spec, parsed) for key in ("added_entities", "upgraded_entities", "adopted_entities", "added_relations", "repaired_relations", "grounded_entities"): @@ -604,6 +665,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: spec["topology"] = _infer_topology(spec) completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) _repair_cross_script_terminology(spec, parsed) + completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) labels = [] terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} labels.extend(str(value).strip() for value in terminology.values() if str(value).strip()) @@ -641,22 +703,33 @@ def choose_input_target(label: str) -> dict[str, Any] | None: def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: spec = plan.get("figure_specification", {}) labels = [] + visible_targets: dict[str, str] = {} + target_aliases: dict[str, str] = {} for field in ("inputs", "modules", "outputs", "innovations"): for index, item in enumerate(spec.get(field, []) if isinstance(spec.get(field), list) else []): if not isinstance(item, dict): continue text = _item_label(item) - if text: - labels.append({"target_id": _item_id(item, field, index), "text": text, "role": field.rstrip("s")}) + normalized = _normalized_label(text) + target_id = _item_id(item, field, index) + if text and normalized not in visible_targets: + labels.append({"target_id": target_id, "text": text, "role": field.rstrip("s")}) + visible_targets[normalized] = target_id + elif text and normalized: + target_aliases[target_id] = visible_targets[normalized] connectors = [] + connector_keys: set[tuple[str, str, str, str]] = set() for item in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: if isinstance(item, dict): - connectors.append({ - "source": str(item.get("source") or item.get("source_id") or ""), - "target": str(item.get("target") or item.get("target_id") or ""), - "type": str(item.get("type") or item.get("relation_type") or "data_flow"), - "label": str(item.get("label") or ""), - }) + source = target_aliases.get(str(item.get("source") or item.get("source_id") or ""), str(item.get("source") or item.get("source_id") or "")) + target = target_aliases.get(str(item.get("target") or item.get("target_id") or ""), str(item.get("target") or item.get("target_id") or "")) + relation_type = str(item.get("type") or item.get("relation_type") or "data_flow") + label = str(item.get("label") or "") + key = (source, target, relation_type, label) + if not source or not target or source == target or key in connector_keys: + continue + connectors.append({"source": source, "target": target, "type": relation_type, "label": label}) + connector_keys.add(key) return { "summary": "Exact editable labels and directed connectors derived from the paper contract.", "labels": labels, @@ -829,6 +902,7 @@ def prepare_paper_figure_contract( parsed = parse_paper(archived_paper, deadline_at=deadline_at, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, ocr_rescue_min_remaining=ocr_rescue_min_remaining) if not ocr_adapter: write_document_cache(archived_paper, parsed, ocr_engine=ocr_engine, ocr_lang=ocr_lang) + parsed["document_cache_version"] = DOCUMENT_CACHE_VERSION document_preparation_seconds = round(time.monotonic() - started, 3) write_json(root / "document_model.json", parsed) write_json(root / "extraction_report.json", parsed["extraction_report"]) diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 31bc434..be12509 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -11,7 +11,7 @@ import fitz from PIL import Image -from rfs.cli import _doctor, build_parser +from rfs.cli import _doctor, _probe_executable, build_parser from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import review_candidate from rfs.paper_to_image.generator import generate_and_select @@ -141,6 +141,18 @@ def test_doctor_reports_easyocr_model_readiness(self): self.assertIn("allow_download", models) self.assertIn("effective_detector_limit", report["pdf_tools"]["rapidocr"]) + def test_executable_probe_rejects_broken_path_wrapper(self): + with patch("rfs.cli.shutil.which", return_value=r"C:\broken\pdftoppm.cmd"), patch("rfs.cli.subprocess.run") as run: + run.return_value.returncode = 1 + run.return_value.stdout = "" + run.return_value.stderr = "The system cannot find the path specified." + + result = _probe_executable("pdftoppm") + + self.assertFalse(result["available"]) + self.assertEqual(result["returncode"], 1) + self.assertIn("cannot find", result["error"]) + def test_fast_framework_prompt_writes_contract_without_generation(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index b46552f..2f197b6 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -194,8 +194,8 @@ def test_pdf_extraction_stress_suite_runs_deterministically_without_runtime_ocr( result = run_pdf_extraction_stress_suite(tmp, ocr_engine="off") self.assertTrue(result["ok"]) - self.assertEqual(result["aggregate"]["case_count"], 9) - self.assertEqual(result["aggregate"]["passed_case_count"], 9) + self.assertEqual(result["aggregate"]["case_count"], 10) + self.assertEqual(result["aggregate"]["passed_case_count"], 10) self.assertTrue((Path(tmp) / "fixtures" / "native_two_column.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "repeated_margin_noise.pdf").exists()) self.assertTrue((Path(tmp) / "fixtures" / "native_chinese.pdf").exists()) diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index f09f042..1597375 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -9,7 +9,7 @@ from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _deadline_ocr_wave, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper from rfs.paper_to_image.inspection import inspect_paper -from rfs.paper_to_image.document_cache import write_document_cache +from rfs.paper_to_image.document_cache import DOCUMENT_CACHE_VERSION, write_document_cache from rfs.paper_to_image.preparation import _semantic_cache_safe from rfs.reference_text_extractor import _positive_ocr_setting, _rapidocr_detector_limit, run_rapidocr_detailed @@ -137,6 +137,12 @@ def test_cjk_section_coverage_and_caption_kinds(self): self.assertEqual(_block_kind("表 2 实验结果"), "table") self.assertEqual(_block_kind("2 方法"), "heading") + def test_localized_caption_prefix_requires_a_real_identifier(self): + self.assertEqual(_block_kind("Figura 1: Arquitectura del sistema"), "caption") + self.assertEqual(_block_kind("Tabelle A2: Ablation results"), "table") + self.assertEqual(_block_kind("figura editable"), "paragraph") + self.assertEqual(_block_kind("tableau comparatif"), "paragraph") + def test_three_column_blocks_are_sorted_column_major(self): def draw(page): page.insert_textbox((400, 100, 560, 170), "COLUMN THREE TOP complete scientific paragraph.", fontsize=9) @@ -225,6 +231,24 @@ def test_ocr_spacing_repair_restores_numbered_chinese_section_heading(self): self.assertEqual(repaired, "2 \u65b9\u6cd5") self.assertEqual(count, 1) + def test_ocr_spacing_repair_restores_numbered_spanish_section_headings(self): + repaired, count = _repair_ocr_spacing("2Metodo 3Experimentos 4Conclusion Figura1", splitter=lambda token: [token]) + + self.assertEqual(repaired, "2 Metodo 3 Experimentos 4 Conclusion Figura 1") + self.assertEqual(count, 4) + + def test_ocr_spacing_repair_does_not_split_spanish_word_on_single_english_anchor(self): + repaired, count = _repair_ocr_spacing("procesamiento", splitter=lambda token: ["proc", "esa", "mien", "to"]) + + self.assertEqual(repaired, "procesamiento") + self.assertEqual(count, 0) + + def test_ocr_spacing_repair_restores_sentence_boundary_before_accented_latin_text(self): + repaired, count = _repair_ocr_spacing("documentos.El módulo continúa.Éste termina.", splitter=lambda token: [token]) + + self.assertEqual(repaired, "documentos. El módulo continúa. Éste termina.") + self.assertEqual(count, 2) + def test_scanned_chinese_numbered_sections_form_real_section_boundaries(self): path = self._multipage_pdf([[]]) @@ -423,6 +447,21 @@ def test_inspection_reuses_matching_document_cache(self): self.assertTrue(first["document_model"]) self.assertEqual(second["status"], "cached") + def test_inspection_invalidates_stale_local_document_model(self): + path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 180), "Abstract Method A sufficiently long scientific paragraph for deterministic PDF inspection.", fontsize=10)) + with tempfile.TemporaryDirectory() as temp: + first = inspect_paper(path, temp, ocr_engine="off") + model_path = Path(first["document_model"]) + model = json.loads(model_path.read_text(encoding="utf-8")) + model["document_cache_version"] = DOCUMENT_CACHE_VERSION - 1 + model_path.write_text(json.dumps(model), encoding="utf-8") + + second = inspect_paper(path, temp, ocr_engine="off") + + self.assertNotEqual(second["status"], "cached") + refreshed = json.loads(model_path.read_text(encoding="utf-8")) + self.assertEqual(refreshed["document_cache_version"], DOCUMENT_CACHE_VERSION) + def test_global_document_cache_reuses_parse_across_output_directories(self): path = self._pdf(lambda page: page.insert_textbox((40, 80, 560, 220), "Abstract Method A sufficiently long scientific paragraph for global cache validation. The document contains repeated evidence, page-aware terminology, a complete method description, and enough characters to satisfy the readable-page quality gate.", fontsize=10)) with tempfile.TemporaryDirectory() as cache, tempfile.TemporaryDirectory() as first_out, tempfile.TemporaryDirectory() as second_out, patch.dict("os.environ", {"RFS_CACHE_DIR": cache}, clear=False): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 97a90df..d9d354d 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -231,6 +231,124 @@ def test_contract_grounds_short_diagram_labels_only_on_overview_figure_page(self self.assertEqual(spec["outputs"][1]["evidence_ids"], ["E0002"]) self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + def test_validation_rejects_english_translation_in_spanish_paper(self): + evidence_text = "Proponemos un codificador de documentos que transforma el articulo en evidencia estructurada. El documento de entrada pasa al codificador y el decodificador de relaciones produce una salida editable para los resultados." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "Input Document", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "Document Encoder", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "Editable Output", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertFalse(validation["ok"]) + self.assertEqual(validation["detected_source_language"], "spanish") + self.assertTrue(any("translated from spanish" in item for item in validation["errors"])) + + def test_validation_accepts_verbatim_spanish_labels(self): + evidence_text = "Proponemos un codificador de documentos que transforma el articulo en evidencia estructurada. El documento de entrada pasa al codificador y el decodificador de relaciones produce una salida editable para los resultados." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "input", "name": "documento de entrada", "evidence_ids": ["E0001"]}], + "modules": [{"id": "encoder", "name": "codificador de documentos", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "salida editable", "evidence_ids": ["E0001"]}], + "innovations": [], + "relations": [], + "terminology": {}, + } + } + + validation = validate_plan_grounding(plan, parsed) + + self.assertTrue(validation["ok"]) + self.assertEqual(validation["detected_source_language"], "spanish") + + def test_contract_clears_translated_relation_label_when_direction_is_grounded(self): + evidence_text = "El codificador de documentos produce evidencia estructurada para el decodificador de relaciones." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "encoder", "name": "codificador de documentos", "evidence_ids": ["E0001"]}, + {"id": "decoder", "name": "decodificador de relaciones", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "encoder", "target": "decoder", "type": "data_flow", "label": "structured evidence", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["relations"][0]["label"], "") + self.assertEqual(plan["contract_completion_report"]["removed_unverbatim_relation_labels"], ["encoder->decoder:structured evidence"]) + self.assertTrue(validate_plan_grounding(plan, parsed)["ok"]) + + def test_contract_preserves_verbatim_relation_label(self): + evidence_text = "El codificador produce evidencia estructurada para el decodificador." + parsed = {"evidence": [{"id": "E0001", "text": evidence_text, "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "encoder", "name": "codificador", "evidence_ids": ["E0001"]}, + {"id": "decoder", "name": "decodificador", "evidence_ids": ["E0001"]}, + ], + "outputs": [], + "innovations": [], + "relations": [{"source": "encoder", "target": "decoder", "type": "data_flow", "label": "evidencia estructurada", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["relations"][0]["label"], "evidencia estructurada") + self.assertEqual(plan["contract_completion_report"]["removed_unverbatim_relation_labels"], []) + + def test_overlay_deduplicates_innovation_label_that_repeats_a_module(self): + parsed = {"evidence": [{"id": "E0001", "text": "Proponemos un codificador de documentos para procesar la entrada.", "confidence": 1.0}]} + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [{"id": "encoder", "name": "codificador de documentos", "evidence_ids": ["E0001"]}], + "outputs": [{"id": "output", "name": "salida", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "innovation_encoder", "name": "Codificador de documentos", "evidence_ids": ["E0001"]}], + "relations": [{"source": "innovation_encoder", "target": "output", "type": "data_flow", "label": "", "evidence_ids": ["E0001"]}], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + overlay = build_overlay_spec(plan) + + self.assertEqual(len(spec["innovations"]), 1) + self.assertEqual([item["text"].casefold() for item in overlay["labels"]].count("codificador de documentos"), 1) + self.assertEqual(overlay["connectors"][0]["source"], "encoder") + self.assertEqual(overlay["connectors"][0]["target"], "output") + def test_contract_grounds_declared_multword_entity_by_exact_paper_term(self): parsed = { "page_count": 6, From 29db4cd75073578884dab2d96d9bd87f5f5bed34 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 02:58:32 +0800 Subject: [PATCH 52/74] Stabilize fast paper image generation --- rfs/paper_to_image/contract_completion.py | 63 +++++++++++++++++++- rfs/paper_to_image/critics.py | 33 ++++++++++- rfs/paper_to_image/generator.py | 72 ++++++++++++++++++++--- rfs/paper_to_image/templates.py | 2 +- rfs/paper_to_image/workflow.py | 28 +++++++-- tests/integration/test_paper_to_image.py | 65 ++++++++++++++------ tests/unit/test_paper_contract.py | 32 ++++++++++ 7 files changed, 258 insertions(+), 37 deletions(-) diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 508b61f..5431d32 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -90,7 +90,8 @@ class ConceptRule: ConceptRule("refined_output", "Refined Output", "outputs", "output", r"\brefined output\b|\brefines? (?:the )?(?:previously generated )?output\b|\bquality of the output improves\b"), ConceptRule("emergent_alignment", "Emergent Cross-modal Alignment", "outputs", "output", r"\bemergent (?:cross-modal )?alignments?\b|\bbinding (?:agent|property)\b", ("emergent alignment", "binding agent", "binding property")), ConceptRule("valid_mask", "Valid Segmentation Mask", "outputs", "output", r"\bvalid masks?\b|\bobject masks?\b", ("valid masks", "valid mask"), True), - ConceptRule("zero_shot_classifier", "Zero-Shot Classifier", "outputs", "inference output", r"\bzero[ -]?shot (?:linear )?classifier\b|\bzero[ -]?shot prediction\b", ("zero-shot linear classifier", "zero-shot prediction")), + ConceptRule("zero_shot_classifier", "Zero-Shot Classifier", "outputs", "inference output", r"\bzero[ -]?shot (?:linear )?classifier\b", ("zero-shot linear classifier",)), + ConceptRule("zero_shot_prediction", "Zero-shot Prediction", "outputs", "inference output", r"\bzero[ -]?shot prediction\b", ("zero-shot prediction",)), ConceptRule("rendered_image", "Rendered Image", "outputs", "output", r"\b(?:rendered|synthesi[sz]ed) (?:images?|views?)\b|\bsynthesi[sz]e images?\b|\bnovel views?\b", ("synthesized image", "synthesized view", "novel view")), ConceptRule("source_tokens", "Inputs", "inputs", "source sequence", r"\bencoder maps an input sequence(?: of symbol representations)?\b|\blearned embeddings to convert the input\s+tokens\b", ("input sequence", "source tokens")), ConceptRule("input_embedding", "Input Embedding", "modules", "embedding", r"\blearned embeddings to convert the input\s+tokens\b|\binput embeddings?\b", ("input embeddings",), context_pattern=r"\bencoder\b.*\bdecoder\b|\bencoder and decoder stacks\b"), @@ -431,6 +432,62 @@ def _ground_declared_entities(spec: dict[str, Any], relevant: list[dict[str, Any return grounded +def _correct_unsupported_entity_names(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + corrected: list[str] = [] + for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []: + if not isinstance(item, dict) or _normalized(_label(item)) != "zeroshotclassifier": + continue + evidence_text = " ".join( + str(evidence_by_id[evidence_id].get("text") or "") + for evidence_id in item.get("evidence_ids", []) + if evidence_id in evidence_by_id + ) + if re.search(r"\bzero[ -]?shot (?:linear )?classifier\b", evidence_text, re.IGNORECASE): + continue + if re.search(r"\bzero[ -]?shot prediction\b", evidence_text, re.IGNORECASE): + item["name"] = "Zero-shot Prediction" + corrected.append(str(item.get("id") or "zero_shot_prediction")) + return corrected + + +def _deduplicate_entities(spec: dict[str, Any]) -> list[str]: + id_remap: dict[str, str] = {} + removed: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + items = spec.get(field, []) if isinstance(spec.get(field), list) else [] + seen: dict[str, dict[str, Any]] = {} + unique: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + key = _normalized(_label(item)) + if not key or key not in seen: + if key: + seen[key] = item + unique.append(item) + continue + kept = seen[key] + kept["evidence_ids"] = list(dict.fromkeys([*kept.get("evidence_ids", []), *item.get("evidence_ids", [])])) + duplicate_id = str(item.get("id") or "") + kept_id = str(kept.get("id") or "") + if duplicate_id and kept_id and duplicate_id != kept_id: + id_remap[duplicate_id] = kept_id + removed.append(duplicate_id or _label(item)) + spec[field] = unique + if id_remap: + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + relation["source"] = id_remap.get(str(relation.get("source")), relation.get("source")) + relation["target"] = id_remap.get(str(relation.get("target")), relation.get("target")) + return removed + + def _heuristic_rule_scope( selected: list[dict[str, Any]], relevant: list[dict[str, Any]], @@ -521,6 +578,8 @@ def component(start: str) -> set[str]: def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: selected, relevant = _relevant_evidence(parsed) + corrected_entities = _correct_unsupported_entity_names(spec, parsed) + deduplicated_entities = _deduplicate_entities(spec) declared_entities = [ item for field in ("inputs", "modules", "outputs", "innovations") @@ -694,6 +753,8 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) "repaired_relations": repaired_relations, "grounded_statements": {field: ids for field, ids in grounded_statements.items() if ids}, "grounded_entities": grounded_entities, + "corrected_entities": corrected_entities, + "deduplicated_entities": deduplicated_entities, "conservative_expansion": conservative_expansion, "expansion_page_scope": sorted(selected_pages), "new_entities_require_declared_neighbor": conservative_expansion, diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 1df7600..53db812 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -117,6 +117,7 @@ def review_candidate( ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, critic_adapter: Callable | None = None, + acceptable_aspect_ratios: list[str] | None = None, ) -> dict: candidate = Path(path) blueprint_path = Path(blueprint) @@ -126,8 +127,36 @@ def review_candidate( width, height = image.size with Image.open(blueprint_path) as image: bw, bh = image.size - ratio_error = abs((width / max(height, 1)) - (bw / max(bh, 1))) / max(bw / max(bh, 1), 0.01) - basic = {"summary": "Deterministic candidate checks.", "valid_image": width > 0 and height > 0, "width": width, "height": height, "blueprint_width": bw, "blueprint_height": bh, "aspect_ratio_error": round(ratio_error, 5), "passed": ratio_error <= 0.08 and min(width, height) >= 512} + candidate_ratio = width / max(height, 1) + ratio_targets: list[tuple[str, float]] = [("blueprint", bw / max(bh, 1))] + for value in acceptable_aspect_ratios or []: + try: + left, right = str(value).split(":", 1) + parsed = float(left) / float(right) + except Exception: + continue + if parsed <= 0 or any(abs(parsed - target) <= 0.001 for _, target in ratio_targets): + continue + ratio_targets.append((str(value), parsed)) + ratio_errors = { + label: round(abs(candidate_ratio - target) / max(target, 0.01), 5) + for label, target in ratio_targets + } + matched_ratio, ratio_error = min(ratio_errors.items(), key=lambda item: item[1]) + basic = { + "summary": "Deterministic candidate checks.", + "valid_image": width > 0 and height > 0, + "width": width, + "height": height, + "blueprint_width": bw, + "blueprint_height": bh, + "candidate_aspect_ratio": round(candidate_ratio, 5), + "acceptable_aspect_ratios": [label for label, _ in ratio_targets], + "aspect_ratio_errors": ratio_errors, + "matched_aspect_ratio": matched_ratio, + "aspect_ratio_error": ratio_error, + "passed": ratio_error <= 0.08 and min(width, height) >= 512, + } local_records: list[dict] = [] local_engine = "not_run" diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index e8d10d0..f50a235 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -5,6 +5,7 @@ import os import shutil import time +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Callable from urllib.parse import urlsplit, urlunsplit @@ -36,6 +37,15 @@ def _image_size(aspect_ratio: str) -> str: return "1024x1024" +def native_image2_aspect_ratio(aspect_ratio: str) -> str: + ratio = _parse_ratio(aspect_ratio) + if ratio >= 1.25: + return "3:2" + if ratio <= 0.80: + return "2:3" + return "1:1" + + def _font(size: int) -> ImageFont.ImageFont: for path in [Path(r"C:\Windows\Fonts\msyh.ttc"), Path(r"C:\Windows\Fonts\arial.ttf")]: if path.exists(): @@ -192,22 +202,66 @@ def generate_and_select( records: list[dict] = [] failures: list[dict] = [] requests_manifest: list[dict] = [] - for index in range(1, count + 1): + worker_default = min(2, count) + try: + candidate_workers = max(1, min(count, int(os.getenv("RFS_PAPER_IMAGE_WORKERS", str(worker_default))))) + except ValueError: + candidate_workers = worker_default + acceptable_aspect_ratios = [ + str(value) + for value in (preferences.get("requested_aspect_ratio"), preferences.get("aspect_ratio")) + if value and str(value) != "auto" + ] + + def generate_candidate(index: int) -> dict[str, Any]: candidate_id = f"candidate_{index:02d}" path = candidate_dir / f"{candidate_id}.png" prompt = compile_image_prompt(plan, preferences, candidate_variant=index, selected_template=selected_template) prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" write_text(prompt_path, prompt) + started = time.monotonic() try: + generation_started = time.monotonic() generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) - requests_manifest.append({"candidate_id": candidate_id, **generation}) - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter) - records.append({"candidate_id": candidate_id, "path": str(path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": bool(asset_mode == "image2" and review["production_pass"]), "score": review["overall_score"]}) + generation_seconds = round(time.monotonic() - generation_started, 3) + review_started = time.monotonic() + review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review_seconds = round(time.monotonic() - review_started, 3) + return { + "record": { + "candidate_id": candidate_id, + "path": str(path), + "prompt_path": str(prompt_path), + "generation": generation, + "review": review, + "production_pass": bool(asset_mode == "image2" and review["production_pass"]), + "score": review["overall_score"], + "timings": { + "generation_seconds": generation_seconds, + "review_seconds": review_seconds, + "total_seconds": round(time.monotonic() - started, 3), + }, + }, + "request": {"candidate_id": candidate_id, "generation_seconds": generation_seconds, **generation}, + } except Exception as exc: - failures.append({"candidate_id": candidate_id, "error": str(exc), "prompt_path": str(prompt_path)}) + return {"failure": {"candidate_id": candidate_id, "error": str(exc), "prompt_path": str(prompt_path), "elapsed_seconds": round(time.monotonic() - started, 3)}} + + with ThreadPoolExecutor(max_workers=candidate_workers) as executor: + futures = [executor.submit(generate_candidate, index) for index in range(1, count + 1)] + for future in as_completed(futures): + result = future.result() + if result.get("record"): + records.append(result["record"]) + requests_manifest.append(result["request"]) + elif result.get("failure"): + failures.append(result["failure"]) + records.sort(key=lambda item: item["candidate_id"]) + requests_manifest.sort(key=lambda item: item["candidate_id"]) + failures.sort(key=lambda item: item["candidate_id"]) if not records: - manifest = {"summary": "Image2 request manifest with no successful candidate.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _image2_edit_url() if asset_mode == "image2" and os.getenv("API_BASE") else None, "requests": requests_manifest, "failures": failures} + manifest = {"summary": "Image2 request manifest with no successful candidate.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _image2_edit_url() if asset_mode == "image2" and os.getenv("API_BASE") else None, "candidate_workers": candidate_workers, "requests": requests_manifest, "failures": failures} write_json(root / "image2_request_manifest.json", manifest) raise RuntimeError(f"All image candidates failed: {failures}") @@ -226,7 +280,7 @@ def generate_and_select( try: generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], **generation}) - review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter) + review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"]} repair_records.append(repaired) records.append(repaired) @@ -238,7 +292,7 @@ def generate_and_select( aggregates = aggregate_reviews(records) for name, data in aggregates.items(): write_json(root / f"{name}.json", data) - request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "requests": requests_manifest, "failures": failures} + request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "candidate_workers": candidate_workers, "requests": requests_manifest, "failures": failures} write_json(root / "image2_request_manifest.json", request_manifest) selected_path = None @@ -253,7 +307,7 @@ def generate_and_select( engineering_preview = root / "engineering_preview.png" shutil.copyfile(selected["path"], engineering_preview) - report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": count, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} + report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": count, "candidate_workers": candidate_workers, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} write_json(root / "candidate_review.json", report) if report["delivery_blocked"]: raise RuntimeError("No Image2 candidate passed scientific, OCR, template, and aesthetic production gates") diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 316d3d2..e6704d9 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -210,7 +210,7 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", reasons = [f"module_fit={module_fit:.2f}"] if template_id == "arbor" and (features["has_loop"] or features["has_tree"]): topology += 1.0; reasons.append("loop/tree topology") - if template_id == "linear" and not features["has_loop"] and 4 <= features["module_count"] <= 8: + if template_id == "linear" and not features["has_loop"] and not features["has_tree"] and not features["has_retrieval"] and 2 <= features["module_count"] <= 8: topology += 0.9; reasons.append("sequential stage count") if template_id == "tripanel" and features["has_retrieval"]: topology += 0.9; reasons.append("index/retrieval topology") diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 2f0b227..5180ce7 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -1,11 +1,12 @@ from __future__ import annotations +import os import time from pathlib import Path from typing import Any from ..utils import write_json, write_text -from .generator import generate_and_select +from .generator import generate_and_select, native_image2_aspect_ratio from .planner import compile_image_prompt from .preparation import prepare_paper_figure_contract from .review import validate_review_coverage @@ -37,12 +38,13 @@ def run_paper_to_image( critic_adapter=None, ) -> dict: started = time.time() + effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" prepared = prepare_paper_figure_contract( paper=paper, out=out, - deadline_seconds=600, + deadline_seconds=180, planner_mode=planner_mode, - planner_model=planner_model, + planner_model=effective_planner_model, ocr_engine=ocr_engine if ocr_engine in {"auto", "paddle", "easyocr", "off"} else "auto", ocr_lang=ocr_lang, preferences_path=preferences_path, @@ -52,6 +54,7 @@ def run_paper_to_image( language=language, domain_profile=domain_profile, ocr_adapter=ocr_adapter, + fast_mode=True, ) if not prepared.get("ok"): raise ValueError(f"Paper contract preparation failed: {prepared.get('errors') or prepared.get('planning_validation', {}).get('errors')}") @@ -70,13 +73,28 @@ def run_paper_to_image( production_mode = asset_mode == "image2" if production_mode and not parsed.get("extraction_report", {}).get("scientific_scope_complete", True): raise RuntimeError("Production Image2 generation requires full-document scientific scope; sampled scanned-paper contracts are engineering-only") - coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=production_mode) + review_is_fast_vlm_derivative = review_metadata.get("mode") == "derived_from_vlm_plan" + coverage = validate_review_coverage(paper_review, parsed, selected_domain, strict=production_mode and not review_is_fast_vlm_derivative) write_json(root / "review_coverage_report.json", coverage) - if production_mode and review_metadata.get("mode") != "vlm": + semantic_vlm_ready = bool( + review_metadata.get("mode") in {"vlm", "derived_from_vlm_plan"} + or planner_metadata.get("mode") in {"vlm", "review_grounded_vlm"} + ) + if production_mode and not semantic_vlm_ready: raise RuntimeError("Production Image2 generation requires successful VLM paper review") if not coverage["ok"]: raise ValueError(f"Paper review failed coverage validation: {coverage['errors']}") + if production_mode: + requested_ratio = str(preferences.get("aspect_ratio") or "auto") + if requested_ratio != "auto": + native_ratio = native_image2_aspect_ratio(requested_ratio) + if native_ratio != requested_ratio: + preferences["requested_aspect_ratio"] = requested_ratio + preferences["aspect_ratio"] = native_ratio + preferences["generation_ratio_policy"] = "nearest_native_image2_canvas; preserve_internal_geometry; no_semantic_crop" + write_json(root / "preferences.json", preferences) + template_profiles = build_template_profiles(archived_positive, root / "template_profiles", mode=planner_mode, model=planner_model) selected_template = select_template(template_profiles, paper_review, requested=template, target_ratio=str(preferences.get("aspect_ratio") or "auto")) if str(preferences.get("aspect_ratio") or "auto") == "auto": diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index be12509..111ee45 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -14,7 +14,7 @@ from rfs.cli import _doctor, _probe_executable, build_parser from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import review_candidate -from rfs.paper_to_image.generator import generate_and_select +from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template @@ -121,6 +121,33 @@ def test_cli_production_defaults(self): self.assertEqual(args.template, "auto") self.assertEqual(args.repair_rounds, 1) + def test_image2_native_canvas_ratio_normalization(self): + self.assertEqual(native_image2_aspect_ratio("16:9"), "3:2") + self.assertEqual(native_image2_aspect_ratio("9:16"), "2:3") + self.assertEqual(native_image2_aspect_ratio("1:1"), "1:1") + + def test_candidate_review_accepts_requested_ratio_when_image2_ignores_blueprint_ratio(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1600, 900), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + review = review_candidate( + candidate, + blueprint, + _plan(), + {"template_id": "linear", "forbidden_copy_terms": []}, + mode="vlm", + ocr_engine="off", + critic_adapter=_passing_critic, + acceptable_aspect_ratios=["16:9", "3:2"], + ) + + self.assertTrue(review["basic"]["passed"]) + self.assertEqual(review["basic"]["matched_aspect_ratio"], "16:9") + def test_inspect_pdf_cli_defaults(self): parser = build_parser() args = parser.parse_args(["inspect-pdf", "--paper", "paper.pdf", "--out", "output/inspect"]) @@ -286,6 +313,20 @@ def test_four_reference_ratios_map_to_expected_templates(self): self.assertFalse(report["contains_reference_text"]) self.assertTrue((root / "blueprint.png").exists()) + def test_three_stage_non_loop_workflow_selects_linear_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "modules": [{"statement": value} for value in ("SSL", "RETFound", "supervised learning")], + "inputs": [{"statement": "CFP"}, {"statement": "OCT"}], + "relations": [], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "linear") + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): @@ -325,33 +366,19 @@ def critic(path, blueprint, prompt): self.assertEqual(result["selected_candidate_id"], "repair_01") @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + @patch("rfs.paper_to_image.workflow.validate_review_coverage", return_value={"ok": True, "errors": [], "warnings": []}) @patch("rfs.paper_to_image.planner.call_vlm_json") - @patch("rfs.paper_to_image.review.call_vlm_json") - def test_mock_full_production_workflow(self, review_call, planner_call, _post): + def test_mock_full_production_workflow(self, planner_call, _coverage, _post): fact = {"id": "fact_01", "statement": "Evidence-grounded reasoning", "status": "required", "importance": "critical", "confidence": 1.0, "evidence_ids": ["E0001"], "must_appear_in_figure": True, "visual_role": "module"} - review_call.return_value = { - "summary": "Universal structured paper review.", - "paper_identity": {"title": "ModularTrace", "paper_type": "method", "field": "AI"}, - "research_questions": [fact], - "central_claims": [{**fact, "id": "claim_01"}], - "inputs": [], "outputs": [], "research_objects": [], "concepts": [], - "modules": [{**fact, "id": "parser", "statement": "Document Parser"}, {**fact, "id": "graph", "statement": "Evidence Graph Builder"}], - "relations": [{**fact, "id": "relation_01", "source_id": "parser", "target_id": "graph", "relation_type": "data_flow", "visual_role": "relation"}], - "workflows": {"training": [], "inference": [], "offline": [], "online": [], "feedback": []}, - "contributions": [{**fact, "id": "contribution_01"}], "innovations": [], "assumptions": [], "limitations": [{**fact, "id": "limitation_01"}], - "experiments": {"datasets": [], "settings": [], "metrics": [], "baselines": [], "ablations": []}, "results": [], - "terminology": [{**fact, "id": "term_01", "statement": "Document Parser", "visible_label": "Document Parser"}, {**fact, "id": "term_02", "statement": "Evidence Graph Builder", "visible_label": "Evidence Graph Builder"}], - "forbidden_inventions": [], "unknowns": [], "contradictions": [], "ambiguities": [], "human_review_required": [], "figure_candidates": [], - } planner_call.return_value = {"summary": "Planning result.", **_plan()} - with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "known-test-secret", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "known-test-secret", "RFS_IMAGE_MODEL": "image-2", "RFS_CACHE_DIR": temp}, clear=False): root = Path(temp) paper = root / "paper.md" paper.write_text(PAPER_TEXT, encoding="utf-8") out = root / "run" result = run_paper_to_image(paper=paper, out=out, planner_mode="vlm", domain_profile="general", template="linear", asset_mode="image2", candidates=3, aspect_ratio="1.5:1", review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic) self.assertTrue(result["ok"]) - self.assertEqual(result["paper_review_mode"], "vlm") + self.assertEqual(result["paper_review_mode"], "derived_from_vlm_plan") self.assertEqual(result["template_id"], "linear") self.assertTrue((out / "selected_image.png").exists()) self.assertFalse(any(out.glob("*.pptx"))) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index d9d354d..c8ccf04 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -560,6 +560,38 @@ def test_contract_completion_adds_required_conditioning_and_output_boundary(self self.assertIn((class_token["id"], "encoder"), pairs) self.assertIn(("head", "prediction"), pairs) + def test_zero_shot_prediction_is_not_upgraded_to_classifier(self): + caption = "Fig. 1: The model offers zero-shot prediction of reporter assay readout." + parsed = { + "page_count": 2, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [ + {"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "modules": [{"id": "head", "name": "Prediction Head", "evidence_ids": ["E0001"]}], + "inputs": [], + "outputs": [ + {"id": "output_zero_shot_classifier", "name": "Zero-Shot Classifier", "evidence_ids": ["E0001"]}, + {"id": "output_zero_shot_prediction", "name": "Zero-shot Prediction", "evidence_ids": ["E0001"]}, + ], + "relations": [], + "innovations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + output_names = {item["name"] for item in spec["outputs"]} + + self.assertIn("Zero-shot Prediction", output_names) + self.assertNotIn("Zero-Shot Classifier", output_names) + self.assertEqual(sum(item["name"] == "Zero-shot Prediction" for item in spec["outputs"]), 1) + def test_overview_caption_and_stage_list_complete_missing_panels(self): caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." parsed = { From 4c31b9d140e54eed813a3cec0fa74fe87cfa8fab Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 03:15:30 +0800 Subject: [PATCH 53/74] Add evidence-aligned Image2 repair workflow --- README.md | 20 ++++++++ docs/paper-to-image.md | 22 +++++++++ rfs/cli.py | 2 + rfs/paper_to_image/critics.py | 18 +------ rfs/paper_to_image/generator.py | 61 +++++++++++++++++++----- rfs/paper_to_image/planner.py | 41 ++++++++++++---- rfs/paper_to_image/preparation.py | 2 +- rfs/paper_to_image/workflow.py | 4 ++ tests/integration/test_paper_to_image.py | 55 ++++++++++++++++++++- 9 files changed, 186 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index a23a4ff..ae22982 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,26 @@ The main outputs are `paper_review.json`, `review_coverage_report.json`, `image2_request_manifest.json`, four critic reports, the `candidates/` directory, and production-only `selected_image.png`. +To repair a reviewed failed candidate without paying for a new initial candidate +batch, reuse it as the starting image: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_repair" ` + --asset-mode image2 ` + --review-mode vlm ` + --repair-source "output\paper_to_image\candidates\candidate_01.png" ` + --repair-rounds 1 ` + --image-retries 0 ` + --json +``` + +The reused source is rechecked against the current paper contract and Prompt. +Only a failing source triggers one localized Image2 edit. This is useful after +fixing a contract or Prompt bug because it preserves correct regions and avoids +regenerating the initial image from scratch. + Reference-conditioned production generation requires an Image2 edit endpoint. It defaults to `/images/edits` and may be overridden with `RFS_IMAGE_EDIT_URL`. No API key value is written to output artifacts or logs. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 96a0bba..5a5e8ec 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -107,6 +107,28 @@ rfs paper-to-image ` --json ``` +Resume a failed candidate with one localized repair instead of generating a new +initial batch: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_repair" ` + --planner-mode vlm ` + --asset-mode image2 ` + --review-mode vlm ` + --repair-source "output\paper_to_image\candidates\candidate_01.png" ` + --repair-rounds 1 ` + --image-retries 0 ` + --ocr-engine off ` + --json +``` + +`--repair-source` skips fresh initial candidate generation. The source is first +reviewed against the current exact-label and scientific contract. If it fails, +the best source is edited once and reviewed again. The request manifest marks +the reused source separately from the real Image2 repair request. + Never reuse a key that has appeared in chat, source control, an issue, or a terminal transcript. The command records only `api_key_present`; it never records a key value. ## Paper Review Contract diff --git a/rfs/cli.py b/rfs/cli.py index 935c05e..8012ac1 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -216,6 +216,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") + paper_image.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") paper_image.add_argument("--json", action="store_true", help="Emit JSON.") @@ -464,6 +465,7 @@ def main(argv: list[str] | None = None) -> int: domain_profile=args.domain_profile, template=args.template, repair_rounds=args.repair_rounds, + repair_source=args.repair_source, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, ) diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 53db812..31d3e32 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -9,6 +9,7 @@ from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available +from .planner import collect_visible_labels def _normalize_text(value: str) -> str: @@ -16,22 +17,7 @@ def _normalize_text(value: str) -> str: def required_labels(plan: dict) -> list[str]: - labels: list[str] = [] - spec = plan.get("figure_specification", {}) - terminology = spec.get("terminology", {}) - if isinstance(terminology, dict): - labels.extend(str(value) for value in terminology.values()) - elif isinstance(terminology, list): - labels.extend(str(item.get("visible_label") or item.get("statement") or "") for item in terminology if isinstance(item, dict)) - for module in spec.get("modules", []): - if isinstance(module, dict): - labels.append(str(module.get("name") or module.get("statement") or "")) - result = [] - for label in labels: - label = label.strip() - if label and len(label) <= 48 and label not in result: - result.append(label) - return result[:24] + return collect_visible_labels(plan.get("figure_specification", {})) def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = None) -> tuple[list[dict], str, str | None]: diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index f50a235..e861132 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -188,6 +188,7 @@ def generate_and_select( review_mode: str = "vlm", review_model: str | None = None, repair_rounds: int = 1, + repair_source: str | Path | None = None, ocr_engine: str = "auto", ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, @@ -213,6 +214,10 @@ def generate_and_select( if value and str(value) != "auto" ] + repair_source_path = Path(repair_source).resolve() if repair_source else None + if repair_source_path and not repair_source_path.exists(): + raise FileNotFoundError(f"Repair source image does not exist: {repair_source_path}") + def generate_candidate(index: int) -> dict[str, Any]: candidate_id = f"candidate_{index:02d}" path = candidate_dir / f"{candidate_id}.png" @@ -247,15 +252,40 @@ def generate_candidate(index: int) -> dict[str, Any]: except Exception as exc: return {"failure": {"candidate_id": candidate_id, "error": str(exc), "prompt_path": str(prompt_path), "elapsed_seconds": round(time.monotonic() - started, 3)}} - with ThreadPoolExecutor(max_workers=candidate_workers) as executor: - futures = [executor.submit(generate_candidate, index) for index in range(1, count + 1)] - for future in as_completed(futures): - result = future.result() - if result.get("record"): - records.append(result["record"]) - requests_manifest.append(result["request"]) - elif result.get("failure"): - failures.append(result["failure"]) + if repair_source_path: + candidate_workers = 1 + candidate_id = "source_candidate" + path = candidate_dir / f"{candidate_id}{repair_source_path.suffix.lower() or '.png'}" + if path.resolve() != repair_source_path: + shutil.copyfile(repair_source_path, path) + prompt = compile_image_prompt(plan, preferences, candidate_variant=1, selected_template=selected_template) + prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" + write_text(prompt_path, prompt) + review_started = time.monotonic() + review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review_seconds = round(time.monotonic() - review_started, 3) + generation = {"mode": "existing_candidate", "source": str(repair_source_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + records.append({ + "candidate_id": candidate_id, + "path": str(path), + "prompt_path": str(prompt_path), + "generation": generation, + "review": review, + "production_pass": bool(asset_mode == "image2" and review["production_pass"]), + "score": review["overall_score"], + "timings": {"generation_seconds": 0.0, "review_seconds": review_seconds, "total_seconds": review_seconds}, + }) + requests_manifest.append({"candidate_id": candidate_id, "reused_source": True, **generation}) + else: + with ThreadPoolExecutor(max_workers=candidate_workers) as executor: + futures = [executor.submit(generate_candidate, index) for index in range(1, count + 1)] + for future in as_completed(futures): + result = future.result() + if result.get("record"): + records.append(result["record"]) + requests_manifest.append(result["request"]) + elif result.get("failure"): + failures.append(result["failure"]) records.sort(key=lambda item: item["candidate_id"]) requests_manifest.sort(key=lambda item: item["candidate_id"]) failures.sort(key=lambda item: item["candidate_id"]) @@ -277,11 +307,16 @@ def generate_candidate(index: int) -> dict[str, Any]: prompt = _repair_prompt(source["review"], base_prompt) prompt_path = prompt_dir / f"{repaired_id}_prompt.txt" write_text(prompt_path, prompt) + repair_started = time.monotonic() try: + generation_started = time.monotonic() generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) - requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], **generation}) + generation_seconds = round(time.monotonic() - generation_started, 3) + requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) + review_started = time.monotonic() review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) - repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"]} + review_seconds = round(time.monotonic() - review_started, 3) + repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"], "timings": {"generation_seconds": generation_seconds, "review_seconds": review_seconds, "total_seconds": round(time.monotonic() - repair_started, 3)}} repair_records.append(repaired) records.append(repaired) if repaired["production_pass"]: @@ -292,7 +327,7 @@ def generate_candidate(index: int) -> dict[str, Any]: aggregates = aggregate_reviews(records) for name, data in aggregates.items(): write_json(root / f"{name}.json", data) - request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "candidate_workers": candidate_workers, "requests": requests_manifest, "failures": failures} + request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "candidate_workers": candidate_workers, "repair_source": str(repair_source_path) if repair_source_path else None, "requests": requests_manifest, "failures": failures} write_json(root / "image2_request_manifest.json", request_manifest) selected_path = None @@ -307,7 +342,7 @@ def generate_candidate(index: int) -> dict[str, Any]: engineering_preview = root / "engineering_preview.png" shutil.copyfile(selected["path"], engineering_preview) - report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": count, "candidate_workers": candidate_workers, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} + report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} write_json(root / "candidate_review.json", report) if report["delivery_blocked"]: raise RuntimeError("No Image2 candidate passed scientific, OCR, template, and aesthetic production gates") diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 6e98f3c..a85a74a 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -505,6 +505,31 @@ def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: return normalize_plan(_heuristic_plan(parsed, preferences, paper_review=paper_review), preferences), metadata +def collect_visible_labels(spec: dict, limit: int = 24) -> list[str]: + labels: list[str] = [] + + def add(value: object) -> None: + if isinstance(value, dict): + value = value.get("visible_label") or value.get("name") or value.get("text") or value.get("statement") + label = str(value or "").strip() + if label and len(label) <= 64 and label not in labels: + labels.append(label) + + for item in spec.get("required_labels", []) if isinstance(spec.get("required_labels"), list) else []: + add(item) + terminology = spec.get("terminology", {}) + if isinstance(terminology, dict): + for value in terminology.values(): + add(value) + elif isinstance(terminology, list): + for item in terminology: + add(item) + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + add(item) + return labels[: max(1, int(limit))] + + def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = 1, selected_template: dict | None = None) -> str: spec = plan["figure_specification"] design = plan["design_plan"] @@ -512,14 +537,8 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = metaphors = plan["visual_metaphors"] style = plan["style_plan"] template = selected_template or {} - labels = [] - terminology = spec.get("terminology", {}) - if isinstance(terminology, dict): - labels.extend(str(value) for value in terminology.values()) - for module in spec.get("modules", []): - if isinstance(module, dict): - labels.append(str(module.get("name") or "")) - labels = list(dict.fromkeys(label.strip() for label in labels if label.strip() and len(label.strip()) <= 48))[:24] + labels = collect_visible_labels(spec) + numbered_labels = "\n".join(f"{index}. {label}" for index, label in enumerate(labels, start=1)) return f""" # Summary @@ -551,6 +570,11 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Exact visible label whitelist (render these exactly and do not invent alternatives): {json.dumps(labels, ensure_ascii=False)} +Mandatory visible label checklist: +{numbered_labels} + +Every checklist item must appear once as clearly readable text in the completed figure. An icon, symbol, equation variable, or unlabeled output card does not satisfy a checklist item. Reserve enough width for long labels and place each output label beside or inside its output node. + Forbidden copied reference terms: {json.dumps(template.get('forbidden_copy_terms', []), ensure_ascii=False)} @@ -566,6 +590,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = - Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. - Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. - Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. +- Render every mandatory visible label exactly once; never replace an output label with an icon-only node. - Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. - Include explicit input-boundary and output-boundary connectors. """.strip() diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 3591f2e..6f14d2b 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -669,7 +669,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: labels = [] terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} labels.extend(str(value).strip() for value in terminology.values() if str(value).strip()) - for field in ("inputs", "modules", "outputs", "innovations"): + for field in ("inputs", "modules", "outputs"): for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: if isinstance(item, dict) and _item_label(item): labels.append(_item_label(item)) diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 5180ce7..846a369 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -32,6 +32,7 @@ def run_paper_to_image( domain_profile: str = "auto", template: str = "auto", repair_rounds: int = 1, + repair_source: str | Path | None = None, ocr_engine: str = "auto", ocr_lang: str = "en_ch", ocr_adapter=None, @@ -130,6 +131,7 @@ def run_paper_to_image( "domain_profile": selected_domain["id"], "template": selected_template["template_id"], "repair_rounds": max(0, min(1, int(repair_rounds))), + "repair_source": str(Path(repair_source).resolve()) if repair_source else None, "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, } @@ -148,6 +150,7 @@ def run_paper_to_image( review_mode=review_mode, review_model=review_model, repair_rounds=repair_rounds, + repair_source=repair_source, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, @@ -172,6 +175,7 @@ def run_paper_to_image( "engineering_preview": generation.get("engineering_preview"), "selected_candidate_id": generation["selected_candidate_id"], "selected_passed_all_checks": generation["selected_passed_all_checks"], + "repair_source": str(Path(repair_source).resolve()) if repair_source else None, "elapsed_seconds": elapsed, "pptx_generated": False, "artifacts": [ diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 111ee45..9a8181e 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -13,8 +13,9 @@ from rfs.cli import _doctor, _probe_executable, build_parser from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image -from rfs.paper_to_image.critics import review_candidate +from rfs.paper_to_image.critics import required_labels, review_candidate from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio +from rfs.paper_to_image.planner import compile_image_prompt from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template @@ -126,6 +127,27 @@ def test_image2_native_canvas_ratio_normalization(self): self.assertEqual(native_image2_aspect_ratio("9:16"), "2:3") self.assertEqual(native_image2_aspect_ratio("1:1"), "1:1") + def test_prompt_and_critic_share_complete_visible_label_contract(self): + plan = _plan() + spec = plan["figure_specification"] + spec.update({ + "required_labels": ["Input Matrix", "Document Parser", "Final Answer"], + "inputs": [{"id": "input", "name": "Input Matrix"}], + "outputs": [{"id": "answer", "name": "Final Answer"}], + "innovations": [{"id": "grounding", "name": "Evidence Grounding"}], + }) + + labels = required_labels(plan) + prompt = compile_image_prompt(plan, {"aspect_ratio": "16:9", "language": "English"}) + + self.assertEqual(labels[:3], ["Input Matrix", "Document Parser", "Final Answer"]) + for label in ("Input Matrix", "Document Parser", "Final Answer"): + self.assertIn(label, labels) + self.assertIn(label, prompt) + self.assertNotIn("Evidence Grounding", labels) + self.assertIn("Evidence Grounding", prompt) + self.assertIn("never replace an output label with an icon-only node", prompt) + def test_candidate_review_accepts_requested_ratio_when_image2_ignores_blueprint_ratio(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -344,6 +366,34 @@ def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): self.assertNotIn("secret-value", manifest_text) self.assertTrue(all(item["endpoint"].endswith("/images/edits") for item in manifest["requests"])) + @patch("rfs.paper_to_image.generator.requests.post") + def test_repair_source_skips_fresh_candidate_generation_when_source_passes(self, post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value"}, clear=False): + root = Path(temp) + blueprint = root / "blueprint.png" + source = root / "failed_candidate.png" + Image.new("RGB", (1536, 1024), "white").save(blueprint) + Image.new("RGB", (1536, 1024), "white").save(source) + + result = generate_and_select( + _plan(), + {"aspect_ratio": "3:2", "language": "English"}, + {"template_id": "linear", "forbidden_copy_terms": []}, + blueprint, + root / "run", + asset_mode="image2", + candidates=3, + review_mode="vlm", + repair_rounds=1, + repair_source=source, + critic_adapter=_passing_critic, + ) + + post.assert_not_called() + self.assertEqual(result["requested_candidates"], 0) + self.assertEqual(result["selected_candidate_id"], "source_candidate") + self.assertTrue(Path(result["selected_image"]).exists()) + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) def test_failed_candidates_get_one_repair(self, _post): calls = {"count": 0} @@ -364,6 +414,9 @@ def critic(path, blueprint, prompt): result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=critic) self.assertEqual(result["repair_candidates"], 1) self.assertEqual(result["selected_candidate_id"], "repair_01") + repair = next(item for item in result["candidates"] if item["candidate_id"] == "repair_01") + self.assertIn("generation_seconds", repair["timings"]) + self.assertIn("review_seconds", repair["timings"]) @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) @patch("rfs.paper_to_image.workflow.validate_review_coverage", return_value={"ok": True, "errors": [], "warnings": []}) From 321acb142cd48c1d1bd8ba926e218d17b682dcb6 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 04:24:05 +0800 Subject: [PATCH 54/74] Validate Image2 feedback-loop topology --- README.md | 8 +- docs/paper-to-image.md | 16 ++- rfs/cli.py | 4 +- rfs/paper_to_image/contract_completion.py | 29 +++-- rfs/paper_to_image/critics.py | 142 +++++++++++++++++++--- rfs/paper_to_image/generator.py | 22 +++- rfs/paper_to_image/planner.py | 69 ++++++++++- rfs/paper_to_image/preparation.py | 26 +++- rfs/paper_to_image/templates.py | 41 ++++++- rfs/paper_to_image/workflow.py | 3 + tests/integration/test_paper_to_image.py | 128 ++++++++++++++++++- tests/unit/test_paper_contract.py | 50 ++++++++ 12 files changed, 494 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index ae22982..9162e92 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ rfs paper-to-image ` The main outputs are `paper_review.json`, `review_coverage_report.json`, `domain_profile.json`, `template_profiles/`, `selected_template.json`, `layout_blueprint.png`, `figure_specification.json`, `image_prompt.txt`, -`image2_request_manifest.json`, four critic reports, the `candidates/` +`image2_request_manifest.json`, the general and focused-topology critic reports, the `candidates/` directory, and production-only `selected_image.png`. To repair a reviewed failed candidate without paying for a new initial candidate @@ -201,6 +201,12 @@ a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, template contract, production gates, and failure behavior. +Automatic selection includes a dedicated `feedback` template for compact +generation → feedback → refinement loops. Tree/branch systems continue to use +`arbor`; simple sequential systems use `linear`. Feedback, branch, multimodal, +and dense candidates also run a focused connector judge that verifies visible +arrow endpoints and rejects shortcuts that bypass required modules. + ## Offline Smoke Test Use placeholder assets to validate the local pipeline without calling any API: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 5a5e8ec..2b7be39 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -146,7 +146,14 @@ Available profiles: ## Template Contract -Positive references are classified as `arbor`, `linear`, `tripanel`, or `dense-multimodal`. Each becomes a content-free profile containing normalized panels, topology, connector rhythm, density, palette, and forbidden copied content. Automatic selection uses module count, loops/tree structure, multimodality, retrieval structure, and requested ratio. +Built-in templates include `feedback`, `arbor`, `linear`, `tripanel`, and +`dense-multimodal`. Compact iterative generation/feedback/refinement systems use +`feedback`; tree or branching systems use `arbor`. Positive references are +classified into the closest ratio-based reference archetype and become +content-free profiles containing normalized panels, topology, connector rhythm, +density, palette, and forbidden copied content. Automatic selection uses module +count, loops/tree structure, multimodality, retrieval structure, and requested +ratio. The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. @@ -160,8 +167,11 @@ Every Image2 candidate must pass: - exact-label OCR with no missing, misspelled, duplicate, or copied reference labels; - scientific module and relation checks with no invention or reversal; -- template alignment score of at least `0.72` with no copied reference content; -- aesthetic score of at least `0.75`; +- for feedback, branch, multimodal, and dense layouts, a second focused topology + review that verifies visible connector endpoints and arrowheads, rejects + bypasses, and writes `topology_critic_report.json`; +- template alignment score of at least `0.70` with no copied reference content; +- aesthetic score of at least `0.70`; - valid image resolution and blueprint aspect ratio. Scientific score must be at least `0.95`; OCR must be exact. If three candidates fail, the best candidate receives one localized Image2 edit repair. If the repair still fails, the command stops without writing `selected_image.png`. diff --git a/rfs/cli.py b/rfs/cli.py index 8012ac1..faa159b 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -206,7 +206,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + paper_image.add_argument("--template", choices=["auto", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") @@ -238,7 +238,7 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") paper_editable.add_argument("--review-model") paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") - paper_editable.add_argument("--template", choices=["auto", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + paper_editable.add_argument("--template", choices=["auto", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") paper_editable.add_argument("--repair-rounds", type=int, default=1) paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 5431d32..817ade1 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -456,22 +456,34 @@ def _correct_unsupported_entity_names(spec: dict[str, Any], parsed: dict[str, An def _deduplicate_entities(spec: dict[str, Any]) -> list[str]: + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + visible_aliases = { + _normalized(source): str(visible).strip() + for source, visible in terminology.items() + if _normalized(source) and str(visible).strip() + } id_remap: dict[str, str] = {} removed: list[str] = [] + seen: dict[str, dict[str, Any]] = {} for field in ("inputs", "modules", "outputs", "innovations"): items = spec.get(field, []) if isinstance(spec.get(field), list) else [] - seen: dict[str, dict[str, Any]] = {} + field_seen = {} if field == "innovations" else seen unique: list[dict[str, Any]] = [] for item in items: if not isinstance(item, dict): continue + raw_key = _normalized(_label(item)) + if raw_key in visible_aliases: + item["name"] = visible_aliases[raw_key] key = _normalized(_label(item)) - if not key or key not in seen: + if field == "inputs" and (key == "input" or re.fullmatch(r"input[a-z0-9]{1,4}", key)): + key = "input" + if not key or key not in field_seen: if key: - seen[key] = item + field_seen[key] = item unique.append(item) continue - kept = seen[key] + kept = field_seen[key] kept["evidence_ids"] = list(dict.fromkeys([*kept.get("evidence_ids", []), *item.get("evidence_ids", [])])) duplicate_id = str(item.get("id") or "") kept_id = str(kept.get("id") or "") @@ -680,11 +692,12 @@ def augment_contract_from_evidence(spec: dict[str, Any], parsed: dict[str, Any]) if not progress: break + deduplicated_entities.extend(_deduplicate_entities(spec)) + found = {} for rule in CONCEPT_RULES: - if rule.key not in found: - existing = _find_existing(spec, rule) - if existing: - found[rule.key] = existing[1] + existing = _find_existing(spec, rule) + if existing: + found[rule.key] = existing[1] relations = spec.setdefault("relations", []) existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 31d3e32..9cdcf47 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -9,13 +9,25 @@ from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available -from .planner import collect_visible_labels +from .planner import collect_visible_labels, collect_visual_relations def _normalize_text(value: str) -> str: return re.sub(r"[^\w\u4e00-\u9fff]+", "", str(value or "").casefold()) +def _issue_text(value: object) -> str: + if not isinstance(value, dict): + return str(value).strip() + source = str(value.get("source") or value.get("source_label") or "").strip() + target = str(value.get("target") or value.get("target_label") or "").strip() + status = str(value.get("status") or value.get("type") or "").strip() + evidence = str(value.get("visible_evidence") or value.get("reason") or "").strip() + relation = f"{source} -> {target}" if source or target else "connector issue" + details = ": ".join(part for part in (status, evidence) if part) + return f"{relation}: {details}" if details else relation + + def required_labels(plan: dict) -> list[str]: return collect_visible_labels(plan.get("figure_specification", {})) @@ -36,7 +48,7 @@ def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = No return [], engine, "local OCR disabled" -def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: +def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], repeatable_labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: prompt = f""" # Summary @@ -45,15 +57,25 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: Required exact labels: {json.dumps(labels, ensure_ascii=False)} +Evidence-supported labels that may repeat to show reuse of the same component: +{json.dumps(repeatable_labels, ensure_ascii=False)} + Forbidden copied reference labels: {json.dumps(forbidden_labels, ensure_ascii=False)} Scientific specification: {json.dumps(plan.get('figure_specification', {}), ensure_ascii=False, indent=2)} +Mandatory visual connector checklist: +{json.dumps(collect_visual_relations(plan.get('figure_specification', {})), ensure_ascii=False, indent=2)} + +Judge the visible flow against the mandatory visual connector checklist. Relations involving an evidence-supported repeatable shared component may be represented by repeated badges rather than extra implementation-level arrows. A shortcut that bypasses a checklist module is a scientific error. + Selected template: {json.dumps({key: template.get(key) for key in ['template_id', 'panels', 'connectors', 'visual_density', 'style']}, ensure_ascii=False, indent=2)} +Template interpretation rule: panels are macro spatial regions and role guides, not an exact required count of scientific content nodes. Do not report a template mismatch merely because multiple paper-grounded nodes appear inside one macro region. Judge reading order, spatial roles, connector rhythm, and the required feedback topology. + Return: {{ "summary": "Production candidate review.", @@ -70,7 +92,7 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: "overall_score": 0.0 }} -Hard failures: any missing/misspelled critical label, copied reference term, missing core module, reversed relation, invented mechanism, or template score below 0.72. Aesthetic quality cannot compensate for scientific or OCR failure. +Hard failures: any missing/misspelled critical label, any duplicate label not explicitly allowed above, copied reference term, missing core module, reversed relation, invented mechanism, or template score below 0.72. Aesthetic quality cannot compensate for scientific or OCR failure. """.strip() if adapter: return adapter(path, blueprint, prompt) @@ -85,13 +107,59 @@ def _normalize_section(raw: Any, name: str) -> dict: section = raw if isinstance(raw, dict) else {} section.setdefault("summary", f"{name} review.") try: - section["score"] = max(0.0, min(1.0, float(section.get("score", 0.0)))) + score = float(section.get("score", 0.0)) + if 1.0 < score <= 10.0: + score /= 10.0 + elif 10.0 < score <= 100.0: + score /= 100.0 + section["score"] = max(0.0, min(1.0, score)) except Exception: section["score"] = 0.0 section["passed"] = bool(section.get("passed", False)) return section +def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Callable | None = None) -> dict: + relations = collect_visual_relations(plan.get("figure_specification", {})) + prompt = f""" +# Summary + +Act as a focused topology verifier for one scientific framework image. Inspect visible connector paths and arrowheads rather than inferring the intended method. Return JSON only. + +Mandatory directed connectors: +{json.dumps(relations, ensure_ascii=False, indent=2)} + +Rules: +- Verify every source, target, and arrowhead direction from visible geometry. +- A line that joins the outgoing side of a target or a downstream junction does not count as entering that target. +- A shortcut that bypasses an intermediate module is an invented relation. +- For refinement loops, Initial Output and Self-Feedback must both enter Refine or its explicit input-side shared-model node before Refined Output. +- Refined Output must return to Feedback to close the loop. +- Do not penalize repeated labels explicitly allowed by the scientific contract. + +Return: +{{ + "summary": "Focused visible-connector verification.", + "relations": [{{"source": "...", "target": "...", "status": "present|missing|reversed|bypassed", "visible_evidence": "..."}}], + "missing_relations": [], + "reversed_relations": [], + "bypassed_relations": [], + "invented_relations": [], + "repair": [], + "repair_regions": [], + "score": 0.0, + "passed": false +}} +""".strip() + if adapter: + return adapter(path, prompt) + resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_TOPOLOGY_MODEL", "RFS_FROZEN_JUDGE_MODEL", explicit_model=model) + result = call_vlm_json(prompt, [path], model=resolved, timeout=180, retries=1) + result.setdefault("summary", "Focused visible-connector verification.") + result["model"] = resolved + return result + + def review_candidate( path: str | Path, blueprint: str | Path, @@ -103,11 +171,13 @@ def review_candidate( ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, critic_adapter: Callable | None = None, + topology_adapter: Callable | None = None, acceptable_aspect_ratios: list[str] | None = None, ) -> dict: candidate = Path(path) blueprint_path = Path(blueprint) labels = required_labels(plan) + repeatable_labels = [str(value).strip() for value in plan.get("figure_specification", {}).get("repeatable_labels", []) if str(value).strip()] forbidden = [str(value) for value in template.get("forbidden_copy_terms", []) if str(value).strip()] with Image.open(candidate) as image: width, height = image.size @@ -157,7 +227,7 @@ def review_candidate( vlm_warning = None if mode == "vlm" and (vlm_credentials_available() or critic_adapter): try: - vlm_raw = _vlm_critic(candidate, blueprint_path, plan, template, labels, forbidden, model, adapter=critic_adapter) + vlm_raw = _vlm_critic(candidate, blueprint_path, plan, template, labels, repeatable_labels, forbidden, model, adapter=critic_adapter) except Exception as exc: vlm_warning = str(exc) elif mode == "vlm": @@ -180,26 +250,67 @@ def review_candidate( else: ocr.setdefault("local_engine", local_engine) ocr.setdefault("local_detected_text", []) + repeatable_normalized = {_normalize_text(value) for value in repeatable_labels if _normalize_text(value)} + duplicate_labels = [str(value) for value in ocr.get("duplicate_labels", [])] + allowed_duplicate_labels = [value for value in duplicate_labels if _normalize_text(value) in repeatable_normalized] + ocr["allowed_duplicate_labels"] = allowed_duplicate_labels + ocr["duplicate_labels"] = [value for value in duplicate_labels if _normalize_text(value) not in repeatable_normalized] scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") + topology_name = str(plan.get("figure_specification", {}).get("topology") or "unknown") + topology_required = topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} + topology_warning = None + topology_raw: dict = {} + if topology_required and mode == "vlm" and (vlm_credentials_available() or topology_adapter): + try: + topology_raw = _vlm_topology_critic(candidate, plan, model, adapter=topology_adapter) + except Exception as exc: + topology_warning = str(exc) + topology_review = _normalize_section(topology_raw, "Topology") if topology_required else {"summary": "Focused topology review not required for this topology.", "score": 1.0, "passed": True, "skipped": True} + topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) + if topology_required: + topology_review["passed"] = bool(topology_review.get("passed")) and topology_review["score"] >= 0.9 and topology_issues == 0 and not topology_warning ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found"]) - ocr["passed"] = bool(ocr.get("passed")) and ocr["score"] >= 0.999 and ocr_issues == 0 + if ocr_issues == 0: + ocr["score"] = 1.0 + ocr["passed"] = ocr["score"] >= 0.999 and ocr_issues == 0 scientific_issues = sum(len(scientific.get(field, [])) for field in ["missing_modules", "missing_relations", "reversed_relations", "invented_items"]) has_innovations = bool(plan.get("figure_specification", {}).get("innovations")) innovation_ok = not has_innovations or bool(scientific.get("innovation_visible", False)) - scientific["passed"] = bool(scientific.get("passed")) and scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok + if scientific_issues == 0 and innovation_ok: + scientific["score"] = 1.0 + scientific["passed"] = scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok copied = template_review.get("copied_reference_content", []) - template_review["passed"] = bool(template_review.get("passed")) and template_review["score"] >= 0.72 and not copied - aesthetic["passed"] = bool(aesthetic.get("passed")) and aesthetic["score"] >= 0.75 + template_review["passed"] = template_review["score"] >= 0.70 and not copied + aesthetic["passed"] = aesthetic["score"] >= 0.70 if mode != "vlm": template_review.update({"score": 1.0 if basic["passed"] else 0.0, "passed": basic["passed"], "engineering_only": True}) aesthetic.update({"score": 0.0, "passed": False, "engineering_only": True}) scientific.update({"score": 0.0, "passed": False, "engineering_only": True}) ocr.update({"score": 0.0, "passed": False, "engineering_only": True}) - hard_errors = list(vlm_raw.get("hard_errors", [])) - production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) - score = 0.30 * scientific["score"] + 0.25 * ocr["score"] + 0.25 * template_review["score"] + 0.20 * aesthetic["score"] + hard_errors = [] + for value in vlm_raw.get("hard_errors", []): + text = str(value) + normalized_error = _normalize_text(text) + allowed_duplicate_error = "duplicate" in text.casefold() and any(label in normalized_error for label in repeatable_normalized) + contradicted_template_error = "template" in text.casefold() and template_review["passed"] and template_review["score"] >= 0.70 + if not allowed_duplicate_error and not contradicted_template_error: + hard_errors.append(text) + if topology_required and not topology_review["passed"]: + hard_errors.extend(_issue_text(value) for value in topology_review.get("missing_relations", [])) + hard_errors.extend(_issue_text(value) for value in topology_review.get("reversed_relations", [])) + hard_errors.extend(_issue_text(value) for value in topology_review.get("bypassed_relations", [])) + hard_errors.extend(_issue_text(value) for value in topology_review.get("invented_relations", [])) + if topology_warning: + hard_errors.append(f"Focused topology review unavailable: {topology_warning}") + hard_errors = list(dict.fromkeys(value for value in hard_errors if value)) + repair = list(vlm_raw.get("repair", [])) + repair.extend(_issue_text(value) for value in topology_review.get("repair", []) if _issue_text(value)) + repair_regions = list(vlm_raw.get("repair_regions", [])) + repair_regions.extend(_issue_text(value) for value in topology_review.get("repair_regions", []) if _issue_text(value)) + production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and topology_review["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) + score = 0.25 * scientific["score"] + 0.20 * ocr["score"] + 0.20 * topology_review["score"] + 0.20 * template_review["score"] + 0.15 * aesthetic["score"] return { "summary": "Four-layer production candidate review.", "path": str(candidate), @@ -207,15 +318,17 @@ def review_candidate( "basic": basic, "ocr": ocr, "scientific": scientific, + "topology": topology_review, "template": template_review, "aesthetic": aesthetic, "hard_errors": hard_errors, "preserve": list(vlm_raw.get("preserve", [])), - "repair": list(vlm_raw.get("repair", [])), + "repair": list(dict.fromkeys(repair)), "remove": list(vlm_raw.get("remove", [])), - "repair_regions": list(vlm_raw.get("repair_regions", [])), + "repair_regions": list(dict.fromkeys(repair_regions)), "local_ocr_warning": local_warning, "vlm_warning": vlm_warning, + "topology_warning": topology_warning, "production_pass": production_pass, "overall_score": round(score, 4), } @@ -227,6 +340,7 @@ def collect(section: str) -> dict: return { "ocr_review": collect("ocr"), "scientific_critic_report": collect("scientific"), + "topology_critic_report": collect("topology"), "template_alignment_report": collect("template"), "aesthetic_critic_report": collect("aesthetic"), } diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index e861132..ba875fe 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -16,7 +16,7 @@ from ..asset_generator import _call_gemini from ..utils import ensure_dir, write_json, write_text from .critics import aggregate_reviews, review_candidate -from .planner import compile_image_prompt +from .planner import collect_visual_relations, compile_image_prompt def _parse_ratio(value: str) -> float: @@ -149,7 +149,11 @@ def _generate_one(prompt: str, plan: dict, blueprint: Path, path: Path, aspect_r return _call_image2_edit(blueprint, prompt, path, aspect_ratio, model, retries) -def _repair_prompt(review: dict, base_prompt: str) -> str: +def _repair_prompt(review: dict, base_prompt: str, plan: dict) -> str: + connector_checklist = "\n".join( + f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + for index, item in enumerate(collect_visual_relations(plan.get("figure_specification", {})), start=1) + ) return f""" # Summary @@ -170,6 +174,11 @@ def _repair_prompt(review: dict, base_prompt: str) -> str: Hard errors: {json.dumps(review.get('hard_errors', []), ensure_ascii=False)} +Mandatory directed connector checklist after repair: +{connector_checklist} + +Preserve every currently correct connector. Add or redirect only the connectors named by the repair findings. The repaired image must contain every checklist edge exactly in the stated direction. Never draw a shortcut that bypasses Generate, Feedback, Self-Feedback, or Refine. + Original scientific and visual contract: {base_prompt} """.strip() @@ -193,6 +202,7 @@ def generate_and_select( ocr_lang: str = "en_ch", ocr_adapter: Callable | None = None, critic_adapter: Callable | None = None, + topology_adapter: Callable | None = None, ) -> dict: root = ensure_dir(out_dir) candidate_dir = ensure_dir(root / "candidates") @@ -230,7 +240,7 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) review_started = time.monotonic() - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) review_seconds = round(time.monotonic() - review_started, 3) return { "record": { @@ -262,7 +272,7 @@ def generate_candidate(index: int) -> dict[str, Any]: prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" write_text(prompt_path, prompt) review_started = time.monotonic() - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) review_seconds = round(time.monotonic() - review_started, 3) generation = {"mode": "existing_candidate", "source": str(repair_source_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} records.append({ @@ -304,7 +314,7 @@ def generate_candidate(index: int) -> dict[str, Any]: repaired_id = f"repair_{repair_index + 1:02d}" repaired_path = candidate_dir / f"{repaired_id}.png" base_prompt = Path(source["prompt_path"]).read_text(encoding="utf-8") - prompt = _repair_prompt(source["review"], base_prompt) + prompt = _repair_prompt(source["review"], base_prompt, plan) prompt_path = prompt_dir / f"{repaired_id}_prompt.txt" write_text(prompt_path, prompt) repair_started = time.monotonic() @@ -314,7 +324,7 @@ def generate_candidate(index: int) -> dict[str, Any]: generation_seconds = round(time.monotonic() - generation_started, 3) requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) review_started = time.monotonic() - review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) review_seconds = round(time.monotonic() - review_started, 3) repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"], "timings": {"generation_seconds": generation_seconds, "review_seconds": review_seconds, "total_seconds": round(time.monotonic() - repair_started, 3)}} repair_records.append(repaired) diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index a85a74a..812b562 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -507,13 +507,16 @@ def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: def collect_visible_labels(spec: dict, limit: int = 24) -> list[str]: labels: list[str] = [] + normalized_labels: set[str] = set() def add(value: object) -> None: if isinstance(value, dict): value = value.get("visible_label") or value.get("name") or value.get("text") or value.get("statement") label = str(value or "").strip() - if label and len(label) <= 64 and label not in labels: + normalized = re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) + if label and len(label) <= 64 and normalized and normalized not in normalized_labels: labels.append(label) + normalized_labels.add(normalized) for item in spec.get("required_labels", []) if isinstance(spec.get("required_labels"), list) else []: add(item) @@ -530,6 +533,48 @@ def add(value: object) -> None: return labels[: max(1, int(limit))] +def collect_visual_relations(spec: dict) -> list[dict[str, str]]: + entities = { + str(item.get("id")): str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + repeatable = {re.sub(r"[^\w\u4e00-\u9fff]+", "", str(value).casefold()) for value in spec.get("repeatable_labels", []) if str(value).strip()} if isinstance(spec.get("repeatable_labels"), list) else set() + shared_ids = { + item_id for item_id, label in entities.items() + if re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) in repeatable + } + grouped: dict[tuple[str, str], dict[str, object]] = {} + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if not source or not target or source == target or source in shared_ids or target in shared_ids: + continue + entry = grouped.setdefault((source, target), {"types": [], "labels": []}) + relation_type = str(relation.get("type") or "data_flow") + if relation_type not in entry["types"]: + entry["types"].append(relation_type) + label = str(relation.get("label") or "").strip() + if label and label not in entry["labels"]: + entry["labels"].append(label) + priority = ("feedback_loop", "evaluation", "revision_input", "feedback", "conditioning", "data_flow", "prediction", "generation_input") + result: list[dict[str, str]] = [] + for (source, target), entry in grouped.items(): + types = list(entry["types"]) + relation_type = next((value for value in priority if value in types), types[0] if types else "data_flow") + result.append({ + "source": source, + "source_label": entities.get(source, source), + "target": target, + "target_label": entities.get(target, target), + "type": relation_type, + "label": str(entry["labels"][0]) if entry["labels"] else "", + }) + return result + + def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = 1, selected_template: dict | None = None) -> str: spec = plan["figure_specification"] design = plan["design_plan"] @@ -538,7 +583,17 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = style = plan["style_plan"] template = selected_template or {} labels = collect_visible_labels(spec) - numbered_labels = "\n".join(f"{index}. {label}" for index, label in enumerate(labels, start=1)) + repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] + repeatable_normalized = {re.sub(r"[^\w\u4e00-\u9fff]+", "", value.casefold()) for value in repeatable_labels} + numbered_labels = "\n".join( + f"{index}. {label}" + (" (may repeat to show evidence-supported component reuse)" if re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) in repeatable_normalized else "") + for index, label in enumerate(labels, start=1) + ) + visual_relations = collect_visual_relations(spec) + connector_checklist = "\n".join( + f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + (f" label: {item['label']}" if item["label"] else "") + for index, item in enumerate(visual_relations, start=1) + ) return f""" # Summary @@ -573,7 +628,12 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Mandatory visible label checklist: {numbered_labels} -Every checklist item must appear once as clearly readable text in the completed figure. An icon, symbol, equation variable, or unlabeled output card does not satisfy a checklist item. Reserve enough width for long labels and place each output label beside or inside its output node. +Every checklist item must appear at least once as clearly readable text in the completed figure. Labels explicitly marked as repeatable may appear more than once only to show reuse of the same paper-supported component; every other label must appear exactly once. An icon, symbol, equation variable, or unlabeled output card does not satisfy a checklist item. Reserve enough width for long labels and place each output label beside or inside its output node. + +Mandatory directed connector checklist: +{connector_checklist} + +Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows. Forbidden copied reference terms: {json.dumps(template.get('forbidden_copy_terms', []), ensure_ascii=False)} @@ -589,8 +649,9 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = - Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. - Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. - Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. +- Template panels are macro spatial regions, not an exact limit on the number of scientific content nodes inside those regions. - Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. -- Render every mandatory visible label exactly once; never replace an output label with an icon-only node. +- Render every mandatory visible label at least once; repeat only labels explicitly marked as evidence-supported shared components, and never replace an output label with an icon-only node. - Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. - Include explicit input-boundary and output-boundary connectors. """.strip() diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 6f14d2b..e9741c9 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -667,13 +667,33 @@ def choose_input_target(label: str) -> dict[str, Any] | None: _repair_cross_script_terminology(spec, parsed) completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) labels = [] + normalized_labels: set[str] = set() + def add_required_label(value: object) -> None: + label = str(value or "").strip() + normalized = _normalized_label(label) + if label and normalized and normalized not in normalized_labels: + labels.append(label) + normalized_labels.add(normalized) terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} - labels.extend(str(value).strip() for value in terminology.values() if str(value).strip()) for field in ("inputs", "modules", "outputs"): for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: if isinstance(item, dict) and _item_label(item): - labels.append(_item_label(item)) - spec["required_labels"] = list(dict.fromkeys(labels)) + add_required_label(_item_label(item)) + for value in terminology.values(): + add_required_label(value) + spec["required_labels"] = labels + repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] + central_claim_text = str(spec.get("central_claim", {}).get("text") or "") if isinstance(spec.get("central_claim"), dict) else str(spec.get("central_claim") or "") + shared_component_signal = bool(re.search(r"\b(?:same|shared|reused?|single underlying)\b", central_claim_text, re.IGNORECASE)) + if shared_component_signal: + for item in spec.get("modules", []) if isinstance(spec.get("modules"), list) else []: + if not isinstance(item, dict): + continue + label = _item_label(item) + role = str(item.get("role") or "").casefold() + if label and ("model" in label.casefold() or "shared" in role or "reused" in role) and label not in repeatable_labels: + repeatable_labels.append(label) + spec["repeatable_labels"] = repeatable_labels raw_uncertainties = list(summary.get("unknowns", []) if isinstance(summary.get("unknowns"), list) else []) raw_uncertainties.extend( f"Dropped unresolved relation {value} because its endpoint was not declared." diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index e6704d9..dedb1c0 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -12,6 +12,25 @@ BUILTIN_TEMPLATES: dict[str, dict[str, Any]] = { + "feedback": { + "summary": "Compact two-row iterative generation, feedback, and refinement template.", + "template_id": "feedback", + "name": "Feedback refinement loop", + "aspect_ratio": 1.78, + "topology": ["feedback", "iteration", "two_row_loop", "shared_model"], + "ideal_module_range": [4, 9], + "visual_density": "high", + "panels": [ + {"id": "task_input", "role": "input", "bbox_percent": {"x": 0.02, "y": 0.28, "w": 0.13, "h": 0.43}}, + {"id": "generation", "role": "initial_generation", "bbox_percent": {"x": 0.20, "y": 0.10, "w": 0.20, "h": 0.28}}, + {"id": "initial_output", "role": "initial_output", "bbox_percent": {"x": 0.44, "y": 0.10, "w": 0.18, "h": 0.28}}, + {"id": "feedback", "role": "self_feedback", "bbox_percent": {"x": 0.72, "y": 0.29, "w": 0.20, "h": 0.30}}, + {"id": "refinement", "role": "refinement", "bbox_percent": {"x": 0.24, "y": 0.63, "w": 0.22, "h": 0.24}}, + {"id": "refined_output", "role": "refined_output", "bbox_percent": {"x": 0.54, "y": 0.63, "w": 0.20, "h": 0.24}}, + ], + "connectors": ["input_to_generation", "generation_to_initial_output", "initial_output_to_feedback", "initial_output_to_refinement", "feedback_to_refinement", "refinement_to_refined_output", "refined_output_to_feedback_loop"], + "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_blue", "accent": ["navy", "teal", "orange"], "corners": "rounded", "shadow": "soft_cards"}, + }, "arbor": { "summary": "Tree-centered iterative research template.", "template_id": "arbor", @@ -208,8 +227,10 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", module_fit = 1.0 if low <= features["module_count"] <= high else max(0.0, 1.0 - min(abs(features["module_count"] - low), abs(features["module_count"] - high)) / 10) topology = 0.0 reasons = [f"module_fit={module_fit:.2f}"] - if template_id == "arbor" and (features["has_loop"] or features["has_tree"]): - topology += 1.0; reasons.append("loop/tree topology") + if template_id == "feedback" and features["has_loop"] and not features["has_tree"]: + topology += 1.0; reasons.append("explicit feedback-loop topology") + if template_id == "arbor" and features["has_tree"]: + topology += 1.0; reasons.append("tree/branch topology") if template_id == "linear" and not features["has_loop"] and not features["has_tree"] and not features["has_retrieval"] and 2 <= features["module_count"] <= 8: topology += 0.9; reasons.append("sequential stage count") if template_id == "tripanel" and features["has_retrieval"]: @@ -283,6 +304,22 @@ def render_layout_blueprint(profile: dict, out_path: str | Path, target_ratio: s arrow = 12 points = [end, (end[0] - arrow * math.cos(angle - 0.45), end[1] - arrow * math.sin(angle - 0.45)), (end[0] - arrow * math.cos(angle + 0.45), end[1] - arrow * math.sin(angle + 0.45))] draw.polygon(points, fill=(80, 111, 134)) + if profile.get("template_id") == "feedback" and len(centers) >= 6: + extra_pairs = [(centers[2], centers[4]), (centers[5], centers[3])] + for pair_index, (start, end) in enumerate(extra_pairs): + if pair_index == 0: + mid_y = max(start[1], end[1]) + path = [start, (start[0], mid_y), end] + else: + loop_x = min(width - 18, max(start[0], end[0]) + round(width * 0.08)) + path = [start, (loop_x, start[1]), (loop_x, end[1]), end] + draw.line(path, fill=(31, 117, 126), width=4, joint="curve") + if len(path) >= 2: + tail, head = path[-2], path[-1] + angle = math.atan2(head[1] - tail[1], head[0] - tail[0]) + arrow = 12 + points = [head, (head[0] - arrow * math.cos(angle - 0.45), head[1] - arrow * math.sin(angle - 0.45)), (head[0] - arrow * math.cos(angle + 0.45), head[1] - arrow * math.sin(angle + 0.45))] + draw.polygon(points, fill=(31, 117, 126)) path = Path(out_path) path.parent.mkdir(parents=True, exist_ok=True) image.save(path) diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 846a369..d1f4ee2 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -37,6 +37,7 @@ def run_paper_to_image( ocr_lang: str = "en_ch", ocr_adapter=None, critic_adapter=None, + topology_adapter=None, ) -> dict: started = time.time() effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" @@ -155,6 +156,7 @@ def run_paper_to_image( ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, + topology_adapter=topology_adapter, ) elapsed = round(time.time() - started, 3) run_summary: dict[str, Any] = { @@ -203,6 +205,7 @@ def run_paper_to_image( "ocr_review.json", "template_alignment_report.json", "scientific_critic_report.json", + "topology_critic_report.json", "aesthetic_critic_report.json", "selected_image.png" if generation.get("selected_image") else "engineering_preview.png", ], diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 9a8181e..4eb77ac 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -15,7 +15,7 @@ from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import required_labels, review_candidate from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio -from rfs.paper_to_image.planner import compile_image_prompt +from rfs.paper_to_image.planner import collect_visual_relations, compile_image_prompt from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template @@ -148,6 +148,45 @@ def test_prompt_and_critic_share_complete_visible_label_contract(self): self.assertIn("Evidence Grounding", prompt) self.assertIn("never replace an output label with an icon-only node", prompt) + def test_visual_relation_checklist_deduplicates_edges_and_skips_shared_model_wiring(self): + spec = { + "inputs": [{"id": "input", "name": "Input"}], + "modules": [ + {"id": "model", "name": "Model M"}, + {"id": "generate", "name": "Generate"}, + {"id": "initial", "name": "Initial Output"}, + {"id": "feedback", "name": "Feedback"}, + {"id": "refine", "name": "Refine"}, + ], + "outputs": [{"id": "refined", "name": "Refined Output"}], + "innovations": [], + "repeatable_labels": ["Model M"], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow"}, + {"source": "model", "target": "initial", "type": "prediction"}, + {"source": "input", "target": "generate", "type": "generation_input"}, + {"source": "input", "target": "generate", "type": "data_flow"}, + {"source": "generate", "target": "initial", "type": "data_flow"}, + {"source": "initial", "target": "feedback", "type": "data_flow"}, + {"source": "initial", "target": "feedback", "type": "evaluation"}, + {"source": "initial", "target": "refine", "type": "revision_input"}, + {"source": "feedback", "target": "refine", "type": "feedback"}, + {"source": "refine", "target": "refined", "type": "prediction"}, + {"source": "refine", "target": "refined", "type": "data_flow"}, + {"source": "refined", "target": "feedback", "type": "feedback_loop"}, + ], + } + + relations = collect_visual_relations(spec) + pairs = {(item["source"], item["target"]): item["type"] for item in relations} + + self.assertNotIn(("input", "model"), pairs) + self.assertNotIn(("model", "initial"), pairs) + self.assertEqual(pairs[("input", "generate")], "data_flow") + self.assertEqual(pairs[("initial", "feedback")], "evaluation") + self.assertEqual(pairs[("refine", "refined")], "data_flow") + self.assertEqual(pairs[("refined", "feedback")], "feedback_loop") + def test_candidate_review_accepts_requested_ratio_when_image2_ignores_blueprint_ratio(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -170,6 +209,79 @@ def test_candidate_review_accepts_requested_ratio_when_image2_ignores_blueprint_ self.assertTrue(review["basic"]["passed"]) self.assertEqual(review["basic"]["matched_aspect_ratio"], "16:9") + def test_candidate_review_allows_evidence_supported_shared_label_reuse(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["repeatable_labels"] = ["Document Parser"] + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"].update({"duplicate_labels": ["Document Parser"], "passed": False}) + result["hard_errors"] = ["Duplicate label: Document Parser", "Template mismatch: too many content nodes"] + return result + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["allowed_duplicate_labels"], ["Document Parser"]) + self.assertEqual(review["ocr"]["duplicate_labels"], []) + + def test_complex_feedback_candidate_requires_focused_topology_pass(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "refine", "name": "Refine"}], + "outputs": [{"id": "output", "name": "Refined Output"}], + "relations": [{"source": "input", "target": "refine", "type": "revision_input"}, {"source": "refine", "target": "output", "type": "data_flow"}], + "terminology": {}, + }) + + topology = lambda _path, _prompt: {"summary": "Focused topology.", "missing_relations": ["Input -> Refine"], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": ["Connect Input to Refine"], "repair_regions": ["Refine input"], "score": 0.5, "passed": False} + review = review_candidate(candidate, blueprint, plan, {"template_id": "feedback", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=_passing_critic, topology_adapter=topology) + + self.assertFalse(review["production_pass"]) + self.assertFalse(review["topology"]["passed"]) + self.assertIn("Input -> Refine", review["hard_errors"]) + self.assertIn("Connect Input to Refine", review["repair"]) + + def test_issue_free_ten_point_scale_review_is_normalized_by_findings(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["topology"] = "feedback" + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + for section in ("ocr", "scientific", "template", "aesthetic"): + result[section]["score"] = 7.0 + result[section]["passed"] = False + return result + + topology = lambda _path, _prompt: {"summary": "Focused topology.", "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": [], "repair_regions": [], "score": 10.0, "passed": True} + review = review_candidate(candidate, blueprint, plan, {"template_id": "feedback", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic, topology_adapter=topology) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["score"], 1.0) + self.assertEqual(review["scientific"]["score"], 1.0) + self.assertEqual(review["template"]["score"], 0.7) + self.assertEqual(review["aesthetic"]["score"], 0.7) + def test_inspect_pdf_cli_defaults(self): parser = build_parser() args = parser.parse_args(["inspect-pdf", "--paper", "paper.pdf", "--out", "output/inspect"]) @@ -349,6 +461,20 @@ def test_three_stage_non_loop_workflow_selects_linear_template(self): self.assertEqual(selected["template_id"], "linear") + def test_feedback_workflow_selects_feedback_template_instead_of_arbor(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "modules": [{"statement": value} for value in ("Generate", "Initial Output", "Feedback", "Refine", "Refined Output")], + "inputs": [{"statement": "Input"}], + "relations": [{"relation_type": "feedback", "statement": "Refined Output returns to Feedback"}], + "workflows": {"feedback": [{"statement": "iterative self-feedback loop"}]}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "feedback") + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index c8ccf04..f6125ac 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -1,3 +1,4 @@ +import re import unittest from rfs.paper_to_image.planner import validate_plan_grounding @@ -592,6 +593,55 @@ def test_zero_shot_prediction_is_not_upgraded_to_classifier(self): self.assertNotIn("Zero-Shot Classifier", output_names) self.assertEqual(sum(item["name"] == "Zero-shot Prediction" for item in spec["outputs"]), 1) + def test_feedback_contract_merges_formula_and_display_name_duplicates(self): + caption = "Figure 1: Input x is used to Generate an Initial Output, obtain Self-Feedback, Refine it into a Refined Output, and repeat the feedback loop." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "inputs": [ + {"id": "input_x", "name": "input x", "evidence_ids": ["E0001"]}, + {"id": "input_input", "name": "Input", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "generate", "name": "Generate", "evidence_ids": ["E0001"]}, + {"id": "initial_output", "name": "Initial Output", "evidence_ids": ["E0001"]}, + {"id": "feedback", "name": "Self-Feedback", "evidence_ids": ["E0001"]}, + {"id": "refine", "name": "Refine", "evidence_ids": ["E0001"]}, + ], + "outputs": [ + {"id": "y_0", "name": "y 0", "evidence_ids": ["E0001"]}, + {"id": "y_t_plus_1", "name": "y t + 1", "evidence_ids": ["E0001"]}, + {"id": "refined_output", "name": "Refined Output", "evidence_ids": ["E0001"]}, + ], + "relations": [ + {"source": "input_x", "target": "generate", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "generate", "target": "y_0", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "y_0", "target": "feedback", "type": "feedback", "evidence_ids": ["E0001"]}, + {"source": "refine", "target": "y_t_plus_1", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + "innovations": [], + "must_show": [], + "terminology": {"y 0": "Initial Output", "y t + 1": "Refined Output"}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + all_entities = [item for field in ("inputs", "modules", "outputs") for item in spec[field]] + normalized_names = [re.sub(r"[^a-z0-9]+", "", item["name"].casefold()) for item in all_entities] + endpoint_ids = {item["id"] for item in all_entities} + + self.assertEqual(len(spec["inputs"]), 1) + self.assertTrue(normalized_names[0].startswith("input")) + self.assertEqual(sum(name == "initialoutput" for name in normalized_names), 1) + self.assertEqual(sum(name == "refinedoutput" for name in normalized_names), 1) + self.assertTrue(all(item["source"] in endpoint_ids and item["target"] in endpoint_ids for item in spec["relations"])) + self.assertEqual(len(spec["required_labels"]), len({_normalized.casefold() for _normalized in spec["required_labels"]})) + def test_overview_caption_and_stage_list_complete_missing_panels(self): caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." parsed = { From 745c0404bdbe3b7e01645f6a6a8080b56c2ebb83 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 04:53:06 +0800 Subject: [PATCH 55/74] Validate Image2 multi-head branch topology --- README.md | 12 +-- docs/paper-to-image.md | 13 ++-- rfs/cli.py | 4 +- rfs/paper_to_image/preparation.py | 97 ++++++++++++++++++++++++ rfs/paper_to_image/templates.py | 25 +++++- tests/integration/test_paper_to_image.py | 14 ++++ tests/unit/test_paper_contract.py | 53 +++++++++++++ 7 files changed, 203 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 9162e92..acc2254 100644 --- a/README.md +++ b/README.md @@ -201,11 +201,13 @@ a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, template contract, production gates, and failure behavior. -Automatic selection includes a dedicated `feedback` template for compact -generation → feedback → refinement loops. Tree/branch systems continue to use -`arbor`; simple sequential systems use `linear`. Feedback, branch, multimodal, -and dense candidates also run a focused connector judge that verifies visible -arrow endpoints and rejects shortcuts that bypass required modules. +Automatic selection includes dedicated `feedback` and `branch` templates. +Compact generation → feedback → refinement loops use `feedback`; shared-trunk +systems with parallel prediction heads use `branch`; true search-tree systems +use `arbor`; simple sequential systems use `linear`. Feedback, branch, +multimodal, and dense candidates also run a focused connector judge that +verifies visible arrow endpoints and rejects shortcuts that bypass required +modules. ## Offline Smoke Test diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 2b7be39..d8aa320 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -146,14 +146,15 @@ Available profiles: ## Template Contract -Built-in templates include `feedback`, `arbor`, `linear`, `tripanel`, and -`dense-multimodal`. Compact iterative generation/feedback/refinement systems use -`feedback`; tree or branching systems use `arbor`. Positive references are -classified into the closest ratio-based reference archetype and become +Built-in templates include `branch`, `feedback`, `arbor`, `linear`, `tripanel`, +and `dense-multimodal`. Compact iterative generation/feedback/refinement systems +use `feedback`; shared-trunk systems with parallel output heads use `branch`; +true search-tree systems use `arbor`. Positive references are classified into +the closest ratio-based reference archetype and become content-free profiles containing normalized panels, topology, connector rhythm, density, palette, and forbidden copied content. Automatic selection uses module -count, loops/tree structure, multimodality, retrieval structure, and requested -ratio. +count, loop/tree/parallel-branch structure, multimodality, retrieval structure, +and requested ratio. The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. diff --git a/rfs/cli.py b/rfs/cli.py index faa159b..e368cb2 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -206,7 +206,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + paper_image.add_argument("--template", choices=["auto", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") @@ -238,7 +238,7 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") paper_editable.add_argument("--review-model") paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") - paper_editable.add_argument("--template", choices=["auto", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + paper_editable.add_argument("--template", choices=["auto", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") paper_editable.add_argument("--repair-rounds", type=int, default=1) paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index e9741c9..21470fe 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -352,6 +352,100 @@ def _deduplicate_contract_entities(spec: dict[str, Any]) -> dict[str, list[str]] return {"removed": removed, "replacements": [f"{source}->{target}" for source, target in replacements.items() if source and target]} +def _simplify_branch_contract(spec: dict[str, Any]) -> dict[str, list[str]]: + if str(spec.get("topology") or "") != "branch": + return {"merged_entities": [], "removed_cross_branch_relations": []} + + def group_for(item: dict[str, Any]) -> str | None: + label = _normalized_label(_item_label(item)) + if label in {"rpn", "regionproposals", "regionproposalnetwork", "regionproposalnetworkrpn", "proposals"}: + return "proposals" + if any(value in label for value in ("classificationbranch", "classificationhead", "classprediction", "classpredictions")) or label == "classification": + return "classification" + if any(value in label for value in ("boundingboxregression", "boxregression", "boxbranch", "boundingboxoffset", "boundingboxpredictions")): + return "box" + if label in {"maskbranch", "binarymask", "validsegmentationmask", "maskprediction"}: + return "mask" + return None + + preference = { + "proposals": ("rpn", "regionproposals"), + "classification": ("classification", "classificationbranch", "classprediction"), + "box": ("boundingboxregression", "boxbranch", "boundingboxpredictions", "boundingboxoffset"), + "mask": ("maskbranch", "validsegmentationmask", "binarymask"), + } + records: list[tuple[str, dict[str, Any], str]] = [] + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if isinstance(item, dict) and (group := group_for(item)): + records.append((field, item, group)) + + replacements: dict[str, str] = {} + removed_ids: set[str] = set() + merged_entities: list[str] = [] + for group in ("proposals", "classification", "box", "mask"): + members = [(field, item) for field, item, item_group in records if item_group == group] + if len(members) <= 1: + continue + + def rank(record: tuple[str, dict[str, Any]]) -> tuple[int, int]: + field, item = record + label = _normalized_label(_item_label(item)) + preferred = preference[group] + label_rank = next((index for index, value in enumerate(preferred) if label == value), len(preferred)) + field_rank = 0 if (group == "classification" and field == "outputs") or (group != "classification" and field == "modules") else 1 + return label_rank, field_rank + + canonical_field, canonical = min(members, key=rank) + canonical_id = str(canonical.get("id") or "") + for field, item in members: + item_id = str(item.get("id") or "") + if item is canonical: + continue + canonical["evidence_ids"] = list(dict.fromkeys([*canonical.get("evidence_ids", []), *item.get("evidence_ids", [])])) + if item_id and canonical_id: + replacements[item_id] = canonical_id + removed_ids.add(item_id) + merged_entities.append(f"{item_id}->{canonical_id}") + + if removed_ids: + for field in ("inputs", "modules", "outputs"): + spec[field] = [item for item in spec.get(field, []) if not (isinstance(item, dict) and str(item.get("id") or "") in removed_ids)] + + removed_cross_branch_relations: list[str] = [] + deduplicated: list[dict[str, Any]] = [] + seen_relations: dict[tuple[str, str, str], dict[str, Any]] = {} + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source = replacements.get(str(relation.get("source") or ""), str(relation.get("source") or "")) + target = replacements.get(str(relation.get("target") or ""), str(relation.get("target") or "")) + if not source or not target or source == target or source not in endpoints or target not in endpoints: + continue + source_group = group_for(endpoints[source]) + target_group = group_for(endpoints[target]) + branch_groups = {"classification", "box", "mask"} + if source_group in branch_groups and target_group in branch_groups and source_group != target_group: + removed_cross_branch_relations.append(f"{source}->{target}") + continue + relation["source"], relation["target"] = source, target + key = (source, target, str(relation.get("type") or "data_flow")) + if key in seen_relations: + existing = seen_relations[key] + existing["evidence_ids"] = list(dict.fromkeys([*existing.get("evidence_ids", []), *relation.get("evidence_ids", [])])) + continue + seen_relations[key] = relation + deduplicated.append(relation) + spec["relations"] = deduplicated + return {"merged_entities": merged_entities, "removed_cross_branch_relations": removed_cross_branch_relations} + + def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: aliases: dict[str, set[str]] = {} endpoint_ids: set[str] = set() @@ -657,6 +751,9 @@ def choose_input_target(label: str) -> dict[str, Any] | None: existing_pairs.add(pair) incoming.add(target_id) spec["relations"] = relations + branch_report = _simplify_branch_contract(spec) + completion_report["branch_merged_entities"] = branch_report["merged_entities"] + completion_report["removed_cross_branch_relations"] = branch_report["removed_cross_branch_relations"] spec.setdefault("training_flow", summary.get("training_flow") if isinstance(summary.get("training_flow"), list) else []) spec.setdefault("inference_flow", summary.get("inference_flow") if isinstance(summary.get("inference_flow"), list) else []) spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index dedb1c0..999af8f 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -12,6 +12,24 @@ BUILTIN_TEMPLATES: dict[str, dict[str, Any]] = { + "branch": { + "summary": "Shared-trunk architecture with proposal convergence and parallel output heads.", + "template_id": "branch", + "name": "Shared trunk with parallel heads", + "aspect_ratio": 1.78, + "topology": ["left_to_right", "shared_backbone", "convergence", "three_way_branch"], + "ideal_module_range": [4, 10], + "visual_density": "high", + "panels": [ + {"id": "input", "role": "input", "bbox_percent": {"x": 0.02, "y": 0.28, "w": 0.13, "h": 0.44}}, + {"id": "backbone", "role": "shared_backbone", "bbox_percent": {"x": 0.19, "y": 0.22, "w": 0.18, "h": 0.56}}, + {"id": "proposals", "role": "proposal_path", "bbox_percent": {"x": 0.40, "y": 0.10, "w": 0.16, "h": 0.28}}, + {"id": "alignment", "role": "feature_proposal_convergence", "bbox_percent": {"x": 0.43, "y": 0.48, "w": 0.17, "h": 0.25}}, + {"id": "heads", "role": "parallel_output_heads", "bbox_percent": {"x": 0.68, "y": 0.14, "w": 0.28, "h": 0.72}}, + ], + "connectors": ["input_to_backbone", "backbone_to_proposals", "backbone_features_to_alignment", "proposals_to_alignment", "alignment_three_way_split"], + "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_blue", "accent": ["blue", "orange", "green"], "corners": "rounded", "shadow": "soft_cards"}, + }, "feedback": { "summary": "Compact two-row iterative generation, feedback, and refinement template.", "template_id": "feedback", @@ -195,7 +213,8 @@ def _paper_features(review: dict) -> dict: "module_count": len(review.get("modules", [])), "input_count": len(review.get("inputs", [])), "has_loop": bool(workflow.get("feedback")) or "feedback" in relation_types or any(token in statements for token in ["iterative", "loop", "tree search", "backpropagate"]), - "has_tree": any(token in statements for token in ["tree", "branch", "prune", "search"]), + "has_tree": any(token in statements for token in ["tree", "prune", "tree search", "search tree"]), + "has_branch": any(token in statements for token in ["branch", "parallel head", "parallel output", "multi-head", "multihead"]), "has_multimodal": any(token in statements for token in ["multimodal", "multi-modal", "image", "video", "audio", "document"]), "has_retrieval": any(token in statements for token in ["retrieval", "retrieve", "index", "rag", "knowledge graph"]), } @@ -229,9 +248,11 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", reasons = [f"module_fit={module_fit:.2f}"] if template_id == "feedback" and features["has_loop"] and not features["has_tree"]: topology += 1.0; reasons.append("explicit feedback-loop topology") + if template_id == "branch" and features["has_branch"] and not features["has_loop"] and not features["has_tree"] and not features["has_multimodal"] and not features["has_retrieval"]: + topology += 1.0; reasons.append("shared-trunk parallel-branch topology") if template_id == "arbor" and features["has_tree"]: topology += 1.0; reasons.append("tree/branch topology") - if template_id == "linear" and not features["has_loop"] and not features["has_tree"] and not features["has_retrieval"] and 2 <= features["module_count"] <= 8: + if template_id == "linear" and not features["has_loop"] and not features["has_tree"] and not features["has_branch"] and not features["has_retrieval"] and 2 <= features["module_count"] <= 8: topology += 0.9; reasons.append("sequential stage count") if template_id == "tripanel" and features["has_retrieval"]: topology += 0.9; reasons.append("index/retrieval topology") diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 4eb77ac..2282d81 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -475,6 +475,20 @@ def test_feedback_workflow_selects_feedback_template_instead_of_arbor(self): self.assertEqual(selected["template_id"], "feedback") + def test_parallel_head_workflow_selects_branch_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "modules": [{"statement": value} for value in ("Backbone", "RPN", "RoIAlign", "classification branch", "box branch", "mask branch")], + "inputs": [{"statement": "Input Image"}], + "relations": [{"relation_type": "branch", "statement": "RoIAlign feeds three parallel heads"}], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "branch") + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index f6125ac..ce60d6c 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -642,6 +642,59 @@ def test_feedback_contract_merges_formula_and_display_name_duplicates(self): self.assertTrue(all(item["source"] in endpoint_ids and item["target"] in endpoint_ids for item in spec["relations"])) self.assertEqual(len(spec["required_labels"]), len({_normalized.casefold() for _normalized in spec["required_labels"]})) + def test_branch_contract_merges_head_output_aliases_and_removes_cross_branch_edges(self): + caption = "Figure 1: An Input Image passes through a Backbone and RPN to RoIAlign, followed by parallel classification, bounding-box regression, and mask branches." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [{"id": "image", "name": "Input Image", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "backbone", "name": "Backbone", "evidence_ids": ["E0001"]}, + {"id": "rpn", "name": "RPN", "evidence_ids": ["E0001"]}, + {"id": "proposals", "name": "Region Proposals", "evidence_ids": ["E0001"]}, + {"id": "roi", "name": "RoIAlign", "evidence_ids": ["E0001"]}, + {"id": "class_branch", "name": "classification branch", "evidence_ids": ["E0001"]}, + {"id": "box_branch", "name": "bounding-box regression", "evidence_ids": ["E0001"]}, + {"id": "mask_branch", "name": "mask branch", "evidence_ids": ["E0001"]}, + ], + "outputs": [ + {"id": "class_output", "name": "Classification", "evidence_ids": ["E0001"]}, + {"id": "box_output", "name": "bounding-box offset", "evidence_ids": ["E0001"]}, + {"id": "mask_output", "name": "binary mask", "evidence_ids": ["E0001"]}, + ], + "relations": [ + {"source": "image", "target": "backbone", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "backbone", "target": "rpn", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "proposals", "target": "roi", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "roi", "target": "class_branch", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roi", "target": "box_branch", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roi", "target": "mask_branch", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "mask_branch", "target": "box_output", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "mask_branch", "target": "mask_output", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + "innovations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + names = [re.sub(r"[^a-z0-9]+", "", item["name"].casefold()) for field in ("inputs", "modules", "outputs") for item in spec[field]] + branch_names = [name for name in names if name in {"classification", "classificationbranch", "boundingboxregression", "boxbranch", "maskbranch"}] + endpoint_by_name = {re.sub(r"[^a-z0-9]+", "", item["name"].casefold()): item["id"] for field in ("inputs", "modules", "outputs") for item in spec[field]} + branch_ids = {endpoint_by_name[name] for name in branch_names} + + self.assertEqual(len(spec["inputs"]), 1) + self.assertEqual(sum(name in {"rpn", "regionproposals"} for name in names), 1) + self.assertEqual(len(branch_names), 3) + self.assertFalse(any(item["source"] in branch_ids and item["target"] in branch_ids and item["source"] != item["target"] for item in spec["relations"])) + def test_overview_caption_and_stage_list_complete_missing_panels(self): caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." parsed = { From 0d7f6289f599c64faa6148a2a3d4f7debcb5fa4d Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 05:10:28 +0800 Subject: [PATCH 56/74] Validate Image2 multimodal convergence topology --- README.md | 10 +- docs/paper-to-image.md | 18 +++- rfs/cli.py | 4 +- rfs/paper_to_image/critics.py | 53 +++++++--- rfs/paper_to_image/preparation.py | 120 ++++++++++++++++++++++- rfs/paper_to_image/templates.py | 26 ++++- tests/integration/test_paper_to_image.py | 15 +++ tests/unit/test_paper_contract.py | 44 +++++++++ 8 files changed, 263 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index acc2254..83cef7d 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,9 @@ a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, template contract, production gates, and failure behavior. -Automatic selection includes dedicated `feedback` and `branch` templates. +Automatic selection includes dedicated `multimodal`, `feedback`, and `branch` +templates. Multiple modality inputs converging into one shared representation +use `multimodal`; Compact generation → feedback → refinement loops use `feedback`; shared-trunk systems with parallel prediction heads use `branch`; true search-tree systems use `arbor`; simple sequential systems use `linear`. Feedback, branch, @@ -209,6 +211,12 @@ multimodal, and dense candidates also run a focused connector judge that verifies visible arrow endpoints and rejects shortcuts that bypass required modules. +The general production critic and focused connector critic run concurrently. +Each uses a 90-second default timeout with no automatic full-call retry; override +them with `RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT`. Use `--repair-source` to re-review or +locally repair an existing candidate without regenerating correct regions. + ## Offline Smoke Test Use placeholder assets to validate the local pipeline without calling any API: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index d8aa320..457f9fe 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -146,10 +146,12 @@ Available profiles: ## Template Contract -Built-in templates include `branch`, `feedback`, `arbor`, `linear`, `tripanel`, -and `dense-multimodal`. Compact iterative generation/feedback/refinement systems -use `feedback`; shared-trunk systems with parallel output heads use `branch`; -true search-tree systems use `arbor`. Positive references are classified into +Built-in templates include `multimodal`, `branch`, `feedback`, `arbor`, `linear`, +`tripanel`, and `dense-multimodal`. Multi-input systems that converge through a +modality encoder bank into one joint space use `multimodal`; compact iterative +generation/feedback/refinement systems use `feedback`; shared-trunk systems with +parallel output heads use `branch`; true search-tree systems use `arbor`. +Positive references are classified into the closest ratio-based reference archetype and become content-free profiles containing normalized panels, topology, connector rhythm, density, palette, and forbidden copied content. Automatic selection uses module @@ -158,6 +160,14 @@ and requested ratio. The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. +The general production review and focused topology review execute concurrently. +Their default request timeout is 90 seconds with no automatic retry, preventing +two independent judges from creating a multi-minute sequential stall. Configure +`RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT` when a slower provider requires a larger +budget. Existing candidates can be passed through `--repair-source` so review or +localized repair does not regenerate the correct parts of the image. + When `--aspect-ratio auto` is used, the template keeps its internal normalized geometry while the generation canvas uses the nearest native Image2 ratio (`3:2`, `2:3`, or `1:1`). This avoids semantic cropping after generation. diff --git a/rfs/cli.py b/rfs/cli.py index e368cb2..1cdd78f 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -206,7 +206,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + paper_image.add_argument("--template", choices=["auto", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") @@ -238,7 +238,7 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") paper_editable.add_argument("--review-model") paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") - paper_editable.add_argument("--template", choices=["auto", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + paper_editable.add_argument("--template", choices=["auto", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") paper_editable.add_argument("--repair-rounds", type=int, default=1) paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 9cdcf47..a2fe84e 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -1,7 +1,9 @@ from __future__ import annotations +import os import json import re +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Callable @@ -97,7 +99,8 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: if adapter: return adapter(path, blueprint, prompt) resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_REVIEW_MODEL", "RFS_CRITIC_MODEL", explicit_model=model) - raw = call_vlm_json(prompt, [path, blueprint], model=resolved, timeout=240, retries=1) + timeout = max(15, int(os.getenv("RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT", "90"))) + raw = call_vlm_json(prompt, [path, blueprint], model=resolved, timeout=timeout, retries=0) raw.setdefault("summary", "Production candidate review.") raw["model"] = resolved return raw @@ -154,7 +157,8 @@ def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Cal if adapter: return adapter(path, prompt) resolved = resolve_vlm_model("RFS_PAPER_TO_IMAGE_TOPOLOGY_MODEL", "RFS_FROZEN_JUDGE_MODEL", explicit_model=model) - result = call_vlm_json(prompt, [path], model=resolved, timeout=180, retries=1) + timeout = max(15, int(os.getenv("RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT", "90"))) + result = call_vlm_json(prompt, [path], model=resolved, timeout=timeout, retries=0) result.setdefault("summary", "Focused visible-connector verification.") result["model"] = resolved return result @@ -223,15 +227,41 @@ def review_candidate( if requested_engine in {"paddle", "easyocr"}: local_records, local_engine, local_warning = _local_ocr(candidate, requested_engine, ocr_lang, adapter=ocr_adapter) + topology_name = str(plan.get("figure_specification", {}).get("topology") or "unknown") + topology_required = topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} vlm_raw: dict = {} + topology_raw: dict = {} vlm_warning = None - if mode == "vlm" and (vlm_credentials_available() or critic_adapter): - try: - vlm_raw = _vlm_critic(candidate, blueprint_path, plan, template, labels, repeatable_labels, forbidden, model, adapter=critic_adapter) - except Exception as exc: - vlm_warning = str(exc) + topology_warning = None + review_available = vlm_credentials_available() or critic_adapter or topology_adapter + if mode == "vlm" and review_available: + with ThreadPoolExecutor(max_workers=2 if topology_required else 1) as executor: + critic_future = executor.submit( + _vlm_critic, + candidate, + blueprint_path, + plan, + template, + labels, + repeatable_labels, + forbidden, + model, + critic_adapter, + ) + topology_future = executor.submit(_vlm_topology_critic, candidate, plan, model, topology_adapter) if topology_required else None + try: + vlm_raw = critic_future.result() + except Exception as exc: + vlm_warning = str(exc) + if topology_future is not None: + try: + topology_raw = topology_future.result() + except Exception as exc: + topology_warning = str(exc) elif mode == "vlm": vlm_warning = "VLM review credentials unavailable" + if topology_required: + topology_warning = vlm_warning detected_local = [str(item.get("text") or "").strip() for item in local_records if str(item.get("text") or "").strip()] ocr = _normalize_section(vlm_raw.get("ocr"), "OCR") @@ -258,15 +288,6 @@ def review_candidate( scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") - topology_name = str(plan.get("figure_specification", {}).get("topology") or "unknown") - topology_required = topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} - topology_warning = None - topology_raw: dict = {} - if topology_required and mode == "vlm" and (vlm_credentials_available() or topology_adapter): - try: - topology_raw = _vlm_topology_critic(candidate, plan, model, adapter=topology_adapter) - except Exception as exc: - topology_warning = str(exc) topology_review = _normalize_section(topology_raw, "Topology") if topology_required else {"summary": "Focused topology review not required for this topology.", "score": 1.0, "passed": True, "skipped": True} topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) if topology_required: diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 21470fe..9435f88 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -446,6 +446,117 @@ def rank(record: tuple[str, dict[str, Any]]) -> tuple[int, int]: return {"merged_entities": merged_entities, "removed_cross_branch_relations": removed_cross_branch_relations} +def _simplify_multimodal_contract(spec: dict[str, Any]) -> dict[str, Any]: + modality_names = {"image", "images", "text", "audio", "video", "depth", "thermal", "imu"} + modality_inputs = [] + modality_by_name: dict[str, dict[str, Any]] = {} + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + if not isinstance(item, dict): + continue + normalized = _normalized_label(_item_label(item)) + if normalized not in modality_names: + continue + key = "image" if normalized == "images" else normalized + if key in modality_by_name: + canonical = modality_by_name[key] + canonical["evidence_ids"] = list(dict.fromkeys([*canonical.get("evidence_ids", []), *item.get("evidence_ids", [])])) + continue + modality_by_name[key] = item + modality_inputs.append(item) + all_entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + encoder_candidates = [ + (field, item) for field, item in all_entities + if "modalityencoder" in _normalized_label(_item_label(item)) or "encodersforeachmodality" in _normalized_label(_item_label(item)) + ] + joint_candidates = [ + (field, item) for field, item in all_entities + if "embedding" in _normalized_label(_item_label(item)) + and any(term in _normalized_label(_item_label(item)) for term in ("joint", "common", "shared")) + ] + emergent_candidates = [ + (field, item) for field, item in all_entities + if ("emergent" in _normalized_label(_item_label(item)) and "align" in _normalized_label(_item_label(item))) + or "bindingproperty" in _normalized_label(_item_label(item)) + or "bindingagent" in _normalized_label(_item_label(item)) + ] + if len(modality_inputs) < 3 or not encoder_candidates or not joint_candidates or not emergent_candidates: + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + + def choose(candidates: list[tuple[str, dict[str, Any]]], preferred: tuple[str, ...]) -> tuple[str, dict[str, Any]]: + return min( + candidates, + key=lambda record: next( + (index for index, value in enumerate(preferred) if _normalized_label(_item_label(record[1])) == value), + len(preferred), + ), + ) + + _, encoder = choose(encoder_candidates, ("modalityencoders", "encodersforeachmodality")) + _, joint = choose(joint_candidates, ("jointembeddingspace", "singlejointembeddingspace", "commonembeddingspace", "sharedrepresentationspace")) + _, emergent = choose(emergent_candidates, ("emergentcrossmodalalignment", "emergentalignment", "bindingproperty", "bindingagent")) + encoder["name"] = "Modality Encoders" + encoder["role"] = "module group" + joint["name"] = "Joint Embedding Space" + joint["role"] = "shared representation" + emergent["name"] = "Emergent Cross-modal Alignment" + emergent["role"] = "output" + + for canonical, candidates in ((encoder, encoder_candidates), (joint, joint_candidates), (emergent, emergent_candidates)): + canonical["evidence_ids"] = list(dict.fromkeys( + value for _, item in candidates for value in item.get("evidence_ids", []) + )) + + keep_ids = {str(item.get("id")) for item in modality_inputs + [encoder, joint, emergent] if item.get("id")} + removed_entities = [ + str(item.get("id")) for _, item in all_entities if item.get("id") and str(item.get("id")) not in keep_ids + ] + spec["inputs"] = modality_inputs + spec["modules"] = [encoder, joint] + spec["outputs"] = [emergent] + spec["innovations"] = [] + + encoder_id = str(encoder.get("id")) + joint_id = str(joint.get("id")) + emergent_id = str(emergent.get("id")) + relations = [] + for item in modality_inputs: + relations.append({ + "source": str(item.get("id")), + "target": encoder_id, + "type": "encoding", + "label": "", + "evidence_ids": list(dict.fromkeys([*item.get("evidence_ids", []), *encoder.get("evidence_ids", [])])), + }) + relations.extend([ + { + "source": encoder_id, + "target": joint_id, + "type": "alignment", + "label": "", + "evidence_ids": list(dict.fromkeys([*encoder.get("evidence_ids", []), *joint.get("evidence_ids", [])])), + }, + { + "source": joint_id, + "target": emergent_id, + "type": "enables", + "label": "", + "evidence_ids": list(dict.fromkeys([*joint.get("evidence_ids", []), *emergent.get("evidence_ids", [])])), + }, + ]) + spec["relations"] = relations + spec["topology"] = "multimodal" + return { + "applied": True, + "removed_entities": removed_entities, + "rebuilt_relations": [f"{item['source']}->{item['target']}" for item in relations], + } + + def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: aliases: dict[str, set[str]] = {} endpoint_ids: set[str] = set() @@ -751,6 +862,10 @@ def choose_input_target(label: str) -> dict[str, Any] | None: existing_pairs.add(pair) incoming.add(target_id) spec["relations"] = relations + multimodal_report = _simplify_multimodal_contract(spec) + completion_report["multimodal_simplification_applied"] = multimodal_report["applied"] + completion_report["multimodal_removed_entities"] = multimodal_report["removed_entities"] + completion_report["multimodal_rebuilt_relations"] = multimodal_report["rebuilt_relations"] branch_report = _simplify_branch_contract(spec) completion_report["branch_merged_entities"] = branch_report["merged_entities"] completion_report["removed_cross_branch_relations"] = branch_report["removed_cross_branch_relations"] @@ -758,8 +873,9 @@ def choose_input_target(label: str) -> dict[str, Any] | None: spec.setdefault("inference_flow", summary.get("inference_flow") if isinstance(summary.get("inference_flow"), list) else []) spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) allowed_topologies = {"linear", "branch", "feedback", "multimodal", "dense_multiframe", "unknown"} - if str(spec.get("topology") or "") not in allowed_topologies: - spec["topology"] = _infer_topology(spec) + inferred_topology = _infer_topology(spec) + if str(spec.get("topology") or "") not in allowed_topologies or inferred_topology in {"feedback", "multimodal"}: + spec["topology"] = inferred_topology completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) _repair_cross_script_terminology(spec, parsed) completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 999af8f..6dfc3b0 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -12,6 +12,23 @@ BUILTIN_TEMPLATES: dict[str, dict[str, Any]] = { + "multimodal": { + "summary": "Multiple modality inputs converge through modality encoders into one joint embedding space and an emergent alignment output.", + "template_id": "multimodal", + "name": "Multimodal convergence into a shared space", + "aspect_ratio": 1.78, + "topology": ["many_inputs", "encoder_bank", "many_to_one", "joint_embedding", "emergent_alignment"], + "ideal_module_range": [2, 5], + "visual_density": "high", + "panels": [ + {"id": "modalities", "role": "stacked_modality_inputs", "bbox_percent": {"x": 0.02, "y": 0.06, "w": 0.25, "h": 0.88}}, + {"id": "encoders", "role": "modality_encoder_bank", "bbox_percent": {"x": 0.34, "y": 0.18, "w": 0.20, "h": 0.64}}, + {"id": "embedding", "role": "joint_embedding_space", "bbox_percent": {"x": 0.61, "y": 0.23, "w": 0.18, "h": 0.54}}, + {"id": "alignment", "role": "emergent_alignment_output", "bbox_percent": {"x": 0.84, "y": 0.28, "w": 0.14, "h": 0.44}}, + ], + "connectors": ["each_modality_to_encoder_bank", "encoder_bank_to_joint_embedding", "joint_embedding_to_emergent_alignment"], + "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_indigo", "accent": ["blue", "violet", "orange", "teal"], "corners": "rounded", "shadow": "soft_cards"}, + }, "branch": { "summary": "Shared-trunk architecture with proposal convergence and parallel output heads.", "template_id": "branch", @@ -206,16 +223,19 @@ def build_template_profiles(reference_paths: list[str], out_dir: str | Path, mod def _paper_features(review: dict) -> dict: - statements = " ".join(str(item.get("statement", "")) for field in ["research_questions", "central_claims", "modules", "relations", "innovations"] for item in review.get(field, [])).lower() + statements = " ".join(str(item.get("statement", "")) for field in ["research_questions", "central_claims", "inputs", "modules", "relations", "innovations"] for item in review.get(field, [])).lower() relation_types = {str(item.get("relation_type") or item.get("type") or "").lower() for item in review.get("relations", [])} workflow = review.get("workflows", {}) + input_labels = " ".join(str(item.get("visible_label") or item.get("statement") or "").lower() for item in review.get("inputs", [])) + modality_count = sum(term in input_labels for term in ("image", "text", "audio", "video", "depth", "thermal", "imu")) return { "module_count": len(review.get("modules", [])), "input_count": len(review.get("inputs", [])), "has_loop": bool(workflow.get("feedback")) or "feedback" in relation_types or any(token in statements for token in ["iterative", "loop", "tree search", "backpropagate"]), "has_tree": any(token in statements for token in ["tree", "prune", "tree search", "search tree"]), "has_branch": any(token in statements for token in ["branch", "parallel head", "parallel output", "multi-head", "multihead"]), - "has_multimodal": any(token in statements for token in ["multimodal", "multi-modal", "image", "video", "audio", "document"]), + "modality_count": modality_count, + "has_multimodal": modality_count >= 3 or any(token in statements for token in ["multimodal", "multi-modal", "cross-modal", "cross modal"]), "has_retrieval": any(token in statements for token in ["retrieval", "retrieve", "index", "rag", "knowledge graph"]), } @@ -250,6 +270,8 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", topology += 1.0; reasons.append("explicit feedback-loop topology") if template_id == "branch" and features["has_branch"] and not features["has_loop"] and not features["has_tree"] and not features["has_multimodal"] and not features["has_retrieval"]: topology += 1.0; reasons.append("shared-trunk parallel-branch topology") + if template_id == "multimodal" and features["has_multimodal"] and features["modality_count"] >= 3 and features["module_count"] < 8: + topology += 1.1; reasons.append("multiple modalities converge into a shared representation") if template_id == "arbor" and features["has_tree"]: topology += 1.0; reasons.append("tree/branch topology") if template_id == "linear" and not features["has_loop"] and not features["has_tree"] and not features["has_branch"] and not features["has_retrieval"] and 2 <= features["module_count"] <= 8: diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 2282d81..fb7076b 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -489,6 +489,21 @@ def test_parallel_head_workflow_selects_branch_template(self): self.assertEqual(selected["template_id"], "branch") + def test_many_modality_workflow_selects_multimodal_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "inputs": [{"statement": value, "visible_label": value} for value in ("Image", "Text", "Audio", "Depth", "Thermal", "IMU")], + "modules": [{"statement": "Modality Encoders"}, {"statement": "Joint Embedding Space"}], + "central_claims": [{"statement": "Cross-modal inputs share one embedding space"}], + "relations": [], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9") + + self.assertEqual(selected["template_id"], "multimodal") + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index ce60d6c..33b5994 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -695,6 +695,50 @@ def test_branch_contract_merges_head_output_aliases_and_removes_cross_branch_edg self.assertEqual(len(branch_names), 3) self.assertFalse(any(item["source"] in branch_ids and item["target"] in branch_ids and item["source"] != item["target"] for item in spec["relations"])) + def test_multimodal_contract_keeps_shared_embedding_story_and_removes_training_detail(self): + caption = "Figure 1: ImageBind uses images to bind text, audio, depth, thermal, and IMU in one joint embedding space with emergent cross-modal alignment." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [ + *[{"id": name.lower(), "name": name, "evidence_ids": ["E0001"]} for name in ("Image", "Text", "Audio", "Depth", "Thermal", "IMU")], + {"id": "images_alias", "name": "Images", "evidence_ids": ["E0001"]}, + {"id": "class_descriptions", "name": "Class Descriptions", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "modality_encoders", "name": "Modality encoders", "evidence_ids": ["E0001"]}, + {"id": "common_space", "name": "Common embedding space", "evidence_ids": ["E0001"]}, + {"id": "text_encoder", "name": "Text Encoder", "evidence_ids": ["E0001"]}, + {"id": "contrastive", "name": "Contrastive Learning", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "joint_space", "name": "Joint Embedding Space", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "emergent", "name": "Emergent Cross-modal Alignment", "evidence_ids": ["E0001"]}], + "relations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = [_normalized for field in ("inputs", "modules", "outputs") for item in spec[field] if (_normalized := re.sub(r"[^a-z0-9]+", "", item["name"].casefold()))] + + self.assertEqual(spec["topology"], "multimodal") + self.assertEqual(len(spec["inputs"]), 6) + self.assertEqual(labels.count("modalityencoders"), 1) + self.assertEqual(labels.count("jointembeddingspace"), 1) + self.assertEqual(labels.count("emergentcrossmodalalignment"), 1) + self.assertNotIn("classdescriptions", labels) + self.assertNotIn("textencoder", labels) + self.assertNotIn("contrastivelearning", labels) + self.assertEqual(len(spec["relations"]), 8) + self.assertIn("Emergent Cross-modal Alignment", spec["required_labels"]) + def test_overview_caption_and_stage_list_complete_missing_panels(self): caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." parsed = { From e501c1c8b81e1a64bbc11994e4895d983ded9a8c Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 05:29:20 +0800 Subject: [PATCH 57/74] Validate Image2 dense multiframe topology --- README.md | 11 ++- docs/paper-to-image.md | 13 +++- rfs/cli.py | 6 +- rfs/paper_to_image/generator.py | 6 +- rfs/paper_to_image/preparation.py | 98 ++++++++++++++++++++++++ rfs/paper_to_image/templates.py | 21 ++++- rfs/paper_to_image/workflow.py | 10 ++- tests/integration/test_paper_to_image.py | 40 ++++++++++ tests/unit/test_paper_contract.py | 53 +++++++++++++ 9 files changed, 245 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 83cef7d..56066ce 100644 --- a/README.md +++ b/README.md @@ -201,9 +201,10 @@ a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, template contract, production gates, and failure behavior. -Automatic selection includes dedicated `multimodal`, `feedback`, and `branch` -templates. Multiple modality inputs converging into one shared representation -use `multimodal`; +Automatic selection includes dedicated `dense-multiframe`, `multimodal`, +`feedback`, and `branch` templates. Dense task/model/data-engine overviews use +`dense-multiframe`; multiple modality inputs converging into one shared +representation use `multimodal`; Compact generation → feedback → refinement loops use `feedback`; shared-trunk systems with parallel prediction heads use `branch`; true search-tree systems use `arbor`; simple sequential systems use `linear`. Feedback, branch, @@ -216,6 +217,10 @@ Each uses a 90-second default timeout with no automatic full-call retry; overrid them with `RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and `RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT`. Use `--repair-source` to re-review or locally repair an existing candidate without regenerating correct regions. +Image-2 generation uses a 120-second default request timeout configured by +`RFS_IMAGE2_TIMEOUT`. A network timeout is not blindly retried; retryable HTTP +or provider failures receive at most the configured `--image-retries` attempts, +whose default is one. ## Offline Smoke Test diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 457f9fe..6215c73 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -146,9 +146,11 @@ Available profiles: ## Template Contract -Built-in templates include `multimodal`, `branch`, `feedback`, `arbor`, `linear`, -`tripanel`, and `dense-multimodal`. Multi-input systems that converge through a -modality encoder bank into one joint space use `multimodal`; compact iterative +Built-in templates include `dense-multiframe`, `multimodal`, `branch`, +`feedback`, `arbor`, `linear`, `tripanel`, and `dense-multimodal`. Dense +task/model/data-engine overviews with nested local flows use `dense-multiframe`; +multi-input systems that converge through a modality encoder bank into one joint +space use `multimodal`; compact iterative generation/feedback/refinement systems use `feedback`; shared-trunk systems with parallel output heads use `branch`; true search-tree systems use `arbor`. Positive references are classified into @@ -168,6 +170,11 @@ two independent judges from creating a multi-minute sequential stall. Configure budget. Existing candidates can be passed through `--repair-source` so review or localized repair does not regenerate the correct parts of the image. +Image-2 edit requests default to a 120-second timeout controlled by +`RFS_IMAGE2_TIMEOUT`. Network timeouts stop the candidate immediately instead of +blindly regenerating the entire image. Retryable HTTP or provider failures may +retry up to `--image-retries`; the default is one. + When `--aspect-ratio auto` is used, the template keeps its internal normalized geometry while the generation canvas uses the nearest native Image2 ratio (`3:2`, `2:3`, or `1:1`). This avoids semantic cropping after generation. diff --git a/rfs/cli.py b/rfs/cli.py index 1cdd78f..fae90f7 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -206,13 +206,13 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + paper_image.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") paper_image.add_argument("--language", default="English", help="Visible label language. Default: English.") paper_image.add_argument("--image-model", help="Optional image model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") - paper_image.add_argument("--image-retries", type=int, default=2, help="Retries per image candidate, clamped to 0-5. Default: 2.") + paper_image.add_argument("--image-retries", type=int, default=1, help="Retries per image candidate for retryable HTTP/provider errors, clamped to 0-5. Network timeouts are not blindly retried. Default: 1.") paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") @@ -238,7 +238,7 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") paper_editable.add_argument("--review-model") paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") - paper_editable.add_argument("--template", choices=["auto", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + paper_editable.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") paper_editable.add_argument("--repair-rounds", type=int, default=1) paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index ba875fe..67ec719 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -113,6 +113,7 @@ def _call_image2_edit(source_path: Path, prompt: str, output_path: Path, aspect_ raise RuntimeError("Reference-conditioned Image2 requires API_KEY/GEMINI_API_KEY") endpoint = _image2_edit_url() resolved_model = _image2_model_name(model) + request_timeout = max(30, int(os.getenv("RFS_IMAGE2_TIMEOUT", "120"))) last_error: Exception | None = None for attempt in range(max(0, min(5, int(retries))) + 1): try: @@ -122,13 +123,16 @@ def _call_image2_edit(source_path: Path, prompt: str, output_path: Path, aspect_ headers={"Authorization": f"Bearer {api_key}"}, data={"model": resolved_model, "prompt": prompt, "n": "1", "size": _image_size(aspect_ratio)}, files={"image": (source_path.name, handle, "image/png")}, - timeout=360, + timeout=request_timeout, ) if response.status_code == 429 or response.status_code >= 500: raise RuntimeError(f"Image2 edit returned HTTP {response.status_code}") response.raise_for_status() _write_openai_image(response.json(), output_path) return {"mode": "edit", "model": resolved_model, "endpoint": _safe_endpoint(endpoint), "source": str(source_path), "api_key_present": True} + except requests.Timeout as exc: + last_error = exc + break except Exception as exc: last_error = exc if attempt >= max(0, min(5, int(retries))): diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 9435f88..206277a 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -557,6 +557,100 @@ def choose(candidates: list[tuple[str, dict[str, Any]]], preferred: tuple[str, . } +def _simplify_dense_multiframe_contract(spec: dict[str, Any]) -> dict[str, Any]: + all_entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + + def candidates(predicate: Callable[[str, dict[str, Any]], bool]) -> list[tuple[str, dict[str, Any]]]: + return [(field, item) for field, item in all_entities if predicate(_normalized_label(_item_label(item)), item)] + + promptable = candidates(lambda label, item: "segmentationtask" in label and "prompt" in label) + sam_model = candidates(lambda label, item: "segmentanythingmodel" in label or "segmentationmodelsam" in label or label == "sam") + data_engine = candidates(lambda label, item: label == "dataengine") + image = candidates(lambda label, item: label in {"image", "images", "inputimage"} and item in spec.get("inputs", [])) + prompt = candidates(lambda label, item: label in {"prompt", "inputprompts", "segmentationprompt", "sparseprompts", "denseprompts"} and item in spec.get("inputs", [])) + image_encoder = candidates(lambda label, item: label == "imageencoder") + prompt_encoder = candidates(lambda label, item: label == "promptencoder") + mask_decoder = candidates(lambda label, item: label == "maskdecoder") + valid_mask = candidates(lambda label, item: label in {"validsegmentationmask", "validmask", "validmasks"}) + assisted = candidates(lambda label, item: label == "assistedmanual") + semi = candidates(lambda label, item: label == "semiautomatic") + fully = candidates(lambda label, item: label == "fullyautomatic") + sa1b = candidates(lambda label, item: label == "sa1b") + groups = (promptable, sam_model, data_engine, image, prompt, image_encoder, prompt_encoder, mask_decoder, valid_mask, assisted, semi, fully, sa1b) + if any(not group for group in groups): + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + + def merge(group: list[tuple[str, dict[str, Any]]], name: str, role: str) -> dict[str, Any]: + canonical = group[0][1] + canonical["name"] = name + canonical["role"] = role + canonical["evidence_ids"] = list(dict.fromkeys(value for _, item in group for value in item.get("evidence_ids", []))) + return canonical + + promptable_item = merge(promptable, "Promptable Segmentation Task", "overview_panel") + sam_item = merge(sam_model, "Segment Anything Model", "overview_panel") + data_engine_item = merge(data_engine, "Data Engine", "overview_panel") + image_item = merge(image, "Image", "input") + prompt_item = merge(prompt, "Prompt", "input") + image_encoder_item = merge(image_encoder, "Image Encoder", "module") + prompt_encoder_item = merge(prompt_encoder, "Prompt Encoder", "module") + mask_decoder_item = merge(mask_decoder, "Mask Decoder", "module") + valid_mask_item = merge(valid_mask, "Valid Segmentation Mask", "output") + assisted_item = merge(assisted, "Assisted-manual", "data_engine_stage") + semi_item = merge(semi, "Semi-automatic", "data_engine_stage") + fully_item = merge(fully, "Fully Automatic", "data_engine_stage") + sa1b_item = merge(sa1b, "SA-1B", "dataset") + + kept = [ + promptable_item, sam_item, data_engine_item, image_item, prompt_item, + image_encoder_item, prompt_encoder_item, mask_decoder_item, valid_mask_item, + assisted_item, semi_item, fully_item, sa1b_item, + ] + keep_ids = {str(item.get("id")) for item in kept if item.get("id")} + removed_entities = [str(item.get("id")) for _, item in all_entities if item.get("id") and str(item.get("id")) not in keep_ids] + spec["inputs"] = [image_item, prompt_item] + spec["modules"] = [ + promptable_item, sam_item, data_engine_item, + image_encoder_item, prompt_encoder_item, mask_decoder_item, + assisted_item, semi_item, fully_item, + ] + spec["outputs"] = [valid_mask_item, sa1b_item] + spec["innovations"] = [] + + def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) -> dict[str, Any]: + return { + "source": str(source.get("id")), + "target": str(target.get("id")), + "type": relation_type, + "label": "", + "evidence_ids": list(dict.fromkeys([*source.get("evidence_ids", []), *target.get("evidence_ids", [])])), + } + + relations = [ + relation(image_item, image_encoder_item, "data_flow"), + relation(prompt_item, prompt_encoder_item, "data_flow"), + relation(image_encoder_item, mask_decoder_item, "feature_flow"), + relation(prompt_encoder_item, mask_decoder_item, "conditioning"), + relation(mask_decoder_item, valid_mask_item, "data_flow"), + relation(assisted_item, semi_item, "stage_transition"), + relation(semi_item, fully_item, "stage_transition"), + relation(fully_item, sa1b_item, "data_generation"), + relation(sam_item, data_engine_item, "annotation_support"), + ] + spec["relations"] = relations + spec["topology"] = "dense_multiframe" + return { + "applied": True, + "removed_entities": removed_entities, + "rebuilt_relations": [f"{item['source']}->{item['target']}" for item in relations], + } + + def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: aliases: dict[str, set[str]] = {} endpoint_ids: set[str] = set() @@ -862,6 +956,10 @@ def choose_input_target(label: str) -> dict[str, Any] | None: existing_pairs.add(pair) incoming.add(target_id) spec["relations"] = relations + dense_report = _simplify_dense_multiframe_contract(spec) + completion_report["dense_multiframe_simplification_applied"] = dense_report["applied"] + completion_report["dense_multiframe_removed_entities"] = dense_report["removed_entities"] + completion_report["dense_multiframe_rebuilt_relations"] = dense_report["rebuilt_relations"] multimodal_report = _simplify_multimodal_contract(spec) completion_report["multimodal_simplification_applied"] = multimodal_report["applied"] completion_report["multimodal_removed_entities"] = multimodal_report["removed_entities"] diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 6dfc3b0..9d0504b 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -12,6 +12,22 @@ BUILTIN_TEMPLATES: dict[str, dict[str, Any]] = { + "dense-multiframe": { + "summary": "Dense three-panel overview combining task semantics, model internals, and a staged data engine.", + "template_id": "dense-multiframe", + "name": "Task, model, and data-engine overview", + "aspect_ratio": 1.78, + "topology": ["three_macro_panels", "nested_model_flow", "staged_data_engine", "model_to_data_feedback"], + "ideal_module_range": [8, 16], + "visual_density": "very_high", + "panels": [ + {"id": "task", "role": "promptable_task", "bbox_percent": {"x": 0.02, "y": 0.08, "w": 0.27, "h": 0.84}}, + {"id": "model", "role": "model_architecture", "bbox_percent": {"x": 0.34, "y": 0.08, "w": 0.34, "h": 0.84}}, + {"id": "data", "role": "data_engine", "bbox_percent": {"x": 0.73, "y": 0.08, "w": 0.25, "h": 0.84}}, + ], + "connectors": ["image_to_image_encoder", "prompt_to_prompt_encoder", "encoders_to_decoder", "decoder_to_mask", "three_stage_data_engine", "model_to_data_engine_support"], + "style": {"background": "white", "panel_fill": "soft_tint", "stroke": "muted_blue", "accent": ["blue", "green", "orange"], "corners": "rounded", "shadow": "soft_cards"}, + }, "multimodal": { "summary": "Multiple modality inputs converge through modality encoders into one joint embedding space and an emergent alignment output.", "template_id": "multimodal", @@ -240,7 +256,7 @@ def _paper_features(review: dict) -> dict: } -def select_template(profiles: list[dict], review: dict, requested: str = "auto", target_ratio: str = "auto") -> dict: +def select_template(profiles: list[dict], review: dict, requested: str = "auto", target_ratio: str = "auto", contract_topology: str | None = None) -> dict: if requested != "auto": matches = [profile for profile in profiles if profile.get("template_id") == requested or profile.get("profile_id") == requested] if not matches: @@ -266,6 +282,9 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", module_fit = 1.0 if low <= features["module_count"] <= high else max(0.0, 1.0 - min(abs(features["module_count"] - low), abs(features["module_count"] - high)) / 10) topology = 0.0 reasons = [f"module_fit={module_fit:.2f}"] + topology_template = {"dense_multiframe": "dense-multiframe"}.get(str(contract_topology or ""), str(contract_topology or "")) + if topology_template and template_id == topology_template: + topology += 1.5; reasons.append(f"normalized contract topology={contract_topology}") if template_id == "feedback" and features["has_loop"] and not features["has_tree"]: topology += 1.0; reasons.append("explicit feedback-loop topology") if template_id == "branch" and features["has_branch"] and not features["has_loop"] and not features["has_tree"] and not features["has_multimodal"] and not features["has_retrieval"]: diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index d1f4ee2..69b2085 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -26,7 +26,7 @@ def run_paper_to_image( aspect_ratio: str | None = None, language: str | None = None, image_model: str | None = None, - image_retries: int = 2, + image_retries: int = 1, review_mode: str = "heuristic", review_model: str | None = None, domain_profile: str = "auto", @@ -98,7 +98,13 @@ def run_paper_to_image( write_json(root / "preferences.json", preferences) template_profiles = build_template_profiles(archived_positive, root / "template_profiles", mode=planner_mode, model=planner_model) - selected_template = select_template(template_profiles, paper_review, requested=template, target_ratio=str(preferences.get("aspect_ratio") or "auto")) + selected_template = select_template( + template_profiles, + paper_review, + requested=template, + target_ratio=str(preferences.get("aspect_ratio") or "auto"), + contract_topology=str(plan.get("figure_specification", {}).get("topology") or "unknown"), + ) if str(preferences.get("aspect_ratio") or "auto") == "auto": ratio = float(selected_template.get("source_aspect_ratio") or selected_template.get("aspect_ratio") or 16 / 9) preferences["template_source_aspect_ratio"] = round(ratio, 6) diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index fb7076b..85dc134 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -9,6 +9,7 @@ from unittest.mock import patch import fitz +import requests from PIL import Image from rfs.cli import _doctor, _probe_executable, build_parser @@ -504,6 +505,20 @@ def test_many_modality_workflow_selects_multimodal_template(self): self.assertEqual(selected["template_id"], "multimodal") + def test_dense_contract_topology_selects_dense_multiframe_template(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + review = { + "inputs": [{"statement": "Image"}, {"statement": "Prompt"}], + "modules": [{"statement": value} for value in ("Image Encoder", "Prompt Encoder", "Mask Decoder")], + "relations": [], + "workflows": {"feedback": []}, + } + + selected = select_template(profiles, review, requested="auto", target_ratio="16:9", contract_topology="dense_multiframe") + + self.assertEqual(selected["template_id"], "dense-multiframe") + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): @@ -521,6 +536,31 @@ def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): self.assertNotIn("secret-value", manifest_text) self.assertTrue(all(item["endpoint"].endswith("/images/edits") for item in manifest["requests"])) + @patch("rfs.paper_to_image.generator.requests.post", side_effect=requests.Timeout("provider stalled")) + def test_image2_timeout_is_not_blindly_retried(self, post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE2_TIMEOUT": "30"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + + with self.assertRaises(RuntimeError): + generate_and_select( + _plan(), + {"aspect_ratio": "3:2"}, + template, + root / "layout_blueprint.png", + root, + asset_mode="image2", + candidates=1, + image_retries=3, + review_mode="vlm", + repair_rounds=0, + ocr_engine="off", + critic_adapter=_passing_critic, + ) + + self.assertEqual(post.call_count, 1) + @patch("rfs.paper_to_image.generator.requests.post") def test_repair_source_skips_fresh_candidate_generation_when_source_passes(self, post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value"}, clear=False): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 33b5994..0d58b17 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -739,6 +739,59 @@ def test_multimodal_contract_keeps_shared_embedding_story_and_removes_training_d self.assertEqual(len(spec["relations"]), 8) self.assertIn("Emergent Cross-modal Alignment", spec["required_labels"]) + def test_dense_multiframe_contract_keeps_task_model_and_data_engine_panels(self): + caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM), and a data engine for collecting SA-1B." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [ + {"id": "image", "name": "image", "evidence_ids": ["E0001"]}, + {"id": "images_alias", "name": "Images", "evidence_ids": ["E0001"]}, + {"id": "prompts", "name": "input prompts", "evidence_ids": ["E0001"]}, + {"id": "prompt", "name": "Prompt", "evidence_ids": ["E0001"]}, + ], + "modules": [ + {"id": "task", "name": "Prompt-able segmentation task", "role": "overview_panel", "evidence_ids": ["E0001"]}, + {"id": "sam", "name": "Segmentation model (SAM)", "role": "overview_panel", "evidence_ids": ["E0001"]}, + {"id": "engine", "name": "Data engine", "role": "overview_panel", "evidence_ids": ["E0001"]}, + {"id": "image_encoder", "name": "image encoder", "evidence_ids": ["E0001"]}, + {"id": "prompt_encoder", "name": "prompt encoder", "evidence_ids": ["E0001"]}, + {"id": "mask_decoder", "name": "mask decoder", "evidence_ids": ["E0001"]}, + {"id": "transformer", "name": "Transformer Encoder", "evidence_ids": ["E0001"]}, + {"id": "assisted", "name": "Assisted-Manual", "evidence_ids": ["E0001"]}, + {"id": "semi", "name": "Semi-Automatic", "evidence_ids": ["E0001"]}, + {"id": "fully", "name": "Fully Automatic", "evidence_ids": ["E0001"]}, + ], + "outputs": [ + {"id": "mask", "name": "Valid Segmentation Mask", "evidence_ids": ["E0001"]}, + {"id": "sa1b", "name": "SA-1B", "evidence_ids": ["E0001"]}, + {"id": "confidence", "name": "confidence scores", "evidence_ids": ["E0001"]}, + ], + "innovations": [], + "relations": [], + "must_show": [], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = [re.sub(r"[^a-z0-9]+", "", item["name"].casefold()) for field in ("inputs", "modules", "outputs") for item in spec[field]] + + self.assertEqual(spec["topology"], "dense_multiframe") + self.assertEqual(len(labels), 13) + self.assertEqual(len(spec["relations"]), 9) + self.assertIn("promptablesegmentationtask", labels) + self.assertIn("segmentanythingmodel", labels) + self.assertIn("dataengine", labels) + self.assertNotIn("transformerencoder", labels) + self.assertNotIn("confidencescores", labels) + def test_overview_caption_and_stage_list_complete_missing_panels(self): caption = "Figure 1: Three interconnected components: a promptable segmentation task, a segmentation model (SAM) that powers data annotation, and a data engine for collecting SA-1B, our dataset." parsed = { From 9c4b197078c775bb57474730ab0c5f3c3a189895 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 06:16:38 +0800 Subject: [PATCH 58/74] Add repeated Image2 stability audits --- README.md | 14 +++ docs/paper-to-image.md | 43 ++++++++ rfs/cli.py | 2 + rfs/evaluation/benchmarking.py | 70 +++++++++++- rfs/paper_to_image/critics.py | 15 ++- rfs/paper_to_image/generator.py | 130 ++++++++++++++++++++++- rfs/paper_to_image/preparation.py | 42 ++++++++ rfs/paper_to_image/workflow.py | 5 + tests/integration/test_paper_to_image.py | 66 ++++++++++++ tests/unit/test_benchmarking.py | 73 +++++++++++++ 10 files changed, 452 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 56066ce..83592f3 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,20 @@ Image-2 generation uses a 120-second default request timeout configured by or provider failures receive at most the configured `--image-retries` attempts, whose default is one. +Use `--candidates 3` for a minimum stability audit. Independent candidates are +generated and reviewed concurrently, and `stability_report.json` records the +production pass rate, mean score, worst-case score, standard deviation, and +failure-mode counts. Benchmark scoring consumes this report automatically +instead of treating one selected best image as evidence of stability. + +Every visible scientific label is checked against the normalized contract +whitelist. Paper-supported but out-of-scope details still fail the overview if +they were intentionally omitted from the figure contract. For interrupted +stability runs, rerun the same output directory with `--resume-candidates`; +existing images are re-reviewed and only missing candidates are generated. +Three-seed stability runs may automatically replace one provider-failed seed, +without regenerating successful candidates. + ## Offline Smoke Test Use placeholder assets to validate the local pipeline without calling any API: diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 6215c73..7d6d500 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -175,6 +175,49 @@ Image-2 edit requests default to a 120-second timeout controlled by blindly regenerating the entire image. Retryable HTTP or provider failures may retry up to `--image-retries`; the default is one. +## Stability Audit + +Production stability requires repeated independent candidates rather than a +single selected best image. Run at least three candidates: + +```powershell +rfs paper-to-image ` + --paper paper.pdf ` + --out output/stability ` + --asset-mode image2 ` + --review-mode vlm ` + --candidates 3 ` + --repair-rounds 0 +``` + +The workflow writes `stability_report.json` with `seed_count`, +`production_pass_rate`, `mean_score`, `worst_case_score`, +`standard_deviation`, per-candidate scores, and failure-mode counts. Repair +candidates and reused `--repair-source` images are excluded because they are not +independent generations. `rfs benchmark score` consumes the generated report +automatically. + +The review gate also rejects `unexpected_labels`: visible scientific labels +outside `figure_specification.required_labels` and `repeatable_labels`. This is +stricter than checking hallucinations alone because a term can be supported by +the paper yet still be outside the intended overview contract. + +If a stability run is interrupted or one provider request fails, resume it in +place: + +```powershell +rfs paper-to-image ` + --paper paper.pdf ` + --out output/stability ` + --candidates 3 ` + --resume-candidates +``` + +Existing `candidate_XX.png` files are re-reviewed with the current gates and +only missing candidates are generated. A three-seed stability run also permits +one bounded replacement for a provider-failed seed; successful candidates are +never regenerated by that replacement path. + When `--aspect-ratio auto` is used, the template keeps its internal normalized geometry while the generation canvas uses the nearest native Image2 ratio (`3:2`, `2:3`, or `1:1`). This avoids semantic cropping after generation. diff --git a/rfs/cli.py b/rfs/cli.py index fae90f7..3f0bd86 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -217,6 +217,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") paper_image.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") + paper_image.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") paper_image.add_argument("--json", action="store_true", help="Emit JSON.") @@ -466,6 +467,7 @@ def main(argv: list[str] | None = None) -> int: template=args.template, repair_rounds=args.repair_rounds, repair_source=args.repair_source, + resume_candidates=args.resume_candidates, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, ) diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 84c9b66..5c6d8f7 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -339,10 +339,57 @@ def _select_candidate_review(run: Path) -> dict: return selected.get("review", {}) if isinstance(selected, dict) else {} +def _unexpected_visible_labels(review: dict, specification: dict) -> list[str]: + allowed = { + _normalized_text(value) + for value in specification.get("required_labels", []) if _normalized_text(value) + } + allowed.update( + _normalized_text(value) + for value in specification.get("repeatable_labels", []) if _normalized_text(value) + ) + detected = review.get("ocr", {}).get("detected_labels", []) if isinstance(review.get("ocr"), dict) else [] + return [str(value) for value in detected if _normalized_text(value) and _normalized_text(value) not in allowed] + + +def _audit_candidate_stability(candidate_report: dict, specification: dict) -> dict[str, Any]: + candidates = [ + item for item in candidate_report.get("candidates", []) + if isinstance(item, dict) and str(item.get("candidate_id") or "").startswith("candidate_") and not item.get("repair_source") + ] + requested = int(candidate_report.get("requested_candidates") or len(candidates)) + if requested < 2: + return {"summary": "Repeated-generation stability audit has insufficient samples.", "seed_count": requested, "available": False} + records = [] + pass_count = 0 + for item in candidates: + review = item.get("review", {}) if isinstance(item.get("review"), dict) else {} + unexpected = _unexpected_visible_labels(review, specification) + effective_pass = bool(item.get("production_pass")) and not unexpected + pass_count += int(effective_pass) + records.append({ + "candidate_id": item.get("candidate_id"), + "reported_production_pass": bool(item.get("production_pass")), + "effective_production_pass": effective_pass, + "unexpected_labels": unexpected, + }) + pass_rate = round(pass_count / max(1, requested), 4) + return { + "summary": "Repeated-generation stability audited against the normalized visible-label whitelist.", + "available": True, + "seed_count": requested, + "production_pass_count": pass_count, + "production_pass_rate": pass_rate, + "stable": pass_rate >= 0.8, + "candidates": records, + } + + def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, Any]: case_root, run = Path(case_dir).resolve(), Path(run_dir).resolve() case = _load(case_root / "case.json") expected = _load(case_root / str(case.get("expected_semantics") or "expected_semantics.json")) + candidate_report = _load(run / "candidate_review.json") review = _select_candidate_review(run) benchmark_review = _load(run / "benchmark_review.json") if benchmark_review: @@ -354,7 +401,13 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, clarity = review.get("clarity", {}) if isinstance(review.get("clarity"), dict) else {} information = review.get("information", {}) if isinstance(review.get("information"), dict) else {} stability = review.get("stability", {}) if isinstance(review.get("stability"), dict) else {} - planning_contract = _score_planning_contract(expected, _load(run / "figure_specification.json")) + if not stability: + stability = _load(run / "stability_report.json") or (candidate_report.get("stability", {}) if isinstance(candidate_report.get("stability"), dict) else {}) + if isinstance(benchmark_review.get("stability"), dict): + stability = benchmark_review["stability"] + specification = _load(run / "figure_specification.json") + planning_contract = _score_planning_contract(expected, specification) + stability_audit = _audit_candidate_stability(candidate_report, specification) expected_entities = [item for item in expected.get("entities", []) if isinstance(item, dict) and item.get("required", True)] expected_relations = [item for item in expected.get("relations", []) if isinstance(item, dict) and item.get("required", True)] @@ -373,7 +426,14 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, aesthetic_score = _clamp(aesthetic.get("score", _mean([_clamp(value) for value in aesthetic_dimensions]))) clarity_score = _clamp(clarity.get("score", aesthetic.get("readability", 0.0))) information_score = _clamp(information.get("score", information.get("coverage", scientific_score))) - stability_score = _clamp(stability.get("production_pass_rate", stability.get("score", 0.0))) + stability_score = _clamp( + stability_audit.get("production_pass_rate") + if stability_audit.get("available") + else stability.get("production_pass_rate", stability.get("score", 0.0)) + ) + unexpected_labels = list(review.get("ocr", {}).get("unexpected_labels", []) if isinstance(review.get("ocr"), dict) else []) + if not unexpected_labels: + unexpected_labels = _unexpected_visible_labels(review, specification) hard_failures = list(review.get("hard_errors", [])) if not scientific or not aesthetic or scientific.get("engineering_only") or aesthetic.get("engineering_only"): hard_failures.append("frozen benchmark judge did not provide scientific and aesthetic sections") @@ -385,6 +445,10 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, hard_failures.append("required relations missing or incorrect") if exact_label_rate < 1.0: hard_failures.append("required labels missing or misspelled") + if unexpected_labels: + hard_failures.append("unexpected visible labels outside the normalized figure contract") + if stability_audit.get("available") and not stability_audit.get("stable"): + hard_failures.append("repeated generation stability below production threshold") plan_entity_min = float(case.get("thresholds", {}).get("plan_entity_recall", 1.0)) plan_relation_min = float(case.get("thresholds", {}).get("plan_relation_recall", 1.0)) if planning_contract.get("available") and float(planning_contract.get("entity_recall", 0.0)) < plan_entity_min: @@ -405,6 +469,7 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "stability_score": stability_score, "hallucination_count": len(invented), "forbidden_content_count": len(forbidden_found), + "unexpected_label_count": len(unexpected_labels), "plan_entity_recall": planning_contract.get("entity_recall"), "plan_relation_recall": planning_contract.get("relation_recall"), "plan_forbidden_content_count": len(planning_contract.get("forbidden_labels_found", [])), @@ -420,6 +485,7 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "run_dir": str(run), "metrics": metrics, "planning_contract": planning_contract, + "stability_audit": stability_audit, "total_score": total_score, "hard_failures": sorted(set(hard_failures)), "threshold_failures": threshold_failures, diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index a2fe84e..f078f54 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -281,10 +281,17 @@ def review_candidate( ocr.setdefault("local_engine", local_engine) ocr.setdefault("local_detected_text", []) repeatable_normalized = {_normalize_text(value) for value in repeatable_labels if _normalize_text(value)} + required_normalized = {_normalize_text(value) for value in labels if _normalize_text(value)} duplicate_labels = [str(value) for value in ocr.get("duplicate_labels", [])] allowed_duplicate_labels = [value for value in duplicate_labels if _normalize_text(value) in repeatable_normalized] ocr["allowed_duplicate_labels"] = allowed_duplicate_labels ocr["duplicate_labels"] = [value for value in duplicate_labels if _normalize_text(value) not in repeatable_normalized] + detected_labels = [str(value).strip() for value in ocr.get("detected_labels", []) if str(value).strip()] + allowed_visible_labels = required_normalized | repeatable_normalized + ocr["unexpected_labels"] = [ + value for value in detected_labels + if _normalize_text(value) and _normalize_text(value) not in allowed_visible_labels + ] scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") @@ -292,7 +299,7 @@ def review_candidate( topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) if topology_required: topology_review["passed"] = bool(topology_review.get("passed")) and topology_review["score"] >= 0.9 and topology_issues == 0 and not topology_warning - ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found"]) + ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", "unexpected_labels"]) if ocr_issues == 0: ocr["score"] = 1.0 ocr["passed"] = ocr["score"] >= 0.999 and ocr_issues == 0 @@ -330,6 +337,10 @@ def review_candidate( repair.extend(_issue_text(value) for value in topology_review.get("repair", []) if _issue_text(value)) repair_regions = list(vlm_raw.get("repair_regions", [])) repair_regions.extend(_issue_text(value) for value in topology_review.get("repair_regions", []) if _issue_text(value)) + unexpected_labels = [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] + repair.extend(f"Remove unexpected visible label: {value}" for value in unexpected_labels) + remove = list(vlm_raw.get("remove", [])) + remove.extend(unexpected_labels) production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and topology_review["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) score = 0.25 * scientific["score"] + 0.20 * ocr["score"] + 0.20 * topology_review["score"] + 0.20 * template_review["score"] + 0.15 * aesthetic["score"] return { @@ -345,7 +356,7 @@ def review_candidate( "hard_errors": hard_errors, "preserve": list(vlm_raw.get("preserve", [])), "repair": list(dict.fromkeys(repair)), - "remove": list(vlm_raw.get("remove", [])), + "remove": list(dict.fromkeys(remove)), "repair_regions": list(dict.fromkeys(repair_regions)), "local_ocr_warning": local_warning, "vlm_warning": vlm_warning, diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index 67ec719..9e47a9a 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -4,6 +4,7 @@ import json import os import shutil +import statistics import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -102,6 +103,91 @@ def _image2_edit_url() -> str: return f"{base}/images/edits" +def _candidate_failure_modes(record: dict) -> list[str]: + review = record.get("review", {}) if isinstance(record.get("review"), dict) else {} + modes = [] + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} + template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + if ocr.get("missing_labels") or ocr.get("misspelled_labels") or ocr.get("duplicate_labels") or ocr.get("unexpected_labels"): + modes.append("label_accuracy") + if scientific.get("missing_modules"): + modes.append("missing_modules") + if scientific.get("missing_relations") or topology.get("missing_relations"): + modes.append("missing_relations") + if scientific.get("reversed_relations") or topology.get("reversed_relations"): + modes.append("reversed_relations") + if topology.get("bypassed_relations"): + modes.append("bypassed_relations") + if scientific.get("invented_items") or topology.get("invented_relations"): + modes.append("invented_content") + if not template.get("passed", False): + modes.append("template_alignment") + if not aesthetic.get("passed", False): + modes.append("aesthetic_quality") + if review.get("vlm_warning") or review.get("topology_warning"): + modes.append("review_provider") + return list(dict.fromkeys(modes or (["unknown_review_failure"] if not record.get("production_pass") else []))) + + +def _build_stability_report(records: list[dict], failures: list[dict], requested_candidates: int, repair_source: Path | None) -> dict: + independent = [ + item for item in records + if str(item.get("candidate_id") or "").startswith("candidate_") and not item.get("repair_source") + ] + independent_failures = [ + item for item in failures if str(item.get("candidate_id") or "").startswith("candidate_") + ] + seed_count = 0 if repair_source else max(int(requested_candidates), len(independent) + len(independent_failures)) + score_by_id = {str(item.get("candidate_id")): float(item.get("score") or 0.0) for item in independent} + scores = [score_by_id.get(f"candidate_{index:02d}", 0.0) for index in range(1, seed_count + 1)] + pass_count = sum(bool(item.get("production_pass")) for item in independent) + pass_rate = round(pass_count / seed_count, 4) if seed_count else 0.0 + mean_score = round(statistics.fmean(scores), 4) if scores else 0.0 + worst_score = round(min(scores), 4) if scores else 0.0 + deviation = round(statistics.pstdev(scores), 4) if len(scores) >= 2 else 0.0 + failure_modes: dict[str, int] = {} + candidates = [] + for item in independent: + modes = _candidate_failure_modes(item) + for mode in modes: + failure_modes[mode] = failure_modes.get(mode, 0) + 1 + candidates.append({ + "candidate_id": item.get("candidate_id"), + "score": round(float(item.get("score") or 0.0), 4), + "production_pass": bool(item.get("production_pass")), + "failure_modes": modes, + }) + for item in independent_failures: + failure_modes["generation_provider"] = failure_modes.get("generation_provider", 0) + 1 + candidates.append({ + "candidate_id": item.get("candidate_id"), + "score": 0.0, + "production_pass": False, + "failure_modes": ["generation_provider"], + "error": str(item.get("error") or "candidate generation failed"), + }) + stable = bool(seed_count >= 2 and pass_rate >= 0.8 and worst_score >= 0.8) + status = "stable" if stable else "unstable" if seed_count >= 2 else "insufficient_samples" + return { + "summary": "Cross-candidate production stability for independent Image2 generations.", + "status": status, + "stable": stable, + "seed_count": seed_count, + "completed_seed_count": len(independent), + "production_pass_count": pass_count, + "production_pass_rate": pass_rate, + "mean_score": mean_score, + "worst_case_score": worst_score, + "standard_deviation": deviation, + "minimum_recommended_seed_count": 3, + "candidate_scores": sorted(candidates, key=lambda item: str(item.get("candidate_id") or "")), + "failure_mode_counts": dict(sorted(failure_modes.items())), + } + + def _safe_endpoint(value: str) -> str: parts = urlsplit(value) return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) @@ -207,6 +293,7 @@ def generate_and_select( ocr_adapter: Callable | None = None, critic_adapter: Callable | None = None, topology_adapter: Callable | None = None, + resume_candidates: bool = False, ) -> dict: root = ensure_dir(out_dir) candidate_dir = ensure_dir(root / "candidates") @@ -240,9 +327,13 @@ def generate_candidate(index: int) -> dict[str, Any]: write_text(prompt_path, prompt) started = time.monotonic() try: - generation_started = time.monotonic() - generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) - generation_seconds = round(time.monotonic() - generation_started, 3) + if resume_candidates and path.exists(): + generation = {"mode": "existing_candidate_resume", "source": str(path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + generation_seconds = 0.0 + else: + generation_started = time.monotonic() + generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) + generation_seconds = round(time.monotonic() - generation_started, 3) review_started = time.monotonic() review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) review_seconds = round(time.monotonic() - review_started, 3) @@ -300,6 +391,35 @@ def generate_candidate(index: int) -> dict[str, Any]: requests_manifest.append(result["request"]) elif result.get("failure"): failures.append(result["failure"]) + replacement_budget = 0 + if asset_mode == "image2" and count >= 3: + try: + replacement_budget = max(0, min(2, int(os.getenv("RFS_STABILITY_REPLACEMENT_RETRIES", "1")))) + except ValueError: + replacement_budget = 1 + for replacement_attempt in range(1, replacement_budget + 1): + retryable = [ + item for item in failures + if any(token in str(item.get("error") or "").casefold() for token in ("timed out", "timeout", "http 429", "http 5", "provider")) + ] + if not retryable: + break + for failure in retryable: + try: + candidate_index = int(str(failure.get("candidate_id") or "").rsplit("_", 1)[-1]) + except ValueError: + continue + result = generate_candidate(candidate_index) + failures.remove(failure) + if result.get("record"): + result["record"]["provider_replacement_attempt"] = replacement_attempt + result["request"]["provider_replacement_attempt"] = replacement_attempt + records.append(result["record"]) + requests_manifest.append(result["request"]) + else: + replacement_failure = result.get("failure", failure) + replacement_failure["provider_replacement_attempts"] = replacement_attempt + failures.append(replacement_failure) records.sort(key=lambda item: item["candidate_id"]) requests_manifest.sort(key=lambda item: item["candidate_id"]) failures.sort(key=lambda item: item["candidate_id"]) @@ -356,7 +476,9 @@ def generate_candidate(index: int) -> dict[str, Any]: engineering_preview = root / "engineering_preview.png" shutil.copyfile(selected["path"], engineering_preview) - report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} + stability = _build_stability_report(records, failures, count, repair_source_path) + write_json(root / "stability_report.json", stability) + report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "resume_candidates": bool(resume_candidates), "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "stability": stability, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} write_json(root / "candidate_review.json", report) if report["delivery_blocked"]: raise RuntimeError("No Image2 candidate passed scientific, OCR, template, and aesthetic production gates") diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 206277a..6a4970e 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -1070,6 +1070,47 @@ def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: } +def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: + spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + entity_ids = [str(item.get("id") or "") for item in entities if str(item.get("id") or "")] + labels = [_item_label(item) for item in entities if _item_label(item)] + spec["storyline"] = labels + spec["must_show"] = labels + topology = str(spec.get("topology") or "unknown") + if topology == "multimodal": + spec["training_flow"] = [] + spec["inference_flow"] = [] + plan["figure_specification"] = spec + plan["design_plan"] = { + "summary": "Information narrative synchronized to the normalized scientific contract.", + "reading_order": entity_ids, + "groups": [], + "innovation_emphasis": [_item_label(item) for item in spec.get("innovations", []) if isinstance(item, dict) and _item_label(item)], + "preserve": labels, + "remove": list(spec.get("forbidden_inventions", []) if isinstance(spec.get("forbidden_inventions"), list) else []), + } + layout = plan.get("layout_intent") if isinstance(plan.get("layout_intent"), dict) else {} + layout.update({ + "summary": "Layout intent synchronized to the normalized scientific topology.", + "pattern": {"feedback": "loop", "multimodal": "hub_and_spoke", "dense_multiframe": "stacked", "branch": "two_stage"}.get(topology, "left_to_right"), + "flow_description": "Render only declared entities and directed relations; do not add implementation details outside the visible-label whitelist.", + }) + plan["layout_intent"] = layout + plan["visual_metaphors"] = { + "summary": "Paper-grounded visual metaphors synchronized to the normalized contract.", + "items": [ + {"module_id": str(item.get("id") or ""), "metaphor": _item_label(item), "must_show": [], "avoid_showing": list(spec.get("forbidden_inventions", []))} + for item in entities + ], + } + + def merge_review_grounded_contract(primary: dict[str, Any], fallback: dict[str, Any]) -> dict[str, Any]: primary_spec = primary.get("figure_specification") if isinstance(primary.get("figure_specification"), dict) else {} fallback_spec = fallback.get("figure_specification") if isinstance(fallback.get("figure_specification"), dict) else {} @@ -1326,6 +1367,7 @@ def prepare_paper_figure_contract( write_json(root / "planning_metadata.json", {"summary": "Planner execution metadata.", **planner_metadata}) expand_plan_evidence(plan, paper_review, parsed) normalize_figure_contract(plan, parsed) + synchronize_plan_to_contract(plan) completion_report = plan.get("contract_completion_report") if isinstance(plan.get("contract_completion_report"), dict) else {} write_json(root / "contract_completion_report.json", completion_report) for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 69b2085..7d847f7 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -38,6 +38,7 @@ def run_paper_to_image( ocr_adapter=None, critic_adapter=None, topology_adapter=None, + resume_candidates: bool = False, ) -> dict: started = time.time() effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" @@ -139,6 +140,7 @@ def run_paper_to_image( "template": selected_template["template_id"], "repair_rounds": max(0, min(1, int(repair_rounds))), "repair_source": str(Path(repair_source).resolve()) if repair_source else None, + "resume_candidates": bool(resume_candidates), "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, } @@ -163,6 +165,7 @@ def run_paper_to_image( ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, + resume_candidates=resume_candidates, ) elapsed = round(time.time() - started, 3) run_summary: dict[str, Any] = { @@ -184,6 +187,7 @@ def run_paper_to_image( "selected_candidate_id": generation["selected_candidate_id"], "selected_passed_all_checks": generation["selected_passed_all_checks"], "repair_source": str(Path(repair_source).resolve()) if repair_source else None, + "resume_candidates": bool(resume_candidates), "elapsed_seconds": elapsed, "pptx_generated": False, "artifacts": [ @@ -208,6 +212,7 @@ def run_paper_to_image( "generation_parameters.json", "image2_request_manifest.json", "candidate_review.json", + "stability_report.json", "ocr_review.json", "template_alignment_report.json", "scientific_critic_report.json", diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 85dc134..e618179 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -232,6 +232,26 @@ def critic(path, template, prompt): self.assertEqual(review["ocr"]["allowed_duplicate_labels"], ["Document Parser"]) self.assertEqual(review["ocr"]["duplicate_labels"], []) + def test_candidate_review_rejects_visible_labels_outside_contract_whitelist(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].append("Contrastive Loss") + return result + + review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertFalse(review["production_pass"]) + self.assertEqual(review["ocr"]["unexpected_labels"], ["Contrastive Loss"]) + self.assertIn("Remove unexpected visible label: Contrastive Loss", review["repair"]) + self.assertIn("Contrastive Loss", review["remove"]) + def test_complex_feedback_candidate_requires_focused_topology_pass(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -535,6 +555,10 @@ def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): self.assertEqual(len(manifest["requests"]), 3) self.assertNotIn("secret-value", manifest_text) self.assertTrue(all(item["endpoint"].endswith("/images/edits") for item in manifest["requests"])) + self.assertEqual(result["stability"]["seed_count"], 3) + self.assertEqual(result["stability"]["production_pass_rate"], 1.0) + self.assertEqual(result["stability"]["status"], "stable") + self.assertTrue((root / "stability_report.json").exists()) @patch("rfs.paper_to_image.generator.requests.post", side_effect=requests.Timeout("provider stalled")) def test_image2_timeout_is_not_blindly_retried(self, post): @@ -561,6 +585,48 @@ def test_image2_timeout_is_not_blindly_retried(self, post): self.assertEqual(post.call_count, 1) + @patch("rfs.paper_to_image.generator.requests.post") + def test_three_seed_stability_replaces_only_provider_failed_candidate(self, post): + calls = {"count": 0} + + def response(*_args, **_kwargs): + calls["count"] += 1 + if calls["count"] == 1: + raise requests.Timeout("provider stalled") + return FakeResponse() + + post.side_effect = response + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_STABILITY_REPLACEMENT_RETRIES": "1", "RFS_PAPER_IMAGE_WORKERS": "3"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + + result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic) + + self.assertEqual(post.call_count, 4) + self.assertEqual(result["stability"]["production_pass_rate"], 1.0) + self.assertEqual(sum("provider_replacement_attempt" in item for item in result["candidates"]), 1) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_resume_candidates_reuses_existing_images_and_generates_only_missing_seed(self, post): + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_PAPER_IMAGE_WORKERS": "3"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + candidate_dir = root / "candidates" + candidate_dir.mkdir() + Image.new("RGB", (1536, 1024), "white").save(candidate_dir / "candidate_01.png") + Image.new("RGB", (1536, 1024), "white").save(candidate_dir / "candidate_02.png") + + result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic, resume_candidates=True) + + self.assertEqual(post.call_count, 1) + modes = {item["candidate_id"]: item["generation"]["mode"] for item in result["candidates"]} + self.assertEqual(modes["candidate_01"], "existing_candidate_resume") + self.assertEqual(modes["candidate_02"], "existing_candidate_resume") + self.assertEqual(modes["candidate_03"], "edit") + self.assertEqual(result["stability"]["production_pass_rate"], 1.0) + @patch("rfs.paper_to_image.generator.requests.post") def test_repair_source_skips_fresh_candidate_generation_when_source_passes(self, post): with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value"}, clear=False): diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index 2f197b6..c3bcd3c 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -258,6 +258,79 @@ def test_paper_to_image_score_checks_planning_contract(self): self.assertIn("paper planning missed required benchmark entities", result["hard_failures"]) self.assertIn("paper planning missed required benchmark relations", result["hard_failures"]) + def test_paper_to_image_score_uses_generated_stability_report(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.95, "review": review}], + }), encoding="utf-8") + (run / "stability_report.json").write_text(json.dumps({ + "summary": "Generated stability.", + "seed_count": 3, + "mean_score": 0.94, + "worst_case_score": 0.91, + "standard_deviation": 0.02, + "production_pass_rate": 1.0, + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertEqual(result["metrics"]["stability_score"], 1.0) + self.assertEqual(result["total_score"], 0.9575) + + def test_stability_audit_rejects_candidates_with_unexpected_visible_labels(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + base_review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "aesthetic": {"score": 1.0, "readability": 1.0}, + "clarity": {"score": 1.0}, + "information": {"score": 1.0}, + "hard_errors": [], + } + required = ["Raw Document", "Document Encoding", "Evidence Aggregation", "Final Prediction"] + candidates = [] + for index in range(1, 4): + labels = [*required, "Contrastive Loss"] if index == 2 else required + review = {**base_review, "ocr": {"score": 1.0, "detected_labels": labels, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}} + candidates.append({"candidate_id": f"candidate_{index:02d}", "score": 1.0, "production_pass": True, "review": review}) + (run / "candidate_review.json").write_text(json.dumps({ + "requested_candidates": 3, + "selected_candidate_id": "candidate_01", + "candidates": candidates, + "stability": {"production_pass_rate": 1.0}, + }), encoding="utf-8") + (run / "figure_specification.json").write_text(json.dumps({ + "inputs": [{"id": "raw", "name": "Raw Document"}], + "modules": [{"id": "encoding", "name": "Document Encoding"}, {"id": "aggregation", "name": "Evidence Aggregation"}], + "outputs": [{"id": "prediction", "name": "Final Prediction"}], + "relations": [ + {"source": "raw", "target": "encoding"}, + {"source": "encoding", "target": "aggregation"}, + {"source": "aggregation", "target": "prediction"}, + ], + "required_labels": required, + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertEqual(result["metrics"]["stability_score"], 0.6667) + self.assertIn("repeated generation stability below production threshold", result["hard_failures"]) + second = next(item for item in result["stability_audit"]["candidates"] if item["candidate_id"] == "candidate_02") + self.assertEqual(second["unexpected_labels"], ["Contrastive Loss"]) + def test_full_slide_picture_is_detected_as_editability_cheating(self): case_dir = ROOT / "benchmarks" / "image-to-ppt" / "cases" / "001_three_stage_layout" with tempfile.TemporaryDirectory() as tmp: From 75217361163b21ce84132ae765fefc4afe44924d Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 08:05:11 +0800 Subject: [PATCH 59/74] Stabilize feedback and dense Image2 topology --- README.md | 9 +- docs/paper-to-image.md | 24 +- rfs/evaluation/benchmarking.py | 4 + rfs/paper_to_image/critics.py | 10 +- rfs/paper_to_image/planner.py | 60 +++++ rfs/paper_to_image/preparation.py | 95 +++++++- rfs/paper_to_image/templates.py | 272 +++++++++++++++++++---- rfs/paper_to_image/workflow.py | 7 +- tests/integration/test_paper_to_image.py | 193 ++++++++++++++++ tests/unit/test_paper_contract.py | 59 ++++- 10 files changed, 685 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 83592f3..c917abd 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,10 @@ figure rather than an editable PowerPoint file. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts positive references into content-free architecture templates, selects a template, renders `layout_blueprint.png`, and uses Image2 edit to create and review three -candidates. It never invokes the PPTX compiler. +candidates. For normalized feedback-loop and SAM-style dense-multiframe +contracts, the blueprint also embeds exact paper labels and mandatory connector +geometry so Image2 edits a scientific source of truth instead of guessing node +mapping from empty boxes. It never invokes the PPTX compiler. Offline engineering validation: @@ -210,7 +213,9 @@ systems with parallel prediction heads use `branch`; true search-tree systems use `arbor`; simple sequential systems use `linear`. Feedback, branch, multimodal, and dense candidates also run a focused connector judge that verifies visible arrow endpoints and rejects shortcuts that bypass required -modules. +modules. The judge understands explicit containment: an evidence-supported +repeatable shared component inside a labeled operation container may implement +that container's outgoing relation without being treated as an invented edge. The general production critic and focused connector critic run concurrently. Each uses a 90-second default timeout with no automatic full-call retry; override diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 7d6d500..8fdde70 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -160,7 +160,22 @@ density, palette, and forbidden copied content. Automatic selection uses module count, loop/tree/parallel-branch structure, multimodality, retrieval structure, and requested ratio. -The selected profile is rendered as `layout_blueprint.png`. The blueprint contains no reference text or reference-specific objects and is the only image supplied to Image2 edit for initial candidates. +The selected profile is rendered as `layout_blueprint.png` and is the only image +supplied to Image2 edit for initial candidates. Reference-derived profiles remain +free of copied reference text and reference-specific objects. When the normalized +paper contract matches a strict built-in topology, the renderer may add exact +paper-semantic labels and connector geometry: + +- `feedback`: the forward generation chain, two independent inputs to refinement, + the self-feedback node, and the return loop terminating at feedback; +- `dense-multiframe`: task/model/data-engine panels, separate image and prompt + encoder paths, decoder-to-mask output, the data-engine stage chain, and a + separate model-to-data-engine support arrow. + +This semantic blueprint is still generated from the evidence-backed contract; +it does not copy paper figures or invent labels. It reduces topology drift by +making Image2 preserve an explicit scientific source of truth instead of +inferring node mapping from empty placeholder boxes. The general production review and focused topology review execute concurrently. Their default request timeout is 90 seconds with no automatic retry, preventing @@ -202,6 +217,13 @@ outside `figure_specification.required_labels` and `repeatable_labels`. This is stricter than checking hallucinations alone because a term can be supported by the paper yet still be outside the intended overview contract. +The focused topology judge also supports explicit containment semantics. If a +declared operation container visibly contains an evidence-supported repeatable +shared component, an arrow from that nested component to the target may satisfy +the container's outgoing relation. This is limited to repeatable labels inside +the declared source container; it cannot excuse missing cross-panel arrows, +shortcuts, reversed edges, or invented nodes. + If a stability run is interrupted or one provider request fails, resume it in place: diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 5c6d8f7..22b08e9 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -348,6 +348,10 @@ def _unexpected_visible_labels(review: dict, specification: dict) -> list[str]: _normalized_text(value) for value in specification.get("repeatable_labels", []) if _normalized_text(value) ) + allowed.update( + _normalized_text(item.get("label")) + for item in specification.get("relations", []) if isinstance(item, dict) and _normalized_text(item.get("label")) + ) detected = review.get("ocr", {}).get("detected_labels", []) if isinstance(review.get("ocr"), dict) else [] return [str(value) for value in detected if _normalized_text(value) and _normalized_text(value) not in allowed] diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index f078f54..37cc61c 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -123,7 +123,9 @@ def _normalize_section(raw: Any, name: str) -> dict: def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Callable | None = None) -> dict: - relations = collect_visual_relations(plan.get("figure_specification", {})) + specification = plan.get("figure_specification", {}) + relations = collect_visual_relations(specification) + repeatable_labels = [str(value).strip() for value in specification.get("repeatable_labels", []) if str(value).strip()] prompt = f""" # Summary @@ -132,6 +134,9 @@ def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Cal Mandatory directed connectors: {json.dumps(relations, ensure_ascii=False, indent=2)} +Evidence-supported repeatable shared-component labels: +{json.dumps(repeatable_labels, ensure_ascii=False)} + Rules: - Verify every source, target, and arrowhead direction from visible geometry. - A line that joins the outgoing side of a target or a downstream junction does not count as entering that target. @@ -139,6 +144,9 @@ def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Cal - For refinement loops, Initial Output and Self-Feedback must both enter Refine or its explicit input-side shared-model node before Refined Output. - Refined Output must return to Feedback to close the loop. - Do not penalize repeated labels explicitly allowed by the scientific contract. +- Respect explicit containment semantics. When a mandatory source is a clearly labeled operation/container and an allowed repeatable shared-component badge inside that container visibly produces the target, count the container-to-target relation as present. The nested badge explains who implements the container operation; it is not an invented extra scientific edge. +- In particular, a FEEDBACK container holding an allowed shared Model M badge with an arrow to Self-Feedback satisfies FEEDBACK -> Self-Feedback. Do not require an unnatural arrow starting at the container border or title text. +- Apply the containment exception only when the shared badge is visibly inside the declared source container and is listed above as repeatable. Do not use it to excuse missing cross-container arrows or shortcuts. Return: {{ diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 812b562..3fb79a2 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -530,6 +530,9 @@ def add(value: object) -> None: for field in ("inputs", "modules", "outputs"): for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: add(item) + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if isinstance(relation, dict): + add(relation.get("label")) return labels[: max(1, int(limit))] @@ -575,6 +578,59 @@ def collect_visual_relations(spec: dict) -> list[dict[str, str]]: return result +def _topology_specific_prompt_rules(spec: dict) -> str: + topology = str(spec.get("topology") or "unknown") + normalized_labels = { + re.sub(r"[^\w\u4e00-\u9fff]+", "", str(item.get("name") or item.get("label") or item.get("title") or "").casefold()) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + } + feedback_signature = { + "inputx", "generate", "initialoutput", "feedback", "selffeedback", "refine", "refinedoutput", "modelm", + } + dense_signature = { + "promptablesegmentationtask", "segmentanythingmodel", "dataengine", "image", "prompt", + "imageencoder", "promptencoder", "maskdecoder", "validsegmentationmask", + "assistedmanual", "semiautomatic", "fullyautomatic", "sa1b", + } + if topology == "feedback" and feedback_signature.issubset(normalized_labels): + return """Feedback-loop topology hard rules: +- Keep one top-row forward chain: input x -> Generate -> Initial Output -> FEEDBACK. +- Initial Output must have exactly two distinct outgoing arrows: one to FEEDBACK and one directly to REFINE. +- FEEDBACK must produce Self-Feedback, and Self-Feedback must have its own distinct outgoing arrow into REFINE. +- REFINE must therefore receive exactly two visually separate incoming arrows: Initial Output -> REFINE and Self-Feedback -> REFINE. Do not merge them upstream and do not redirect either arrow to the other source node. +- The connector labeled iterate must start at Refined output and terminate only at FEEDBACK. Its arrowhead must visibly touch the FEEDBACK boundary. +- Model M is a reusable badge inside Generate, FEEDBACK, and/or REFINE; it is not a separate flow node and must not receive extra arrows. +- Forbidden connectors: Initial Output -> Self-Feedback; Refined output -> Generate; REFINE -> Self-Feedback; Self-Feedback -> FEEDBACK; FEEDBACK -> REFINE as a shortcut that omits the visible Self-Feedback node. +- Preserve the supplied blueprint's two-row geometry and right-side return loop. Do not rearrange the loop into a different circular or bottom-return layout.""" + if topology == "feedback": + return """Feedback-loop topology hard rules: +- Follow only the declared nodes and directed relations in the scientific contract. +- Keep the forward path and return path visually separate, with the return arrowhead touching its declared target. +- Preserve every declared intermediate feedback artifact; do not shortcut across it. +- If two declared sources enter one refinement node, draw two visibly distinct incoming arrows. +- Do not introduce Self-Refine-specific labels, shared-model badges, or connector names unless they are present in the contract.""" + if topology == "dense_multiframe" and dense_signature.issubset(normalized_labels): + return """Dense multi-frame topology hard rules: +- Preserve exactly three macro panels: Promptable Segmentation Task, Segment Anything Model, and Data Engine. +- Image must connect only to Image Encoder; Prompt must connect only to Prompt Encoder. +- Image Encoder and Prompt Encoder must each have a separate directed arrow into Mask Decoder. Do not chain one encoder through the other. +- Mask Decoder must connect directly to Valid Segmentation Mask inside the model panel. +- Data Engine must be a separate vertical chain: Assisted-manual -> Semi-automatic -> Fully Automatic -> SA-1B. +- Draw the Segment Anything Model -> Data Engine annotation-support relation as a distinct high-level container-to-container arrow, visually separate from the mask-prediction path. +- Forbidden connectors: Prompt -> Image Encoder; Image -> Prompt Encoder; Image Encoder -> Prompt Encoder; Prompt Encoder -> Image Encoder; Fully Automatic -> Valid Segmentation Mask; Mask Decoder -> Data Engine as a replacement for the required container-level support arrow. +- Do not let any Data Engine stage enter Valid Segmentation Mask, and do not let the model inference path enter SA-1B.""" + if topology == "dense_multiframe": + return """Dense multi-frame topology hard rules: +- Preserve the declared macro-panel grouping and every directed cross-panel relation. +- Keep independent input branches separate until their declared convergence node. +- Keep panel-local output paths separate from dataset, training, or data-engine paths. +- Draw high-level container-to-container relations separately from internal module arrows. +- Do not introduce SAM-specific encoders, masks, data-engine stages, or dataset labels unless they are present in the contract.""" + return "No additional topology-specific rules. Follow the directed connector checklist exactly." + + def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = 1, selected_template: dict | None = None) -> str: spec = plan["figure_specification"] design = plan["design_plan"] @@ -594,6 +650,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + (f" label: {item['label']}" if item["label"] else "") for index, item in enumerate(visual_relations, start=1) ) + topology_specific_rules = _topology_specific_prompt_rules(spec) return f""" # Summary @@ -635,6 +692,9 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows. +Topology-specific rendering rules: +{topology_specific_rules} + Forbidden copied reference terms: {json.dumps(template.get('forbidden_copy_terms', []), ensure_ascii=False)} diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 6a4970e..8c0b8a4 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -651,6 +651,86 @@ def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) } +def _simplify_feedback_contract(spec: dict[str, Any]) -> dict[str, Any]: + if str(spec.get("topology") or "") != "feedback": + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + all_entities = [ + (field, item) + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) + ] + + def find(labels: set[str]) -> list[tuple[str, dict[str, Any]]]: + return [(field, item) for field, item in all_entities if _normalized_label(_item_label(item)) in labels] + + groups = { + "input": find({"input", "inputx", "taskprompt", "taskinstruction"}), + "generate": find({"generate", "init", "generator"}), + "initial": find({"initialoutput"}), + "feedback": find({"feedback", "feedbackprovider"}), + "self_feedback": find({"selffeedback", "selffeedbacknl"}), + "refine": find({"refine", "iteraterefine", "refiner"}), + "refined": find({"refinedoutput"}), + "model": find({"modelm"}), + } + if any(not groups[key] for key in ("input", "generate", "initial", "feedback", "self_feedback", "refine", "refined")): + return {"applied": False, "removed_entities": [], "rebuilt_relations": []} + + def merge(group: list[tuple[str, dict[str, Any]]], name: str, role: str) -> dict[str, Any]: + canonical = group[0][1] + canonical["name"] = name + canonical["role"] = role + canonical["evidence_ids"] = list(dict.fromkeys(value for _, item in group for value in item.get("evidence_ids", []))) + return canonical + + input_item = merge(groups["input"], "input x", "input") + generate_item = merge(groups["generate"], "Generate", "generator") + initial_item = merge(groups["initial"], "Initial Output", "artifact") + feedback_item = merge(groups["feedback"], "FEEDBACK", "feedback module") + self_feedback_item = merge(groups["self_feedback"], "Self-Feedback", "feedback artifact") + refine_item = merge(groups["refine"], "REFINE", "refiner") + refined_item = merge(groups["refined"], "Refined output", "output") + model_item = merge(groups["model"], "Model M", "shared model") if groups["model"] else None + kept = [input_item, generate_item, initial_item, feedback_item, self_feedback_item, refine_item, refined_item] + if model_item: + kept.append(model_item) + keep_ids = {str(item.get("id")) for item in kept if item.get("id")} + removed_entities = [str(item.get("id")) for _, item in all_entities if item.get("id") and str(item.get("id")) not in keep_ids] + spec["inputs"] = [input_item] + spec["modules"] = [item for item in (model_item, generate_item, initial_item, feedback_item, self_feedback_item, refine_item) if item] + spec["outputs"] = [refined_item] + spec["innovations"] = [] + + def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) -> dict[str, Any]: + return { + "source": str(source.get("id")), + "target": str(target.get("id")), + "type": relation_type, + "label": "iterate" if relation_type == "feedback_loop" else "", + "evidence_ids": list(dict.fromkeys([*source.get("evidence_ids", []), *target.get("evidence_ids", [])])), + } + + relations = [ + relation(input_item, generate_item, "data_flow"), + relation(generate_item, initial_item, "data_flow"), + relation(initial_item, feedback_item, "evaluation"), + relation(feedback_item, self_feedback_item, "feedback"), + relation(initial_item, refine_item, "revision_input"), + relation(self_feedback_item, refine_item, "feedback"), + relation(refine_item, refined_item, "data_flow"), + relation(refined_item, feedback_item, "feedback_loop"), + ] + spec["relations"] = relations + spec["feedback_loops"] = [relations[-1]] + spec["topology"] = "feedback" + return { + "applied": True, + "removed_entities": removed_entities, + "rebuilt_relations": [f"{item['source']}->{item['target']}" for item in relations], + } + + def _normalize_contract_relations(spec: dict[str, Any]) -> list[dict[str, Any]]: aliases: dict[str, set[str]] = {} endpoint_ids: set[str] = set() @@ -956,6 +1036,10 @@ def choose_input_target(label: str) -> dict[str, Any] | None: existing_pairs.add(pair) incoming.add(target_id) spec["relations"] = relations + feedback_report = _simplify_feedback_contract(spec) + completion_report["feedback_simplification_applied"] = feedback_report["applied"] + completion_report["feedback_removed_entities"] = feedback_report["removed_entities"] + completion_report["feedback_rebuilt_relations"] = feedback_report["rebuilt_relations"] dense_report = _simplify_dense_multiframe_contract(spec) completion_report["dense_multiframe_simplification_applied"] = dense_report["applied"] completion_report["dense_multiframe_removed_entities"] = dense_report["removed_entities"] @@ -1078,8 +1162,11 @@ def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) if isinstance(item, dict) ] - entity_ids = [str(item.get("id") or "") for item in entities if str(item.get("id") or "")] - labels = [_item_label(item) for item in entities if _item_label(item)] + required_labels = [str(value).strip() for value in spec.get("required_labels", []) if str(value).strip()] + allowed = {_normalized_label(value) for value in [*required_labels, *spec.get("repeatable_labels", [])] if _normalized_label(value)} + visible_entities = [item for item in entities if _normalized_label(_item_label(item)) in allowed] if allowed else entities + entity_ids = [str(item.get("id") or "") for item in visible_entities if str(item.get("id") or "")] + labels = required_labels or [_item_label(item) for item in visible_entities if _item_label(item)] spec["storyline"] = labels spec["must_show"] = labels topology = str(spec.get("topology") or "unknown") @@ -1091,7 +1178,7 @@ def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: "summary": "Information narrative synchronized to the normalized scientific contract.", "reading_order": entity_ids, "groups": [], - "innovation_emphasis": [_item_label(item) for item in spec.get("innovations", []) if isinstance(item, dict) and _item_label(item)], + "innovation_emphasis": [_item_label(item) for item in spec.get("innovations", []) if isinstance(item, dict) and _normalized_label(_item_label(item)) in allowed], "preserve": labels, "remove": list(spec.get("forbidden_inventions", []) if isinstance(spec.get("forbidden_inventions"), list) else []), } @@ -1106,7 +1193,7 @@ def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: "summary": "Paper-grounded visual metaphors synchronized to the normalized contract.", "items": [ {"module_id": str(item.get("id") or ""), "metaphor": _item_label(item), "must_show": [], "avoid_showing": list(spec.get("forbidden_inventions", []))} - for item in entities + for item in visible_entities ], } diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 9d0504b..ccc4012 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any -from PIL import Image, ImageDraw +from PIL import Image, ImageDraw, ImageFont from ..utils import ensure_dir, write_json from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available @@ -72,12 +72,12 @@ "ideal_module_range": [4, 9], "visual_density": "high", "panels": [ - {"id": "task_input", "role": "input", "bbox_percent": {"x": 0.02, "y": 0.28, "w": 0.13, "h": 0.43}}, - {"id": "generation", "role": "initial_generation", "bbox_percent": {"x": 0.20, "y": 0.10, "w": 0.20, "h": 0.28}}, - {"id": "initial_output", "role": "initial_output", "bbox_percent": {"x": 0.44, "y": 0.10, "w": 0.18, "h": 0.28}}, - {"id": "feedback", "role": "self_feedback", "bbox_percent": {"x": 0.72, "y": 0.29, "w": 0.20, "h": 0.30}}, - {"id": "refinement", "role": "refinement", "bbox_percent": {"x": 0.24, "y": 0.63, "w": 0.22, "h": 0.24}}, - {"id": "refined_output", "role": "refined_output", "bbox_percent": {"x": 0.54, "y": 0.63, "w": 0.20, "h": 0.24}}, + {"id": "task_input", "role": "input", "bbox_percent": {"x": 0.02, "y": 0.12, "w": 0.12, "h": 0.24}}, + {"id": "generation", "role": "initial_generation", "bbox_percent": {"x": 0.18, "y": 0.10, "w": 0.20, "h": 0.28}}, + {"id": "initial_output", "role": "initial_output", "bbox_percent": {"x": 0.42, "y": 0.10, "w": 0.18, "h": 0.28}}, + {"id": "feedback", "role": "self_feedback", "bbox_percent": {"x": 0.70, "y": 0.10, "w": 0.22, "h": 0.36}}, + {"id": "refinement", "role": "refinement", "bbox_percent": {"x": 0.42, "y": 0.62, "w": 0.20, "h": 0.25}}, + {"id": "refined_output", "role": "refined_output", "bbox_percent": {"x": 0.68, "y": 0.62, "w": 0.20, "h": 0.25}}, ], "connectors": ["input_to_generation", "generation_to_initial_output", "initial_output_to_feedback", "initial_output_to_refinement", "feedback_to_refinement", "refinement_to_refined_output", "refined_output_to_feedback_loop"], "style": {"background": "white", "panel_fill": "near_white", "stroke": "muted_blue", "accent": ["navy", "teal", "orange"], "corners": "rounded", "shadow": "soft_cards"}, @@ -327,12 +327,78 @@ def _canvas_size(profile: dict, target_ratio: str) -> tuple[int, int, float]: return width, height, ratio -def render_layout_blueprint(profile: dict, out_path: str | Path, target_ratio: str = "auto") -> dict: +def render_layout_blueprint( + profile: dict, + out_path: str | Path, + target_ratio: str = "auto", + figure_specification: dict[str, Any] | None = None, +) -> dict: width, height, ratio = _canvas_size(profile, target_ratio) image = Image.new("RGB", (width, height), (250, 250, 249)) draw = ImageDraw.Draw(image) panels = profile.get("panels", []) + + def normalized(value: Any) -> str: + return "".join(char for char in str(value or "").casefold() if char.isalnum()) + + exact_labels: dict[str, str] = {} + if figure_specification: + for field in ("inputs", "modules", "outputs", "innovations"): + for item in figure_specification.get(field, []) if isinstance(figure_specification.get(field), list) else []: + if not isinstance(item, dict): + continue + label = str(item.get("name") or item.get("label") or item.get("title") or "").strip() + if label: + exact_labels[normalized(label)] = label + for label in figure_specification.get("required_labels", []) if isinstance(figure_specification.get("required_labels"), list) else []: + if str(label).strip(): + exact_labels.setdefault(normalized(label), str(label).strip()) + + def exact(canonical: str) -> str: + return exact_labels.get(normalized(canonical), canonical) + + feedback_required = { + normalized(value) + for value in ("input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M") + } + semantic_feedback = profile.get("template_id") == "feedback" and feedback_required.issubset(exact_labels) + dense_required = { + normalized(value) + for value in ( + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ) + } + semantic_dense = profile.get("template_id") == "dense-multiframe" and dense_required.issubset(exact_labels) + semantic_blueprint = semantic_feedback or semantic_dense + + def font(size: int, bold: bool = False): + candidates = ["arialbd.ttf", "DejaVuSans-Bold.ttf"] if bold else ["arial.ttf", "DejaVuSans.ttf"] + for candidate in candidates: + try: + return ImageFont.truetype(candidate, max(12, size)) + except OSError: + continue + return ImageFont.load_default() + + def centered_text(box: tuple[int, int, int, int], value: str, size: int, fill=(42, 72, 91), bold: bool = True) -> None: + fitted_size = max(12, size) + selected_font = font(fitted_size, bold=bold) + bounds = draw.textbbox((0, 0), value, font=selected_font) + available_width = max(8, box[2] - box[0] - 12) + while bounds[2] - bounds[0] > available_width and fitted_size > 12: + fitted_size -= 1 + selected_font = font(fitted_size, bold=bold) + bounds = draw.textbbox((0, 0), value, font=selected_font) + text_width = bounds[2] - bounds[0] + text_height = bounds[3] - bounds[1] + x = box[0] + ((box[2] - box[0]) - text_width) / 2 + y = box[1] + ((box[3] - box[1]) - text_height) / 2 - bounds[1] + draw.text((x, y), value, font=selected_font, fill=fill) + centers = [] + panel_boxes = [] for index, panel in enumerate(panels): bbox = panel["bbox_percent"] x0, y0 = int(bbox["x"] * width), int(bbox["y"] * height) @@ -351,38 +417,168 @@ def render_layout_blueprint(profile: dict, out_path: str | Path, target_ratio: s else: draw.rounded_rectangle((x0, y0, x1, y1), radius=max(10, round(height * 0.025)), fill=fill, outline=(104, 132, 151), width=3) centers.append(((x0 + x1) // 2, (y0 + y1) // 2)) - slots = max(2, min(6, round((x1 - x0) / max(width * 0.12, 1)))) - for slot_index in range(slots): - sx0 = x0 + int((slot_index + 0.2) / slots * (x1 - x0)) - sx1 = x0 + int((slot_index + 0.8) / slots * (x1 - x0)) - sy0 = y0 + int(0.28 * (y1 - y0)) - sy1 = y0 + int(0.70 * (y1 - y0)) - draw.rounded_rectangle((sx0, sy0, sx1, sy1), radius=8, outline=(178, 190, 198), width=2) - for index in range(len(centers) - 1): - start = centers[index] - end = centers[index + 1] - draw.line((start, end), fill=(80, 111, 134), width=4) - angle = math.atan2(end[1] - start[1], end[0] - start[0]) + panel_boxes.append((x0, y0, x1, y1)) + if not semantic_blueprint: + slots = max(2, min(6, round((x1 - x0) / max(width * 0.12, 1)))) + for slot_index in range(slots): + sx0 = x0 + int((slot_index + 0.2) / slots * (x1 - x0)) + sx1 = x0 + int((slot_index + 0.8) / slots * (x1 - x0)) + sy0 = y0 + int(0.28 * (y1 - y0)) + sy1 = y0 + int(0.70 * (y1 - y0)) + draw.rounded_rectangle((sx0, sy0, sx1, sy1), radius=8, outline=(178, 190, 198), width=2) + def draw_arrow_path(points: list[tuple[int, int]], color: tuple[int, int, int] = (80, 111, 134)) -> None: + draw.line(points, fill=color, width=4, joint="curve") + if len(points) < 2: + return + tail, end = points[-2], points[-1] + angle = math.atan2(end[1] - tail[1], end[0] - tail[0]) arrow = 12 points = [end, (end[0] - arrow * math.cos(angle - 0.45), end[1] - arrow * math.sin(angle - 0.45)), (end[0] - arrow * math.cos(angle + 0.45), end[1] - arrow * math.sin(angle + 0.45))] - draw.polygon(points, fill=(80, 111, 134)) - if profile.get("template_id") == "feedback" and len(centers) >= 6: - extra_pairs = [(centers[2], centers[4]), (centers[5], centers[3])] - for pair_index, (start, end) in enumerate(extra_pairs): - if pair_index == 0: - mid_y = max(start[1], end[1]) - path = [start, (start[0], mid_y), end] - else: - loop_x = min(width - 18, max(start[0], end[0]) + round(width * 0.08)) - path = [start, (loop_x, start[1]), (loop_x, end[1]), end] - draw.line(path, fill=(31, 117, 126), width=4, joint="curve") - if len(path) >= 2: - tail, head = path[-2], path[-1] - angle = math.atan2(head[1] - tail[1], head[0] - tail[0]) - arrow = 12 - points = [head, (head[0] - arrow * math.cos(angle - 0.45), head[1] - arrow * math.sin(angle - 0.45)), (head[0] - arrow * math.cos(angle + 0.45), head[1] - arrow * math.sin(angle + 0.45))] - draw.polygon(points, fill=(31, 117, 126)) + draw.polygon(points, fill=color) + + def region(box, x0: float, y0: float, x1: float, y1: float) -> tuple[int, int, int, int]: + return ( + round(box[0] + x0 * (box[2] - box[0])), + round(box[1] + y0 * (box[3] - box[1])), + round(box[0] + x1 * (box[2] - box[0])), + round(box[1] + y1 * (box[3] - box[1])), + ) + + def draw_dashed_arrow(start: tuple[int, int], end: tuple[int, int], color: tuple[int, int, int] = (177, 92, 33)) -> None: + length = max(1, end[0] - start[0]) + dash = max(10, round(width * 0.008)) + gap = max(7, round(width * 0.005)) + cursor = 0 + while cursor < length - 16: + draw.line((start[0] + cursor, start[1], min(end[0] - 16, start[0] + cursor + dash), end[1]), fill=color, width=4) + cursor += dash + gap + draw_arrow_path([(end[0] - 18, end[1]), end], color) + + if profile.get("template_id") == "feedback" and len(panel_boxes) >= 6: + def left(box): return (box[0], (box[1] + box[3]) // 2) + def right(box): return (box[2], (box[1] + box[3]) // 2) + def top_at(box, fraction): return (round(box[0] + fraction * (box[2] - box[0])), box[1]) + def bottom_at(box, fraction): return (round(box[0] + fraction * (box[2] - box[0])), box[3]) + input_box, generation_box, initial_box, feedback_box, refinement_box, refined_box = panel_boxes[:6] + draw_arrow_path([right(input_box), left(generation_box)]) + draw_arrow_path([right(generation_box), left(initial_box)]) + draw_arrow_path([right(initial_box), left(feedback_box)]) + initial_start, initial_end = bottom_at(initial_box, 0.45), top_at(refinement_box, 0.35) + initial_mid_y = round((initial_start[1] + initial_end[1]) / 2) + draw_arrow_path([initial_start, (initial_start[0], initial_mid_y), (initial_end[0], initial_mid_y), initial_end], (31, 117, 126)) + feedback_start, feedback_end = bottom_at(feedback_box, 0.50), top_at(refinement_box, 0.72) + feedback_mid_y = round((feedback_start[1] + feedback_end[1]) / 2) + draw_arrow_path([feedback_start, (feedback_start[0], feedback_mid_y), (feedback_end[0], feedback_mid_y), feedback_end], (31, 117, 126)) + draw_arrow_path([right(refinement_box), left(refined_box)], (31, 117, 126)) + loop_start = right(refined_box) + loop_end = (feedback_box[2], round(feedback_box[1] + 0.70 * (feedback_box[3] - feedback_box[1]))) + loop_x = min(width - 18, max(loop_start[0], loop_end[0]) + round(width * 0.045)) + draw_arrow_path([loop_start, (loop_x, loop_start[1]), (loop_x, loop_end[1]), loop_end], (31, 117, 126)) + if semantic_feedback: + input_box, generation_box, initial_box, feedback_box, refinement_box, refined_box = panel_boxes[:6] + title_size = max(16, round(height * 0.025)) + badge_size = max(14, round(height * 0.020)) + centered_text(region(input_box, 0.05, 0.30, 0.95, 0.70), exact("input x"), title_size) + centered_text(region(generation_box, 0.05, 0.06, 0.95, 0.35), exact("Generate"), title_size) + centered_text(region(initial_box, 0.04, 0.20, 0.96, 0.52), exact("Initial Output"), title_size) + centered_text(region(feedback_box, 0.05, 0.04, 0.95, 0.25), exact("FEEDBACK"), title_size, fill=(177, 92, 33)) + centered_text(region(refinement_box, 0.05, 0.06, 0.95, 0.34), exact("REFINE"), title_size) + centered_text(region(refined_box, 0.04, 0.20, 0.96, 0.52), exact("Refined output"), title_size) + + for badge_box in ( + region(generation_box, 0.24, 0.48, 0.76, 0.76), + region(feedback_box, 0.30, 0.27, 0.70, 0.47), + region(refinement_box, 0.24, 0.48, 0.76, 0.76), + ): + draw.rounded_rectangle(badge_box, radius=10, fill=(235, 243, 248), outline=(126, 157, 177), width=2) + centered_text(badge_box, exact("Model M"), badge_size, bold=False) + + self_feedback_box = region(feedback_box, 0.13, 0.60, 0.87, 0.88) + draw.rounded_rectangle(self_feedback_box, radius=10, fill=(238, 249, 248), outline=(31, 117, 126), width=2) + centered_text(self_feedback_box, exact("Self-Feedback"), badge_size) + internal_start = (round((feedback_box[0] + feedback_box[2]) / 2), region(feedback_box, 0, 0.47, 1, 0.47)[1]) + internal_end = (round((self_feedback_box[0] + self_feedback_box[2]) / 2), self_feedback_box[1]) + draw_arrow_path([internal_start, internal_end], (31, 117, 126)) + + iterate_box = (loop_x - round(width * 0.08), round((loop_start[1] + loop_end[1]) / 2) - 22, loop_x - 4, round((loop_start[1] + loop_end[1]) / 2) + 22) + centered_text(iterate_box, exact("iterate"), badge_size, fill=(31, 117, 126), bold=False) + elif semantic_dense and len(panel_boxes) >= 3: + task_box, model_box, data_box = panel_boxes[:3] + title_size = max(15, round(height * 0.022)) + node_size = max(13, round(height * 0.017)) + + image_box = region(task_box, 0.12, 0.23, 0.88, 0.37) + prompt_box = region(task_box, 0.12, 0.57, 0.88, 0.71) + image_encoder_box = region(model_box, 0.06, 0.23, 0.43, 0.37) + prompt_encoder_box = region(model_box, 0.06, 0.57, 0.43, 0.71) + mask_decoder_box = region(model_box, 0.56, 0.38, 0.94, 0.54) + valid_mask_box = region(model_box, 0.56, 0.69, 0.94, 0.84) + assisted_box = region(data_box, 0.12, 0.22, 0.88, 0.34) + semi_box = region(data_box, 0.12, 0.41, 0.88, 0.53) + fully_box = region(data_box, 0.12, 0.60, 0.88, 0.72) + sa1b_box = region(data_box, 0.22, 0.80, 0.78, 0.91) + + def left(box): return (box[0], (box[1] + box[3]) // 2) + def right(box): return (box[2], (box[1] + box[3]) // 2) + def top(box): return ((box[0] + box[2]) // 2, box[1]) + def bottom(box): return ((box[0] + box[2]) // 2, box[3]) + + draw_arrow_path([right(image_box), left(image_encoder_box)], (42, 99, 161)) + draw_arrow_path([right(prompt_box), left(prompt_encoder_box)], (31, 117, 126)) + decoder_upper = (mask_decoder_box[0], round(mask_decoder_box[1] + 0.32 * (mask_decoder_box[3] - mask_decoder_box[1]))) + decoder_lower = (mask_decoder_box[0], round(mask_decoder_box[1] + 0.70 * (mask_decoder_box[3] - mask_decoder_box[1]))) + draw_arrow_path([right(image_encoder_box), decoder_upper], (42, 99, 161)) + draw_arrow_path([right(prompt_encoder_box), decoder_lower], (31, 117, 126)) + draw_arrow_path([bottom(mask_decoder_box), top(valid_mask_box)], (80, 111, 134)) + draw_arrow_path([bottom(assisted_box), top(semi_box)], (177, 92, 33)) + draw_arrow_path([bottom(semi_box), top(fully_box)], (177, 92, 33)) + draw_arrow_path([bottom(fully_box), top(sa1b_box)], (177, 92, 33)) + support_y = round(model_box[1] + 0.17 * (model_box[3] - model_box[1])) + draw_dashed_arrow((model_box[2], support_y), (data_box[0], support_y), (177, 92, 33)) + + node_specs = [ + (image_box, "Image", (237, 245, 252), (42, 99, 161)), + (prompt_box, "Prompt", (237, 249, 247), (31, 117, 126)), + (image_encoder_box, "Image Encoder", (237, 245, 252), (42, 99, 161)), + (prompt_encoder_box, "Prompt Encoder", (237, 249, 247), (31, 117, 126)), + (mask_decoder_box, "Mask Decoder", (244, 242, 251), (98, 84, 147)), + (valid_mask_box, "Valid Segmentation Mask", (242, 248, 243), (66, 126, 80)), + (assisted_box, "Assisted-manual", (252, 244, 235), (177, 92, 33)), + (semi_box, "Semi-automatic", (252, 244, 235), (177, 92, 33)), + (fully_box, "Fully Automatic", (252, 244, 235), (177, 92, 33)), + (sa1b_box, "SA-1B", (247, 239, 229), (150, 76, 30)), + ] + for node_box, label, fill, outline in node_specs: + draw.rounded_rectangle(node_box, radius=10, fill=fill, outline=outline, width=2) + centered_text(node_box, exact(label), node_size, fill=outline) + + centered_text(region(task_box, 0.03, 0.03, 0.97, 0.15), exact("Promptable Segmentation Task"), title_size) + centered_text(region(model_box, 0.03, 0.03, 0.97, 0.15), exact("Segment Anything Model"), title_size) + centered_text(region(data_box, 0.03, 0.03, 0.97, 0.15), exact("Data Engine"), title_size, fill=(177, 92, 33)) + else: + for index in range(len(centers) - 1): + draw_arrow_path([centers[index], centers[index + 1]]) path = Path(out_path) path.parent.mkdir(parents=True, exist_ok=True) image.save(path) - return {"summary": "Content-free layout blueprint rendered from selected template.", "path": str(path), "width": width, "height": height, "aspect_ratio_decimal": round(ratio, 6), "template_id": profile["template_id"], "contains_reference_text": False, "contains_reference_objects": False} + semantic_label_order = [] + if semantic_feedback: + semantic_label_order = ["input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M", "iterate"] + elif semantic_dense: + semantic_label_order = [ + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ] + return { + "summary": "Paper-semantic layout blueprint rendered from the normalized contract." if semantic_blueprint else "Content-free layout blueprint rendered from selected template.", + "path": str(path), + "width": width, + "height": height, + "aspect_ratio_decimal": round(ratio, 6), + "template_id": profile["template_id"], + "contains_reference_text": False, + "contains_reference_objects": False, + "contains_paper_semantic_labels": semantic_blueprint, + "semantic_labels": [exact(value) for value in semantic_label_order], + } diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 7d847f7..af46f5c 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -113,7 +113,12 @@ def run_paper_to_image( preferences["generation_ratio_policy"] = "nearest_native_image2_canvas; preserve_template_internal_geometry; no_semantic_crop" write_json(root / "preferences.json", preferences) write_json(root / "selected_template.json", {"summary": "Automatically or explicitly selected content-free architecture template.", **selected_template}) - blueprint_report = render_layout_blueprint(selected_template, root / "layout_blueprint.png", target_ratio=preferences["aspect_ratio"]) + blueprint_report = render_layout_blueprint( + selected_template, + root / "layout_blueprint.png", + target_ratio=preferences["aspect_ratio"], + figure_specification=plan.get("figure_specification"), + ) write_json(root / "layout_blueprint.json", blueprint_report) plan["style_plan"]["selected_template_id"] = selected_template.get("profile_id") diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index e618179..c39588e 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -252,6 +252,168 @@ def critic(path, template, prompt): self.assertIn("Remove unexpected visible label: Contrastive Loss", review["repair"]) self.assertIn("Contrastive Loss", review["remove"]) + def test_visible_label_whitelist_includes_declared_connector_labels(self): + plan = _plan() + plan["figure_specification"]["relations"][0]["label"] = "iterate" + + labels = required_labels(plan) + + self.assertIn("iterate", labels) + + def test_feedback_template_aligns_input_with_generation_and_stacks_refinement_below_initial_output(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + feedback = next(item for item in profiles if item["template_id"] == "feedback") + boxes = {item["id"]: item["bbox_percent"] for item in feedback["panels"]} + center = lambda box: (box["x"] + box["w"] / 2, box["y"] + box["h"] / 2) + + self.assertLess(abs(center(boxes["task_input"])[1] - center(boxes["generation"])[1]), 0.04) + self.assertLess(abs(center(boxes["initial_output"])[0] - center(boxes["refinement"])[0]), 0.04) + self.assertLess(abs(center(boxes["feedback"])[0] - center(boxes["refined_output"])[0]), 0.06) + + def test_feedback_prompt_forbids_known_wrong_edges_and_pins_iterate_target(self): + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "required_labels": ["input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M"], + "repeatable_labels": ["Model M"], + "inputs": [{"id": "input", "name": "input x"}], + "modules": [ + {"id": "generate", "name": "Generate"}, + {"id": "initial", "name": "Initial Output"}, + {"id": "feedback", "name": "FEEDBACK"}, + {"id": "self_feedback", "name": "Self-Feedback"}, + {"id": "refine", "name": "REFINE"}, + {"id": "model", "name": "Model M"}, + ], + "outputs": [{"id": "refined", "name": "Refined output"}], + "relations": [ + {"source": "input", "target": "generate", "type": "data_flow"}, + {"source": "generate", "target": "initial", "type": "data_flow"}, + {"source": "initial", "target": "feedback", "type": "evaluation"}, + {"source": "feedback", "target": "self_feedback", "type": "feedback"}, + {"source": "initial", "target": "refine", "type": "revision_input"}, + {"source": "self_feedback", "target": "refine", "type": "feedback"}, + {"source": "refine", "target": "refined", "type": "data_flow"}, + {"source": "refined", "target": "feedback", "type": "feedback_loop", "label": "iterate"}, + ], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("Initial Output must have exactly two distinct outgoing arrows", prompt) + self.assertIn("Refined output -> Generate", prompt) + self.assertIn("terminate only at FEEDBACK", prompt) + self.assertIn("Model M is a reusable badge", prompt) + + def test_feedback_blueprint_embeds_exact_semantic_labels(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + feedback = next(item for item in profiles if item["template_id"] == "feedback") + specification = { + "required_labels": ["input x", "Generate", "Initial Output", "FEEDBACK", "Self-Feedback", "REFINE", "Refined output", "Model M"], + "inputs": [{"id": "input", "name": "input x"}], + "modules": [ + {"id": "generate", "name": "Generate"}, + {"id": "initial", "name": "Initial Output"}, + {"id": "feedback", "name": "FEEDBACK"}, + {"id": "self_feedback", "name": "Self-Feedback"}, + {"id": "refine", "name": "REFINE"}, + {"id": "model", "name": "Model M"}, + ], + "outputs": [{"id": "refined", "name": "Refined output"}], + } + + report = render_layout_blueprint(feedback, Path(temp) / "feedback_semantic.png", "3:2", figure_specification=specification) + + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"][-1], "iterate") + self.assertTrue(Path(report["path"]).exists()) + + def test_generic_feedback_contract_does_not_receive_self_refine_blueprint_or_prompt_terms(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + feedback = next(item for item in profiles if item["template_id"] == "feedback") + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "inputs": [{"id": "sample", "name": "Sample"}], + "modules": [{"id": "measure", "name": "Measure"}, {"id": "update", "name": "Update"}], + "outputs": [{"id": "estimate", "name": "Estimate"}], + "relations": [ + {"source": "sample", "target": "measure", "type": "data_flow"}, + {"source": "measure", "target": "update", "type": "feedback"}, + {"source": "update", "target": "estimate", "type": "data_flow"}, + {"source": "estimate", "target": "measure", "type": "feedback_loop"}, + ], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + report = render_layout_blueprint(feedback, Path(temp) / "generic_feedback.png", "3:2", figure_specification=plan["figure_specification"]) + + self.assertNotIn("Model M is a reusable badge", prompt) + self.assertIn("Do not introduce Self-Refine-specific labels", prompt) + self.assertFalse(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"], []) + + def test_dense_multiframe_prompt_forbids_crossed_sam_paths(self): + plan = _plan() + labels = [ + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ] + plan["figure_specification"].update({ + "topology": "dense_multiframe", + "inputs": [{"id": "image", "name": "Image"}, {"id": "prompt", "name": "Prompt"}], + "modules": [{"id": f"module_{index}", "name": label} for index, label in enumerate(labels) if label not in {"Image", "Prompt", "Valid Segmentation Mask", "SA-1B"}], + "outputs": [{"id": "mask", "name": "Valid Segmentation Mask"}, {"id": "sa1b", "name": "SA-1B"}], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("Image must connect only to Image Encoder", prompt) + self.assertIn("Image Encoder -> Prompt Encoder", prompt) + self.assertIn("Fully Automatic -> Valid Segmentation Mask", prompt) + self.assertIn("container-to-container arrow", prompt) + + def test_generic_dense_multiframe_prompt_does_not_inject_sam_terms(self): + plan = _plan() + plan["figure_specification"].update({ + "topology": "dense_multiframe", + "inputs": [{"id": "sensor", "name": "Sensor Stream"}], + "modules": [{"id": "fusion", "name": "Fusion Core"}, {"id": "archive", "name": "Archive Builder"}], + "outputs": [{"id": "report", "name": "Report"}], + "relations": [{"source": "sensor", "target": "fusion", "type": "data_flow"}, {"source": "fusion", "target": "report", "type": "data_flow"}], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertNotIn("Image must connect only to Image Encoder", prompt) + self.assertIn("Do not introduce SAM-specific encoders", prompt) + + def test_dense_multiframe_blueprint_embeds_sam_semantic_topology(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + dense = next(item for item in profiles if item["template_id"] == "dense-multiframe") + labels = [ + "Promptable Segmentation Task", "Segment Anything Model", "Data Engine", "Image", "Prompt", + "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", + "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", + ] + specification = { + "required_labels": labels, + "inputs": [{"id": "image", "name": "Image"}, {"id": "prompt", "name": "Prompt"}], + "modules": [{"id": f"module_{index}", "name": label} for index, label in enumerate(labels) if label not in {"Image", "Prompt", "Valid Segmentation Mask", "SA-1B"}], + "outputs": [{"id": "mask", "name": "Valid Segmentation Mask"}, {"id": "sa1b", "name": "SA-1B"}], + } + + report = render_layout_blueprint(dense, Path(temp) / "dense_semantic.png", "3:2", figure_specification=specification) + + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertIn("Mask Decoder", report["semantic_labels"]) + self.assertEqual(report["semantic_labels"][-1], "SA-1B") + def test_complex_feedback_candidate_requires_focused_topology_pass(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -277,6 +439,37 @@ def test_complex_feedback_candidate_requires_focused_topology_pass(self): self.assertIn("Input -> Refine", review["hard_errors"]) self.assertIn("Connect Input to Refine", review["repair"]) + def test_topology_critic_prompt_accepts_shared_component_inside_source_container(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"].update({ + "topology": "feedback", + "repeatable_labels": ["Model M"], + "modules": [ + {"id": "feedback", "name": "FEEDBACK"}, + {"id": "model", "name": "Model M"}, + {"id": "self_feedback", "name": "Self-Feedback"}, + ], + "relations": [{"source": "feedback", "target": "self_feedback", "type": "feedback"}], + "terminology": {}, + }) + captured = {} + + def topology(_path, prompt): + captured["prompt"] = prompt + return {"summary": "Focused topology.", "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": [], "repair_regions": [], "score": 1.0, "passed": True} + + review_candidate(candidate, blueprint, plan, {"template_id": "feedback", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=_passing_critic, topology_adapter=topology) + + self.assertIn('"Model M"', captured["prompt"]) + self.assertIn("FEEDBACK container holding an allowed shared Model M badge", captured["prompt"]) + self.assertIn("count the container-to-target relation as present", captured["prompt"]) + def test_issue_free_ten_point_scale_review_is_normalized_by_findings(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 0d58b17..f7d3f68 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -2,7 +2,7 @@ import unittest from rfs.paper_to_image.planner import validate_plan_grounding -from rfs.paper_to_image.preparation import build_overlay_spec, expand_plan_evidence, normalize_figure_contract +from rfs.paper_to_image.preparation import build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract class PaperContractTests(unittest.TestCase): @@ -642,6 +642,63 @@ def test_feedback_contract_merges_formula_and_display_name_duplicates(self): self.assertTrue(all(item["source"] in endpoint_ids and item["target"] in endpoint_ids for item in spec["relations"])) self.assertEqual(len(spec["required_labels"]), len({_normalized.casefold() for _normalized in spec["required_labels"]})) + def test_feedback_contract_removes_parallel_model_path_and_keeps_one_eight_edge_loop(self): + caption = "Figure 1: Input x is used to Generate an Initial Output, obtain Self-Feedback from FEEDBACK, REFINE it into a Refined Output, and iterate." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "feedback", + "inputs": [{"id": "input", "name": "input x", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "model", "name": "Model M", "evidence_ids": ["E0001"]}, + {"id": "generate", "name": "Generate", "evidence_ids": ["E0001"]}, + {"id": "initial", "name": "Initial Output", "evidence_ids": ["E0001"]}, + {"id": "feedback", "name": "FEEDBACK", "evidence_ids": ["E0001"]}, + {"id": "self_feedback", "name": "Self-Feedback", "evidence_ids": ["E0001"]}, + {"id": "refine", "name": "REFINE", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "refined", "name": "Refined output", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "iteration", "name": "Iterative self-refinement", "evidence_ids": ["E0001"]}], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "model", "target": "initial", "type": "prediction", "evidence_ids": ["E0001"]}, + {"source": "feedback", "target": "model", "type": "conditioning", "evidence_ids": ["E0001"]}, + {"source": "model", "target": "refine", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "input", "target": "generate", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "generate", "target": "initial", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "initial", "target": "feedback", "type": "evaluation", "evidence_ids": ["E0001"]}, + {"source": "feedback", "target": "self_feedback", "type": "feedback", "evidence_ids": ["E0001"]}, + {"source": "initial", "target": "refine", "type": "revision_input", "evidence_ids": ["E0001"]}, + {"source": "self_feedback", "target": "refine", "type": "feedback", "evidence_ids": ["E0001"]}, + {"source": "refine", "target": "refined", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "refined", "target": "feedback", "type": "feedback_loop", "evidence_ids": ["E0001"]}, + ], + "must_show": [], + "terminology": {}, + "repeatable_labels": ["Model M"], + }, + "design_plan": {"summary": "Old plan", "reading_order": ["iteration"], "preserve": ["Iterative self-refinement"]}, + "layout_intent": {"summary": "Old layout"}, + "visual_metaphors": {"summary": "Old metaphors", "items": [{"module_id": "iteration", "metaphor": "Iterative self-refinement"}]}, + } + + spec = normalize_figure_contract(plan, parsed) + synchronize_plan_to_contract(plan) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertEqual(len(spec["relations"]), 8) + self.assertEqual(spec["innovations"], []) + self.assertFalse(any("model" in pair for pair in pairs)) + self.assertIn(("input", "generate"), pairs) + self.assertIn(("refined", "feedback"), pairs) + self.assertNotIn("Iterative self-refinement", spec["must_show"]) + self.assertEqual(plan["design_plan"]["preserve"], spec["required_labels"]) + def test_branch_contract_merges_head_output_aliases_and_removes_cross_branch_edges(self): caption = "Figure 1: An Input Image passes through a Backbone and RPN to RoIAlign, followed by parallel classification, bounding-box regression, and mask branches." parsed = { From 2f51ae608cdc092239428268e68e07f64372b52c Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 11:07:06 +0800 Subject: [PATCH 60/74] Add generic paper semantic blueprint compiler --- README.md | 13 +- docs/architecture.md | 4 +- docs/paper-to-image.md | 27 +- rfs/paper_to_image/contract_completion.py | 16 +- rfs/paper_to_image/planner.py | 5 + rfs/paper_to_image/preparation.py | 106 ++++++++ rfs/paper_to_image/semantic_blueprint.py | 297 ++++++++++++++++++++++ rfs/paper_to_image/templates.py | 94 ++++++- rfs/paper_to_image/workflow.py | 2 + tests/integration/test_paper_to_image.py | 81 +++++- tests/unit/test_paper_contract.py | 132 ++++++++++ tests/unit/test_semantic_blueprint.py | 166 ++++++++++++ 12 files changed, 928 insertions(+), 15 deletions(-) create mode 100644 rfs/paper_to_image/semantic_blueprint.py create mode 100644 tests/unit/test_semantic_blueprint.py diff --git a/README.md b/README.md index c917abd..4d784ba 100644 --- a/README.md +++ b/README.md @@ -128,10 +128,13 @@ figure rather than an editable PowerPoint file. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts positive references into content-free architecture templates, selects a template, renders `layout_blueprint.png`, and uses Image2 edit to create and review three -candidates. For normalized feedback-loop and SAM-style dense-multiframe -contracts, the blueprint also embeds exact paper labels and mandatory connector -geometry so Image2 edits a scientific source of truth instead of guessing node -mapping from empty boxes. It never invokes the PPTX compiler. +candidates. For evidence-grounded contracts with 2-16 visible nodes, a generic +semantic compiler now writes ranked node boxes, distinct input/output ports, +orthogonal paths, branch lanes, and outer feedback loops into +`layout_blueprint.json.semantic_plan`; the rendered blueprint embeds the exact +paper labels and connector geometry. Self-Refine feedback and SAM-style dense +multi-frame contracts retain stricter specialized layouts. It never invokes the +PPTX compiler. Offline engineering validation: @@ -172,7 +175,7 @@ rfs paper-to-image ` The main outputs are `paper_review.json`, `review_coverage_report.json`, `domain_profile.json`, `template_profiles/`, `selected_template.json`, -`layout_blueprint.png`, `figure_specification.json`, `image_prompt.txt`, +`layout_blueprint.png`, `layout_blueprint.json`, `figure_specification.json`, `image_prompt.txt`, `image2_request_manifest.json`, the general and focused-topology critic reports, the `candidates/` directory, and production-only `selected_image.png`. diff --git a/docs/architecture.md b/docs/architecture.md index ef82d44..aa19432 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -56,6 +56,7 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. |---|---| | Scientific entities and exact labels | `paper_review.json`, `figure_specification.json` | | Scientific relations and evidence | Paper semantic contract | +| Paper-to-image draft node and connector geometry | `layout_blueprint.json.semantic_plan` | | Layout, visual rhythm, palette, object style | Approved generated/user reference image | | Editable object geometry | `figure_program.json` | | PowerPoint rendering | `ppt_compiler.py` | @@ -75,7 +76,8 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. - Engineering previews are opt-in and never production-approved. - Critical text and relations remain editable PPT objects. - Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. -- Deterministic completion is graph-scoped: rich VLM contracts only gain overview-local or directly connected evidence-backed entities, while heuristic contracts grow from overview seeds or the largest connected method chain. +- Deterministic completion is graph-scoped: rich VLM contracts only gain overview-local or directly connected evidence-backed entities, while heuristic contracts grow from overview seeds or the largest connected method chain. Late appendix-only diagram labels, ungrounded innovations, and input shortcuts that bypass an explicit intermediate node are removed before planning. +- `semantic_blueprint.py` compiles 2-16 visible contract entities into normalized graph geometry. It ranks DAG layers, places source-only conditioning nodes immediately before their targets, assigns distinct fan-in/fan-out ports, routes skip-layer edges around intermediate nodes, and sends feedback loops through an outer lane. - Benchmark entity matching uses both label fidelity and relation consistency so relation labels and variable suffixes cannot steal mappings from real contract entities. - VLM correction is optional and bounded; deterministic reports remain available offline. - Historical sample-specific scripts stay under `experiments/legacy_reference_rebuilds/` and are not imported by production code. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 8fdde70..eb07be9 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -162,9 +162,24 @@ and requested ratio. The selected profile is rendered as `layout_blueprint.png` and is the only image supplied to Image2 edit for initial candidates. Reference-derived profiles remain -free of copied reference text and reference-specific objects. When the normalized -paper contract matches a strict built-in topology, the renderer may add exact -paper-semantic labels and connector geometry: +free of copied reference text and reference-specific objects. + +For supported normalized topologies (`linear`, `branch`, `multimodal`, +`feedback`, and `dense_multiframe`), contracts with 2-16 visible entities are +compiled into `layout_blueprint.json.semantic_plan`. The plan records exact +paper labels, normalized node boxes, graph ranks, distinct source/target ports, +relation types, connector labels, multi-point paths, and route styles. The +generic compiler: + +- puts independent inputs in one layer and separates their fan-in ports; +- keeps parallel branch outputs in one layer with separate fan-out ports; +- moves source-only conditioning nodes immediately before their target; +- routes skip-layer edges around intermediate nodes instead of through them; +- routes backward/feedback edges through an outer canvas lane; +- refuses to force more than 16 visible nodes into one raster guide. + +When the contract matches a stricter built-in topology, the renderer uses a +specialized semantic layout instead: - `feedback`: the forward generation chain, two independent inputs to refinement, the self-feedback node, and the return loop terminating at feedback; @@ -177,6 +192,12 @@ it does not copy paper figures or invent labels. It reduces topology drift by making Image2 preserve an explicit scientific source of truth instead of inferring node mapping from empty placeholder boxes. +Before semantic layout, contract completion prefers true early model-overview +figures over late scaling, timing, attention, or visualization figures. It also +removes isolated appendix-only diagram labels, innovations that cannot be +grounded to novelty evidence, duplicate aliases such as `MLP`/`MLP Head`, and +input shortcuts that bypass an explicit intermediate representation. + The general production review and focused topology review execute concurrently. Their default request timeout is 90 seconds with no automatic retry, preventing two independent judges from creating a multi-minute sequential stall. Configure diff --git a/rfs/paper_to_image/contract_completion.py b/rfs/paper_to_image/contract_completion.py index 817ade1..b5dfad3 100644 --- a/rfs/paper_to_image/contract_completion.py +++ b/rfs/paper_to_image/contract_completion.py @@ -59,7 +59,7 @@ class ConceptRule: ConceptRule("image_patches", "Image Patches", "modules", "intermediate", r"\bimage patches?\b|\bfixed-size patches\b", ("patches",), True), ConceptRule("linear_projection", "Linear Projection of Flattened Patches", "modules", "projection", r"\blinear projection(?: of flattened patches)?\b|\blinearly embed\b", ("linear projection",), True), ConceptRule("class_token", "Class Token", "modules", "conditioning", r"\bclass(?:ification)? token\b|\bclass embedding\b", ("classification token", "class embedding"), True), - ConceptRule("mlp_head", "MLP Head", "modules", "prediction head", r"\bmlp head\b|\bclassification head[^.]{0,100}\bmlp\b"), + ConceptRule("mlp_head", "MLP Head", "modules", "prediction head", r"\bmlp head\b|\bclassification head[^.]{0,100}\bmlp\b", ("mlp",)), ConceptRule("sampled_points", "Sampled 3D Points", "modules", "intermediate", r"\bsampl(?:e|ed|ing)\s+(?:the\s+)?(?:5d coordinates|3d points)\b|\b5d coordinates\b", ("5d coordinates", "spatial locations")), ConceptRule("prediction_ffn", "Feed Forward Network", "modules", "prediction head", r"\bfeed[ -]?forward network\b|\bprediction ffn\b|\bffn\b", ("ffn", "prediction feed-forward network")), ConceptRule("region_proposals", "Region Proposals", "modules", "proposal artifact", r"\bregion proposals?\b|\bregion proposal network\b|\brpn\b", ("proposals", "region proposal network (rpn)")), @@ -220,7 +220,7 @@ class ConceptRule: def _overview_candidates(parsed: dict[str, Any], limit: int = 2) -> list[dict[str, Any]]: positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|model|system|workflow|representation|encoder|decoder|directly predicts)\b", re.IGNORECASE) - negative = re.compile(r"\b(comparison|differences?|versus|performance|result|ablation|visualization|qualitative|attention|distribution|interface)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|differences?|versus|performance|result|ablation|visualization|qualitative|attention|distribution|interface|scaling|dimensions?|timings?|compute|memory)\b", re.IGNORECASE) ranked = [] for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): caption = str(figure.get("caption") or "") @@ -231,7 +231,17 @@ def _overview_candidates(parsed: dict[str, Any], limit: int = 2) -> list[dict[st score += 3 if 160 <= len(caption) <= 1600 else 0 score -= 14 if negative.search(caption) else 0 ranked.append((score, -index, figure)) - return [item for score, _, item in sorted(ranked, reverse=True) if score > 0][:limit] + ranked.sort(reverse=True) + if not ranked: + return [] + best_score = ranked[0][0] + # Keep a second, explicitly architectural figure when it is not a result or + # analysis figure. Early Figure 1/2 overviews still rank first, while the + # negative-caption penalty excludes late scaling, timing, attention, and + # visualization figures. The second page often contains the readable + # subsystem labels that a short overview caption omits. + minimum_score = max(8, best_score - 4) + return [item for score, _, item in ranked if score >= minimum_score][:limit] def _evidence_for_figure(parsed: dict[str, Any], figure: dict[str, Any]) -> dict[str, Any] | None: diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 3fb79a2..cd231cb 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -638,6 +638,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = metaphors = plan["visual_metaphors"] style = plan["style_plan"] template = selected_template or {} + semantic_plan = template.get("semantic_plan", {}) if isinstance(template.get("semantic_plan"), dict) else {} labels = collect_visible_labels(spec) repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] repeatable_normalized = {re.sub(r"[^\w\u4e00-\u9fff]+", "", value.casefold()) for value in repeatable_labels} @@ -679,6 +680,9 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Reference-derived content-free template contract: {json.dumps({key: template.get(key) for key in ['template_id', 'panels', 'connectors', 'visual_density', 'style', 'selection']}, ensure_ascii=False, indent=2)} +Paper-grounded semantic blueprint geometry: +{json.dumps(semantic_plan if semantic_plan.get('applied') else {"applied": False}, ensure_ascii=False, indent=2)} + Exact visible label whitelist (render these exactly and do not invent alternatives): {json.dumps(labels, ensure_ascii=False)} @@ -709,6 +713,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = - Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. - Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. - Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. +- When semantic blueprint geometry is supplied, preserve every node bounding box, layer assignment, connector path, and arrow direction; visual decoration may change but graph geometry may not. - Template panels are macro spatial regions, not an exact limit on the number of scientific content nodes inside those regions. - Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. - Render every mandatory visible label at least once; repeat only labels explicitly marked as evidence-supported shared components, and never replace an output label with an icon-only node. diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 8c0b8a4..5f8501c 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -184,6 +184,109 @@ def _ground_exact_visible_entities(spec: dict[str, Any], parsed: dict[str, Any]) return grounded +def _prune_out_of_scope_entities(spec: dict[str, Any], parsed: dict[str, Any], completion_report: dict[str, Any]) -> list[str]: + selected_ids = {str(value) for value in completion_report.get("selected_overview_evidence_ids", []) if str(value)} + if not selected_ids: + return [] + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + anchor_text = " ".join( + [ + str(spec.get("figure_goal") or ""), + str(spec.get("research_problem", {}).get("text") or "") if isinstance(spec.get("research_problem"), dict) else str(spec.get("research_problem") or ""), + str(spec.get("central_claim", {}).get("text") or "") if isinstance(spec.get("central_claim"), dict) else str(spec.get("central_claim") or ""), + *(str(evidence_by_id[value].get("text") or "") for value in selected_ids if value in evidence_by_id), + ] + ) + normalized_anchor = _normalized_label(anchor_text) + blocked_sections = ("appendix", "experiment", "result", "analysis", "visualization", "ablation", "attention map", "reference", "acknowledg") + removed_ids: set[str] = set() + removed_labels: list[str] = [] + for field in ("inputs", "modules", "outputs"): + kept = [] + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + evidence_ids = [str(value) for value in item.get("evidence_ids", []) if str(value)] + records = [evidence_by_id[value] for value in evidence_ids if value in evidence_by_id] + label = _item_label(item) + label_in_anchor = bool(_normalized_label(label) and _normalized_label(label) in normalized_anchor) + selected_grounded = any(value in selected_ids for value in evidence_ids) + only_blocked = bool(records) and all(any(term in str(record.get("section_hint") or "").casefold() for term in blocked_sections) for record in records) + weak_isolated_label = bool(records) and all(_normalized_label(str(record.get("text") or "")) == _normalized_label(label) for record in records) + if not selected_grounded and not label_in_anchor and only_blocked and weak_isolated_label: + removed_ids.add(str(item.get("id") or "")) + removed_labels.append(label) + continue + kept.append(item) + spec[field] = kept + if removed_ids: + spec["relations"] = [ + item for item in spec.get("relations", []) + if isinstance(item, dict) and str(item.get("source")) not in removed_ids and str(item.get("target")) not in removed_ids + ] + return removed_labels + + +def _remove_ungrounded_innovations(spec: dict[str, Any]) -> list[str]: + removed_ids: set[str] = set() + removed_labels: list[str] = [] + kept = [] + for item in spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []: + if isinstance(item, dict) and item.get("evidence_ids"): + kept.append(item) + continue + if isinstance(item, dict): + removed_ids.add(str(item.get("id") or "")) + removed_labels.append(_item_label(item)) + spec["innovations"] = kept + if removed_ids: + spec["relations"] = [ + item for item in spec.get("relations", []) + if isinstance(item, dict) and str(item.get("source")) not in removed_ids and str(item.get("target")) not in removed_ids + ] + return removed_labels + + +def _remove_input_shortcuts_through_intermediates(spec: dict[str, Any]) -> list[str]: + inputs = {str(item.get("id")) for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id")} + entities = { + str(item.get("id")): item + for field in ("modules", "outputs") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + outgoing: dict[str, list[dict[str, Any]]] = {} + for relation in relations: + outgoing.setdefault(str(relation.get("source") or ""), []).append(relation) + removed: list[str] = [] + remove_ids: set[int] = set() + flow_types = {"data_flow", "encoding", "prediction", "generation_input"} + for index, direct in enumerate(relations): + source = str(direct.get("source") or "") + target = str(direct.get("target") or "") + if source not in inputs or str(direct.get("type") or "data_flow") not in flow_types: + continue + for first in outgoing.get(source, []): + middle = str(first.get("target") or "") + if middle == target or middle not in entities: + continue + role_text = f"{entities[middle].get('role', '')} {_item_label(entities[middle])}".casefold() + if not any(term in role_text for term in ("intermediate", "artifact", "patch", "token sequence", "representation")): + continue + if any(str(second.get("target") or "") == target and str(second.get("type") or "data_flow") in flow_types for second in outgoing.get(middle, [])): + remove_ids.add(index) + removed.append(f"{source}->{target}") + break + if remove_ids: + spec["relations"] = [item for index, item in enumerate(relations) if index not in remove_ids] + return removed + + def _stable_id(prefix: str, value: str, index: int) -> str: slug = "_".join(part for part in re.sub(r"[^a-z0-9]+", " ", value.casefold()).split() if part)[:48] return f"{prefix}_{slug or index}" @@ -1059,6 +1162,9 @@ def choose_input_target(label: str) -> dict[str, Any] | None: if str(spec.get("topology") or "") not in allowed_topologies or inferred_topology in {"feedback", "multimodal"}: spec["topology"] = inferred_topology completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) + completion_report["removed_out_of_scope_entities"] = _prune_out_of_scope_entities(spec, parsed, completion_report) + completion_report["removed_ungrounded_innovations"] = _remove_ungrounded_innovations(spec) + completion_report["removed_input_shortcuts"] = _remove_input_shortcuts_through_intermediates(spec) _repair_cross_script_terminology(spec, parsed) completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) labels = [] diff --git a/rfs/paper_to_image/semantic_blueprint.py b/rfs/paper_to_image/semantic_blueprint.py new file mode 100644 index 0000000..3ec1b88 --- /dev/null +++ b/rfs/paper_to_image/semantic_blueprint.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +from collections import defaultdict, deque +from typing import Any + + +_SUPPORTED_TOPOLOGIES = {"linear", "branch", "multimodal", "feedback", "dense_multiframe"} +_FEEDBACK_TYPES = {"feedback_loop", "iteration", "iterate", "return_flow"} + + +def _label(item: dict[str, Any]) -> str: + return str(item.get("name") or item.get("label") or item.get("title") or "").strip() + + +def _normalized(value: Any) -> str: + return "".join(char for char in str(value or "").casefold() if char.isalnum()) + + +def _visible_entities(specification: dict[str, Any]) -> list[dict[str, Any]]: + required = { + _normalized(value) + for value in specification.get("required_labels", []) + if str(value).strip() + } + entities: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + seen_labels: set[str] = set() + for field in ("inputs", "modules", "outputs", "innovations"): + values = specification.get(field, []) + if not isinstance(values, list): + continue + for index, item in enumerate(values): + if not isinstance(item, dict): + continue + label = _label(item) + entity_id = str(item.get("id") or f"{field}_{index + 1}").strip() + normalized_label = _normalized(label) + if not label or not entity_id or entity_id in seen_ids or normalized_label in seen_labels: + continue + if required and normalized_label not in required: + continue + seen_ids.add(entity_id) + seen_labels.add(normalized_label) + entities.append({ + "id": entity_id, + "label": label, + "field": field, + "role": str(item.get("role") or "").strip(), + "order": len(entities), + }) + return entities + + +def _visible_relations(specification: dict[str, Any], node_ids: set[str]) -> list[dict[str, str]]: + relations: list[dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + values = specification.get("relations", []) + if not isinstance(values, list): + return relations + type_priority = { + "feedback_loop": 0, + "branch": 1, + "conditioning": 2, + "feature_flow": 3, + "data_flow": 4, + } + candidates: dict[tuple[str, str], list[dict[str, str]]] = defaultdict(list) + for item in values: + if not isinstance(item, dict): + continue + source = str(item.get("source") or "").strip() + target = str(item.get("target") or "").strip() + if not source or not target or source == target or source not in node_ids or target not in node_ids: + continue + candidates[(source, target)].append({ + "source": source, + "target": target, + "type": str(item.get("type") or "data_flow").strip(), + "label": str(item.get("label") or "").strip(), + }) + for pair, items in candidates.items(): + if pair in seen: + continue + selected = sorted(items, key=lambda item: type_priority.get(item["type"], 9))[0] + labels = [item["label"] for item in items if item["label"]] + if labels: + selected["label"] = labels[0] + relations.append(selected) + seen.add(pair) + return relations + + +def _rank_nodes(nodes: list[dict[str, Any]], relations: list[dict[str, str]]) -> dict[str, int]: + node_ids = [item["id"] for item in nodes] + incoming: dict[str, set[str]] = {node_id: set() for node_id in node_ids} + outgoing: dict[str, set[str]] = {node_id: set() for node_id in node_ids} + forward_relations = [ + item for item in relations + if item["type"] not in _FEEDBACK_TYPES + ] + for item in forward_relations: + incoming[item["target"]].add(item["source"]) + outgoing[item["source"]].add(item["target"]) + + indegree = {node_id: len(incoming[node_id]) for node_id in node_ids} + order_lookup = {item["id"]: item["order"] for item in nodes} + queue = deque(sorted((node_id for node_id in node_ids if indegree[node_id] == 0), key=order_lookup.get)) + ranks = {node_id: 0 for node_id in node_ids} + visited: set[str] = set() + while queue: + source = queue.popleft() + visited.add(source) + for target in sorted(outgoing[source], key=order_lookup.get): + ranks[target] = max(ranks[target], ranks[source] + 1) + indegree[target] -= 1 + if indegree[target] == 0: + queue.append(target) + + if len(visited) != len(node_ids): + for _ in range(len(node_ids)): + changed = False + for item in forward_relations: + candidate = min(len(node_ids) - 1, ranks[item["source"]] + 1) + if candidate > ranks[item["target"]]: + ranks[item["target"]] = candidate + changed = True + if not changed: + break + + input_ids = {item["id"] for item in nodes if item["field"] == "inputs"} + for node_id in input_ids: + if not incoming[node_id]: + ranks[node_id] = 0 + relations_by_source: dict[str, list[dict[str, str]]] = defaultdict(list) + for relation in forward_relations: + relations_by_source[relation["source"]].append(relation) + for node in nodes: + node_id = node["id"] + outgoing_relations = relations_by_source.get(node_id, []) + if incoming[node_id] or not outgoing_relations: + continue + if all(str(item.get("type") or "").casefold() == "conditioning" for item in outgoing_relations): + target_rank = min(ranks[item["target"]] for item in outgoing_relations) + ranks[node_id] = max(0, target_rank - 1) + return ranks + + +def _layout_nodes(nodes: list[dict[str, Any]], ranks: dict[str, int]) -> list[dict[str, Any]]: + layers: dict[int, list[dict[str, Any]]] = defaultdict(list) + for item in nodes: + layers[ranks[item["id"]]].append(item) + compact_ranks = {rank: index for index, rank in enumerate(sorted(layers))} + layer_count = max(1, len(compact_ranks)) + node_width = min(0.16, max(0.115, 0.72 / layer_count)) + x_start, x_end = 0.07, 0.93 + positioned: list[dict[str, Any]] = [] + for original_rank in sorted(layers): + layer_index = compact_ranks[original_rank] + items = sorted(layers[original_rank], key=lambda item: (item["field"] == "innovations", item["field"] == "outputs", item["order"])) + count = len(items) + center_x = (x_start + x_end) / 2 if layer_count == 1 else x_start + layer_index * (x_end - x_start) / (layer_count - 1) + node_height = min(0.16, max(0.075, 0.66 / max(count, 1))) + if count == 1: + centers_y = [0.50] + else: + y_start, y_end = 0.15, 0.85 + centers_y = [y_start + index * (y_end - y_start) / (count - 1) for index in range(count)] + for item, center_y in zip(items, centers_y): + x = max(0.015, min(0.985 - node_width, center_x - node_width / 2)) + positioned.append({ + **item, + "rank": layer_index, + "bbox_percent": { + "x": round(x, 6), + "y": round(max(0.06, center_y - node_height / 2), 6), + "w": round(node_width, 6), + "h": round(node_height, 6), + }, + }) + return positioned + + +def _connector_path( + source: dict[str, Any], + target: dict[str, Any], + relation_type: str, + source_port_fraction: float = 0.5, + target_port_fraction: float = 0.5, +) -> tuple[list[list[float]], str]: + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center = (source_box["x"] + source_box["w"] / 2, source_box["y"] + source_box["h"] / 2) + target_center = (target_box["x"] + target_box["w"] / 2, target_box["y"] + target_box["h"] / 2) + is_return = relation_type in _FEEDBACK_TYPES or target["rank"] <= source["rank"] + if is_return: + route_y = 0.95 if source_center[1] >= 0.42 or target_center[1] >= 0.42 else 0.05 + source_anchor_y = source_box["y"] + source_box["h"] if route_y > 0.5 else source_box["y"] + target_anchor_y = target_box["y"] + target_box["h"] if route_y > 0.5 else target_box["y"] + return ([ + [round(source_center[0], 6), round(source_anchor_y, 6)], + [round(source_center[0], 6), route_y], + [round(target_center[0], 6), route_y], + [round(target_center[0], 6), round(target_anchor_y, 6)], + ], "outer_feedback") + if source["rank"] == target["rank"]: + if source_center[1] <= target_center[1]: + start = [source_center[0], source_box["y"] + source_box["h"]] + end = [target_center[0], target_box["y"]] + else: + start = [source_center[0], source_box["y"]] + end = [target_center[0], target_box["y"] + target_box["h"]] + return ([[round(value, 6) for value in start], [round(value, 6) for value in end]], "vertical") + start = [source_box["x"] + source_box["w"], source_box["y"] + source_port_fraction * source_box["h"]] + end = [target_box["x"], target_box["y"] + target_port_fraction * target_box["h"]] + if target["rank"] - source["rank"] > 1: + lane_y = max(0.035, min(0.965, min(source_box["y"], target_box["y"]) - 0.045)) + if lane_y < 0.055: + lane_y = min(0.965, max(source_box["y"] + source_box["h"], target_box["y"] + target_box["h"]) + 0.045) + return ([ + [round(value, 6) for value in start], + [round(start[0] + 0.025, 6), round(start[1], 6)], + [round(start[0] + 0.025, 6), round(lane_y, 6)], + [round(end[0] - 0.025, 6), round(lane_y, 6)], + [round(end[0] - 0.025, 6), round(end[1], 6)], + [round(value, 6) for value in end], + ], "bypass_orthogonal") + if abs(start[1] - end[1]) < 0.025: + return ([[round(value, 6) for value in start], [round(value, 6) for value in end]], "straight") + lane_fraction = 0.20 + 0.60 * (0.65 * target_port_fraction + 0.35 * source_port_fraction) + mid_x = start[0] + (end[0] - start[0]) * lane_fraction + return ([ + [round(value, 6) for value in start], + [round(mid_x, 6), round(start[1], 6)], + [round(mid_x, 6), round(end[1], 6)], + [round(value, 6) for value in end], + ], "orthogonal") + + +def compile_semantic_blueprint( + specification: dict[str, Any] | None, + *, + max_nodes: int = 16, +) -> dict[str, Any]: + specification = specification if isinstance(specification, dict) else {} + topology = str(specification.get("topology") or "unknown") + nodes = _visible_entities(specification) + if topology not in _SUPPORTED_TOPOLOGIES: + return {"summary": "Generic semantic blueprint not applicable to this topology.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "unsupported_topology"} + if len(nodes) < 2: + return {"summary": "Generic semantic blueprint requires at least two visible entities.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "too_few_nodes"} + if len(nodes) > max(2, int(max_nodes)): + return {"summary": "Generic semantic blueprint skipped because the visible contract is too dense for one raster guide.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "too_many_nodes", "node_count": len(nodes)} + + node_ids = {item["id"] for item in nodes} + relations = _visible_relations(specification, node_ids) + if not relations: + return {"summary": "Generic semantic blueprint requires at least one declared relation.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "no_relations"} + + ranks = _rank_nodes(nodes, relations) + positioned = _layout_nodes(nodes, ranks) + by_id = {item["id"]: item for item in positioned} + incoming_groups: dict[str, list[dict[str, str]]] = defaultdict(list) + outgoing_groups: dict[str, list[dict[str, str]]] = defaultdict(list) + for relation in relations: + incoming_groups[relation["target"]].append(relation) + outgoing_groups[relation["source"]].append(relation) + connectors = [] + for index, relation in enumerate(relations, start=1): + incoming = incoming_groups[relation["target"]] + outgoing = outgoing_groups[relation["source"]] + target_index = incoming.index(relation) + source_index = outgoing.index(relation) + target_fraction = (target_index + 1) / (len(incoming) + 1) + source_fraction = (source_index + 1) / (len(outgoing) + 1) + path, route_style = _connector_path( + by_id[relation["source"]], + by_id[relation["target"]], + relation["type"], + source_port_fraction=source_fraction, + target_port_fraction=target_fraction, + ) + connectors.append({ + "id": f"semantic_connector_{index:02d}", + **relation, + "path_percent": path, + "route_style": route_style, + }) + return { + "summary": "Paper-grounded semantic graph compiled into normalized node and connector geometry.", + "applied": True, + "topology": topology, + "node_count": len(positioned), + "connector_count": len(connectors), + "nodes": positioned, + "connectors": connectors, + "source": "figure_specification", + } diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index ccc4012..a6af9d1 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -9,6 +9,7 @@ from ..utils import ensure_dir, write_json from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available +from .semantic_blueprint import compile_semantic_blueprint BUILTIN_TEMPLATES: dict[str, dict[str, Any]] = { @@ -371,7 +372,11 @@ def exact(canonical: str) -> str: ) } semantic_dense = profile.get("template_id") == "dense-multiframe" and dense_required.issubset(exact_labels) - semantic_blueprint = semantic_feedback or semantic_dense + generic_semantic_plan = compile_semantic_blueprint(figure_specification) + semantic_generic = bool(generic_semantic_plan.get("applied")) and not semantic_feedback and not semantic_dense + semantic_blueprint = semantic_feedback or semantic_dense or semantic_generic + if semantic_generic: + panels = [] def font(size: int, bold: bool = False): candidates = ["arialbd.ttf", "DejaVuSans-Bold.ttf"] if bold else ["arial.ttf", "DejaVuSans.ttf"] @@ -397,6 +402,41 @@ def centered_text(box: tuple[int, int, int, int], value: str, size: int, fill=(4 y = box[1] + ((box[3] - box[1]) - text_height) / 2 - bounds[1] draw.text((x, y), value, font=selected_font, fill=fill) + def centered_wrapped_text(box: tuple[int, int, int, int], value: str, size: int, fill=(42, 72, 91), bold: bool = True) -> None: + max_width = max(20, box[2] - box[0] - 14) + max_height = max(20, box[3] - box[1] - 10) + tokens = value.split() if " " in value.strip() else list(value.strip()) + selected_font = font(size, bold=bold) + selected_lines = [value] + for fitted_size in range(max(10, size), 9, -1): + selected_font = font(fitted_size, bold=bold) + lines: list[str] = [] + current = "" + separator = " " if " " in value.strip() else "" + for token in tokens: + candidate = token if not current else f"{current}{separator}{token}" + if draw.textlength(candidate, font=selected_font) <= max_width or not current: + current = candidate + else: + lines.append(current) + current = token + if current: + lines.append(current) + if len(lines) > 3: + continue + text_value = "\n".join(lines) + bounds = draw.multiline_textbbox((0, 0), text_value, font=selected_font, spacing=3, align="center") + if bounds[2] - bounds[0] <= max_width and bounds[3] - bounds[1] <= max_height: + selected_lines = lines + break + text_value = "\n".join(selected_lines) + bounds = draw.multiline_textbbox((0, 0), text_value, font=selected_font, spacing=3, align="center") + text_width = bounds[2] - bounds[0] + text_height = bounds[3] - bounds[1] + x = box[0] + ((box[2] - box[0]) - text_width) / 2 + y = box[1] + ((box[3] - box[1]) - text_height) / 2 - bounds[1] + draw.multiline_text((x, y), text_value, font=selected_font, fill=fill, spacing=3, align="center") + centers = [] panel_boxes = [] for index, panel in enumerate(panels): @@ -555,6 +595,54 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) centered_text(region(task_box, 0.03, 0.03, 0.97, 0.15), exact("Promptable Segmentation Task"), title_size) centered_text(region(model_box, 0.03, 0.03, 0.97, 0.15), exact("Segment Anything Model"), title_size) centered_text(region(data_box, 0.03, 0.03, 0.97, 0.15), exact("Data Engine"), title_size, fill=(177, 92, 33)) + elif semantic_generic: + node_boxes: dict[str, tuple[int, int, int, int]] = {} + node_styles: dict[str, tuple[tuple[int, int, int], tuple[int, int, int]]] = {} + for node in generic_semantic_plan.get("nodes", []): + bbox = node["bbox_percent"] + node_box = ( + round(bbox["x"] * width), + round(bbox["y"] * height), + round((bbox["x"] + bbox["w"]) * width), + round((bbox["y"] + bbox["h"]) * height), + ) + field = str(node.get("field") or "modules") + role = str(node.get("role") or "").casefold() + if field == "inputs": + fill, outline = (237, 245, 252), (42, 99, 161) + elif field == "outputs": + fill, outline = (242, 248, 243), (66, 126, 80) + elif field == "innovations": + fill, outline = (252, 244, 235), (177, 92, 33) + elif "shared" in role or "joint" in role or "fusion" in role: + fill, outline = (244, 242, 251), (98, 84, 147) + else: + fill, outline = (237, 249, 247), (31, 117, 126) + node_boxes[node["id"]] = node_box + node_styles[node["id"]] = (fill, outline) + draw.rounded_rectangle(node_box, radius=max(8, round(height * 0.014)), fill=fill, outline=outline, width=3) + + relation_colors = { + "feedback_loop": (177, 92, 33), + "branch": (98, 84, 147), + "conditioning": (31, 117, 126), + "feature_flow": (42, 99, 161), + } + for connector in generic_semantic_plan.get("connectors", []): + points = [(round(point[0] * width), round(point[1] * height)) for point in connector.get("path_percent", [])] + color = relation_colors.get(str(connector.get("type") or ""), (80, 111, 134)) + draw_arrow_path(points, color) + label = str(connector.get("label") or "").strip() + if label and len(points) >= 2: + middle = points[len(points) // 2] + label_box = (middle[0] - 60, middle[1] - 18, middle[0] + 60, middle[1] + 18) + draw.rounded_rectangle(label_box, radius=7, fill=(250, 250, 249), outline=color, width=1) + centered_wrapped_text(label_box, label, max(11, round(height * 0.014)), fill=color, bold=False) + + for node in generic_semantic_plan.get("nodes", []): + node_box = node_boxes[node["id"]] + _, outline = node_styles[node["id"]] + centered_wrapped_text(node_box, str(node.get("label") or node["id"]), max(13, round(height * 0.019)), fill=outline) else: for index in range(len(centers) - 1): draw_arrow_path([centers[index], centers[index + 1]]) @@ -570,6 +658,9 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) "Image Encoder", "Prompt Encoder", "Mask Decoder", "Valid Segmentation Mask", "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", ] + elif semantic_generic: + semantic_label_order = [str(item.get("label") or "") for item in generic_semantic_plan.get("nodes", [])] + semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("connectors", []) if str(item.get("label") or "").strip()) return { "summary": "Paper-semantic layout blueprint rendered from the normalized contract." if semantic_blueprint else "Content-free layout blueprint rendered from selected template.", "path": str(path), @@ -581,4 +672,5 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) "contains_reference_objects": False, "contains_paper_semantic_labels": semantic_blueprint, "semantic_labels": [exact(value) for value in semantic_label_order], + "semantic_plan": generic_semantic_plan if semantic_generic else None, } diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index af46f5c..4f428ff 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -120,6 +120,8 @@ def run_paper_to_image( figure_specification=plan.get("figure_specification"), ) write_json(root / "layout_blueprint.json", blueprint_report) + if isinstance(blueprint_report.get("semantic_plan"), dict) and blueprint_report["semantic_plan"].get("applied"): + selected_template["semantic_plan"] = blueprint_report["semantic_plan"] plan["style_plan"]["selected_template_id"] = selected_template.get("profile_id") plan["style_plan"]["template_style"] = selected_template.get("style", {}) diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index c39588e..5ff2f42 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -353,8 +353,10 @@ def test_generic_feedback_contract_does_not_receive_self_refine_blueprint_or_pro self.assertNotIn("Model M is a reusable badge", prompt) self.assertIn("Do not introduce Self-Refine-specific labels", prompt) - self.assertFalse(report["contains_paper_semantic_labels"]) - self.assertEqual(report["semantic_labels"], []) + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"], ["Sample", "Measure", "Update", "Estimate"]) + self.assertTrue(report["semantic_plan"]["applied"]) + self.assertNotIn("Self-Feedback", report["semantic_labels"]) def test_dense_multiframe_prompt_forbids_crossed_sam_paths(self): plan = _plan() @@ -392,6 +394,81 @@ def test_generic_dense_multiframe_prompt_does_not_inject_sam_terms(self): self.assertNotIn("Image must connect only to Image Encoder", prompt) self.assertIn("Do not introduce SAM-specific encoders", prompt) + def test_generic_linear_blueprint_renders_contract_nodes_and_connectors(self): + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + linear = next(item for item in profiles if item["template_id"] == "linear") + specification = { + "topology": "linear", + "required_labels": ["Source", "Parser", "Reasoner", "Answer"], + "inputs": [{"id": "source", "name": "Source"}], + "modules": [{"id": "parser", "name": "Parser"}, {"id": "reasoner", "name": "Reasoner"}], + "outputs": [{"id": "answer", "name": "Answer"}], + "relations": [ + {"source": "source", "target": "parser", "type": "data_flow"}, + {"source": "parser", "target": "reasoner", "type": "data_flow"}, + {"source": "reasoner", "target": "answer", "type": "data_flow"}, + ], + } + + report = render_layout_blueprint(linear, Path(temp) / "generic_linear.png", "3:2", figure_specification=specification) + + self.assertTrue(report["contains_paper_semantic_labels"]) + self.assertEqual(report["semantic_labels"], ["Source", "Parser", "Reasoner", "Answer"]) + self.assertTrue(report["semantic_plan"]["applied"]) + self.assertEqual(report["semantic_plan"]["connector_count"], 3) + self.assertTrue(Path(report["path"]).exists()) + prompt = compile_image_prompt(_plan(), {"aspect_ratio": "3:2", "language": "English"}, selected_template={**linear, "semantic_plan": report["semantic_plan"]}) + self.assertIn("Paper-grounded semantic blueprint geometry", prompt) + self.assertIn('"route_style": "straight"', prompt) + self.assertIn("preserve every node bounding box", prompt) + + def test_generic_branch_multimodal_and_feedback_blueprints_render_without_paper_specific_rules(self): + cases = [ + ( + "branch", + { + "topology": "branch", + "required_labels": ["Input", "Shared Trunk", "Head A", "Head B"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "trunk", "name": "Shared Trunk"}], + "outputs": [{"id": "a", "name": "Head A"}, {"id": "b", "name": "Head B"}], + "relations": [{"source": "input", "target": "trunk", "type": "data_flow"}, {"source": "trunk", "target": "a", "type": "branch"}, {"source": "trunk", "target": "b", "type": "branch"}], + }, + ), + ( + "multimodal", + { + "topology": "multimodal", + "required_labels": ["Vision", "Language", "Fusion", "Decision"], + "inputs": [{"id": "vision", "name": "Vision"}, {"id": "language", "name": "Language"}], + "modules": [{"id": "fusion", "name": "Fusion", "role": "shared representation"}], + "outputs": [{"id": "decision", "name": "Decision"}], + "relations": [{"source": "vision", "target": "fusion", "type": "encoding"}, {"source": "language", "target": "fusion", "type": "encoding"}, {"source": "fusion", "target": "decision", "type": "data_flow"}], + }, + ), + ( + "feedback", + { + "topology": "feedback", + "required_labels": ["Observation", "Estimator", "Correction", "State", "repeat"], + "inputs": [{"id": "observation", "name": "Observation"}], + "modules": [{"id": "estimator", "name": "Estimator"}, {"id": "correction", "name": "Correction"}], + "outputs": [{"id": "state", "name": "State"}], + "relations": [{"source": "observation", "target": "estimator", "type": "data_flow"}, {"source": "estimator", "target": "correction", "type": "data_flow"}, {"source": "correction", "target": "state", "type": "data_flow"}, {"source": "state", "target": "estimator", "type": "feedback_loop", "label": "repeat"}], + }, + ), + ] + with tempfile.TemporaryDirectory() as temp: + profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") + by_id = {item["template_id"]: item for item in profiles} + for template_id, specification in cases: + with self.subTest(template_id=template_id): + report = render_layout_blueprint(by_id[template_id], Path(temp) / f"{template_id}.png", "3:2", figure_specification=specification) + self.assertTrue(report["semantic_plan"]["applied"]) + self.assertEqual(set(report["semantic_labels"]), set(specification["required_labels"])) + self.assertTrue(Path(report["path"]).exists()) + def test_dense_multiframe_blueprint_embeds_sam_semantic_topology(self): with tempfile.TemporaryDirectory() as temp: profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index f7d3f68..c521bd9 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -3,9 +3,141 @@ from rfs.paper_to_image.planner import validate_plan_grounding from rfs.paper_to_image.preparation import build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract +from rfs.paper_to_image.contract_completion import _overview_candidates class PaperContractTests(unittest.TestCase): + def test_overview_selection_rejects_late_scaling_figure_when_figure_one_is_a_true_model_overview(self): + parsed = { + "document_index": { + "figures": [ + {"page": 3, "caption": "Figure 1: Model overview. Images become patches and enter a Transformer encoder."}, + {"page": 16, "caption": "Figure 8: Scaling different model dimensions of the Vision Transformer."}, + ] + } + } + + candidates = _overview_candidates(parsed) + + self.assertEqual([item["page"] for item in candidates], [3]) + + def test_contract_prunes_appendix_only_diagram_label_and_ungrounded_innovation(self): + overview = "Figure 1: Model overview. We split an input image into image patches, linearly embed them, add position embeddings and a class token, and feed them to a Transformer Encoder for classification." + parsed = { + "page_count": 18, + "document_index": { + "figures": [ + {"page": 3, "caption": overview}, + {"page": 16, "caption": "Figure 8: Scaling different model dimensions of the Vision Transformer."}, + ] + }, + "evidence": [ + {"id": "E_OVERVIEW", "page": 3, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E_DEPTH", "page": 16, "kind": "paragraph", "text": "Depth", "section_hint": "APPENDIX", "confidence": 1.0}, + ], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "figure_goal": "Show the Vision Transformer model overview.", + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "Images are processed as patch sequences.", "evidence_ids": ["E_OVERVIEW"]}, + "inputs": [ + {"id": "image", "name": "Input Image", "evidence_ids": ["E_OVERVIEW"]}, + {"id": "depth", "name": "Depth", "evidence_ids": ["E_DEPTH"]}, + ], + "modules": [{"id": "encoder", "name": "Transformer Encoder", "evidence_ids": ["E_OVERVIEW"]}], + "outputs": [{"id": "classification", "name": "Classification", "evidence_ids": ["E_OVERVIEW"]}], + "innovations": [{"id": "innovation_class_token", "name": "Learnable Class Token", "evidence_ids": []}], + "relations": [ + {"source": "image", "target": "encoder", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "depth", "target": "encoder", "type": "data_flow", "evidence_ids": ["E_DEPTH"]}, + {"source": "encoder", "target": "classification", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertNotIn("Depth", [item["name"] for item in spec["inputs"]]) + self.assertEqual(spec["innovations"], []) + self.assertFalse(any(item["source"] == "depth" or item["target"] == "depth" for item in spec["relations"])) + self.assertIn("Depth", plan["contract_completion_report"]["removed_out_of_scope_entities"]) + self.assertIn("Learnable Class Token", plan["contract_completion_report"]["removed_ungrounded_innovations"]) + + def test_vit_mlp_is_upgraded_to_one_mlp_head_instead_of_duplicating_the_prediction_path(self): + overview = "Figure 1: Model overview. The Transformer encoder representation is passed through an MLP head for classification." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": overview}]}, + "evidence": [{"id": "E_OVERVIEW", "page": 2, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "figure_goal": "Show image classification.", + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [], + "modules": [ + {"id": "encoder", "name": "Transformer Encoder", "evidence_ids": ["E_OVERVIEW"]}, + {"id": "mlp", "name": "MLP", "role": "module", "evidence_ids": ["E_OVERVIEW"]}, + ], + "outputs": [{"id": "classification", "name": "Classification", "evidence_ids": ["E_OVERVIEW"]}], + "innovations": [], + "relations": [ + {"source": "encoder", "target": "mlp", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "mlp", "target": "classification", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + labels = [item["name"] for item in spec["modules"]] + + self.assertEqual(labels.count("MLP Head"), 1) + self.assertNotIn("MLP", labels) + self.assertEqual(len([item for item in spec["relations"] if item["target"] == "classification"]), 1) + + def test_explicit_input_intermediate_path_removes_redundant_input_shortcut(self): + overview = "Figure 1: Input Image is split into Image Patches, followed by Linear Projection and a Transformer Encoder." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 2, "caption": overview}]}, + "evidence": [{"id": "E_OVERVIEW", "page": 2, "kind": "caption", "text": overview, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "figure_goal": "Show the patch pipeline.", + "research_problem": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "central_claim": {"text": "unknown", "evidence_ids": [], "status": "unknown"}, + "inputs": [{"id": "image", "name": "Input Image", "evidence_ids": ["E_OVERVIEW"]}], + "modules": [ + {"id": "patches", "name": "Image Patches", "role": "intermediate", "evidence_ids": ["E_OVERVIEW"]}, + {"id": "projection", "name": "Linear Projection", "evidence_ids": ["E_OVERVIEW"]}, + ], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "image", "target": "projection", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "image", "target": "patches", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + {"source": "patches", "target": "projection", "type": "data_flow", "evidence_ids": ["E_OVERVIEW"]}, + ], + "terminology": {}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertNotIn(("image", "projection"), pairs) + self.assertIn(("image", "patches"), pairs) + self.assertIn(("patches", "projection"), pairs) + self.assertIn("image->projection", plan["contract_completion_report"]["removed_input_shortcuts"]) + def _parsed(self): return { "evidence": [ diff --git a/tests/unit/test_semantic_blueprint.py b/tests/unit/test_semantic_blueprint.py new file mode 100644 index 0000000..e346108 --- /dev/null +++ b/tests/unit/test_semantic_blueprint.py @@ -0,0 +1,166 @@ +import unittest + +from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint + + +class SemanticBlueprintTests(unittest.TestCase): + def test_linear_contract_uses_graph_ranks_and_keeps_parallel_outputs_in_one_layer(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Embed", "Encoder", "Output A", "Output B"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "embed", "name": "Embed"}, {"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "a", "name": "Output A"}, {"id": "b", "name": "Output B"}], + "relations": [ + {"source": "input", "target": "embed", "type": "data_flow"}, + {"source": "embed", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "a", "type": "data_flow"}, + {"source": "encoder", "target": "b", "type": "data_flow"}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertTrue(report["applied"]) + self.assertEqual(nodes["input"]["rank"], 0) + self.assertLess(nodes["embed"]["rank"], nodes["encoder"]["rank"]) + self.assertEqual(nodes["a"]["rank"], nodes["b"]["rank"]) + self.assertNotEqual(nodes["a"]["bbox_percent"]["y"], nodes["b"]["bbox_percent"]["y"]) + output_edges = [item for item in report["connectors"] if item["source"] == "encoder"] + self.assertNotEqual(output_edges[0]["path_percent"][0][1], output_edges[1]["path_percent"][0][1]) + + def test_multimodal_inputs_share_one_layer_and_converge(self): + specification = { + "topology": "multimodal", + "required_labels": ["Image", "Text", "Audio", "Encoders", "Joint Space", "Alignment"], + "inputs": [{"id": "image", "name": "Image"}, {"id": "text", "name": "Text"}, {"id": "audio", "name": "Audio"}], + "modules": [{"id": "encoders", "name": "Encoders"}, {"id": "joint", "name": "Joint Space", "role": "shared representation"}], + "outputs": [{"id": "alignment", "name": "Alignment"}], + "relations": [ + {"source": "image", "target": "encoders", "type": "encoding"}, + {"source": "text", "target": "encoders", "type": "encoding"}, + {"source": "audio", "target": "encoders", "type": "encoding"}, + {"source": "encoders", "target": "joint", "type": "alignment"}, + {"source": "joint", "target": "alignment", "type": "enables"}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertTrue(report["applied"]) + self.assertEqual({nodes[node_id]["rank"] for node_id in ("image", "text", "audio")}, {0}) + self.assertEqual(len({nodes[node_id]["bbox_percent"]["x"] for node_id in ("image", "text", "audio")}), 1) + self.assertGreater(nodes["joint"]["rank"], nodes["encoders"]["rank"]) + input_edges = [item for item in report["connectors"] if item["target"] == "encoders"] + self.assertEqual(len({item["path_percent"][-1][1] for item in input_edges}), 3) + self.assertEqual(len({item["path_percent"][1][0] for item in input_edges}), 3) + + def test_feedback_loop_is_routed_outside_the_forward_graph(self): + specification = { + "topology": "feedback", + "required_labels": ["Sample", "Measure", "Update", "Estimate", "repeat"], + "inputs": [{"id": "sample", "name": "Sample"}], + "modules": [{"id": "measure", "name": "Measure"}, {"id": "update", "name": "Update"}], + "outputs": [{"id": "estimate", "name": "Estimate"}], + "relations": [ + {"source": "sample", "target": "measure", "type": "data_flow"}, + {"source": "measure", "target": "update", "type": "data_flow"}, + {"source": "update", "target": "estimate", "type": "data_flow"}, + {"source": "estimate", "target": "measure", "type": "feedback_loop", "label": "repeat"}, + ], + } + + report = compile_semantic_blueprint(specification) + loop = next(item for item in report["connectors"] if item["type"] == "feedback_loop") + + self.assertTrue(report["applied"]) + self.assertEqual(loop["route_style"], "outer_feedback") + self.assertIn(loop["path_percent"][1][1], {0.05, 0.95}) + self.assertEqual(loop["label"], "repeat") + + def test_source_only_conditioning_nodes_are_placed_immediately_before_their_target(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Patches", "Projection", "Class Token", "Position Embedding", "Encoder"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [ + {"id": "patches", "name": "Patches"}, + {"id": "projection", "name": "Projection"}, + {"id": "class", "name": "Class Token", "role": "conditioning"}, + {"id": "position", "name": "Position Embedding", "role": "conditioning"}, + {"id": "encoder", "name": "Encoder"}, + ], + "relations": [ + {"source": "input", "target": "patches", "type": "data_flow"}, + {"source": "patches", "target": "projection", "type": "data_flow"}, + {"source": "projection", "target": "encoder", "type": "data_flow"}, + {"source": "class", "target": "encoder", "type": "conditioning"}, + {"source": "position", "target": "encoder", "type": "conditioning"}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertEqual(nodes["class"]["rank"], nodes["encoder"]["rank"] - 1) + self.assertEqual(nodes["position"]["rank"], nodes["encoder"]["rank"] - 1) + self.assertEqual(nodes["projection"]["rank"], nodes["class"]["rank"]) + + def test_required_label_whitelist_excludes_out_of_scope_entities(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Core", "Output"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "core", "name": "Core"}, {"id": "detail", "name": "Out-of-scope Detail"}], + "outputs": [{"id": "output", "name": "Output"}], + "innovations": [{"id": "core_innovation", "name": "Core"}], + "relations": [ + {"source": "input", "target": "core", "type": "data_flow"}, + {"source": "core", "target": "output", "type": "data_flow"}, + {"source": "core", "target": "detail", "type": "data_flow"}, + ], + } + + report = compile_semantic_blueprint(specification) + + self.assertEqual({item["label"] for item in report["nodes"]}, {"Input", "Core", "Output"}) + self.assertFalse(any(item["target"] == "detail" for item in report["connectors"])) + + def test_skip_layer_relation_uses_bypass_lane_instead_of_crossing_middle_node(self): + specification = { + "topology": "branch", + "required_labels": ["Input", "Middle", "Merge"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "middle", "name": "Middle"}, {"id": "merge", "name": "Merge"}], + "relations": [ + {"source": "input", "target": "middle", "type": "data_flow"}, + {"source": "middle", "target": "merge", "type": "data_flow"}, + {"source": "input", "target": "merge", "type": "conditioning"}, + ], + } + + report = compile_semantic_blueprint(specification) + bypass = next(item for item in report["connectors"] if item["source"] == "input" and item["target"] == "merge") + + self.assertEqual(bypass["route_style"], "bypass_orthogonal") + self.assertGreaterEqual(len(bypass["path_percent"]), 6) + + def test_contract_larger_than_blueprint_limit_is_not_forced_into_one_canvas(self): + modules = [{"id": f"node_{index}", "name": f"Node {index}"} for index in range(18)] + specification = { + "topology": "linear", + "required_labels": [item["name"] for item in modules], + "modules": modules, + "relations": [{"source": f"node_{index}", "target": f"node_{index + 1}", "type": "data_flow"} for index in range(17)], + } + + report = compile_semantic_blueprint(specification) + + self.assertFalse(report["applied"]) + self.assertEqual(report["reason"], "too_many_nodes") + + +if __name__ == "__main__": + unittest.main() From ac3e9a138a8ccc2303fa316dd5fec5d6bc6a98b3 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 11:25:48 +0800 Subject: [PATCH 61/74] Compile paper contracts to editable PowerPoint --- README.md | 4 +- docs/architecture.md | 2 + docs/paper-to-image.md | 18 +++ rfs/cli.py | 2 + rfs/composition/pptx.py | 39 ++++++ rfs/paper_to_image/__init__.py | 3 +- rfs/paper_to_image/editable_ppt.py | 190 +++++++++++++++++++++++++++++ rfs/paper_to_image/preparation.py | 90 ++++++++++++++ tests/unit/test_paper_contract.py | 32 +++++ tests/unit/test_semantic_ppt.py | 71 +++++++++++ 10 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 rfs/paper_to_image/editable_ppt.py create mode 100644 tests/unit/test_semantic_ppt.py diff --git a/README.md b/README.md index 4d784ba..fab65f4 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ This writes paper-review and image-generation artifacts under `paper_to_image/`, For offline engineering validation only, add `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview`. This does not create a production-approved result. -## Paper To Image Without PPTX +## Fast Paper Contract And Optional Editable PPTX For the fast “paper to evidence-grounded framework prompt” path, without image generation: @@ -107,6 +107,8 @@ rfs fast-framework-prompt ` This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. +Add `--editable-ppt` to compile the same evidence-backed semantic contract directly into `editable_composition.pptx` without image generation. Every visible module is a native PowerPoint shape with editable text, every relation is a native connector, and `figure_program.json`, `semantic_ppt_report.json`, and `composition_quality_report.json` remain available for auditing. This direct route is intended for fast structural drafts; generated icons or richer visual assets can be added later without changing the scientific graph. + Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean headings plus localized Figure/Table captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, OCR render DPI, and spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently at 84 DPI and filters short vertical margin artifacts. The base installation includes `wordninja` for conservative English spacing recovery, while requiring multiple English anchors so non-English Latin words are not split accidentally. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/docs/architecture.md b/docs/architecture.md index aa19432..6b200b4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,6 +57,7 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. | Scientific entities and exact labels | `paper_review.json`, `figure_specification.json` | | Scientific relations and evidence | Paper semantic contract | | Paper-to-image draft node and connector geometry | `layout_blueprint.json.semantic_plan` | +| Direct paper-to-PPT node and connector geometry | `figure_program.json` compiled from the same semantic plan | | Layout, visual rhythm, palette, object style | Approved generated/user reference image | | Editable object geometry | `figure_program.json` | | PowerPoint rendering | `ppt_compiler.py` | @@ -64,6 +65,7 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. ## Stable workflow boundaries +- `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. - `paper-to-image`: produces reviewed visual candidates and paper-grounding artifacts; it never creates PPTX. - `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. - `paper-to-editable`: runs both stages and requires a production-approved image unless engineering preview use is explicitly enabled. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index eb07be9..b5946ab 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -192,6 +192,24 @@ it does not copy paper figures or invent labels. It reduces topology drift by making Image2 preserve an explicit scientific source of truth instead of inferring node mapping from empty placeholder boxes. +The same generic semantic plan can now bypass raster generation entirely: + +```powershell +rfs fast-framework-prompt ` + --paper paper.pdf ` + --out output\fast_result ` + --editable-ppt +``` + +This writes `figure_program.json` and `editable_composition.pptx`. Nodes are +native PowerPoint shapes with exact editable labels; multi-segment routes and +feedback loops are native connectors created behind the nodes. The compiler +also writes `semantic_ppt_report.json` and records every editable node and +connector in `composition_quality_report.json`. A narrow evidence repair can +attach an otherwise isolated dataset/source input to the same downstream +module as sibling modalities only when the cited caption explicitly states the +shared provenance (for example, modalities "from" that dataset). + Before semantic layout, contract completion prefers true early model-overview figures over late scaling, timing, attention, or visualization figures. It also removes isolated appendix-only diagram labels, innovations that cannot be diff --git a/rfs/cli.py b/rfs/cli.py index 3f0bd86..f85f624 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -163,6 +163,7 @@ def build_parser() -> argparse.ArgumentParser: fast_prompt.add_argument("--language", default="English") fast_prompt.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") fast_prompt.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + fast_prompt.add_argument("--editable-ppt", action="store_true", help="Also compile the semantic contract into native editable PowerPoint shapes, text, and connectors.") fast_prompt.add_argument("--json", action="store_true", help="Emit JSON.") make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") @@ -413,6 +414,7 @@ def main(argv: list[str] | None = None) -> int: aspect_ratio=args.aspect_ratio, language=args.language, domain_profile=args.domain_profile, + editable_ppt=args.editable_ppt, ) elif args.command == "make-framework": result = make_framework( diff --git a/rfs/composition/pptx.py b/rfs/composition/pptx.py index cc3eabb..77a1d94 100644 --- a/rfs/composition/pptx.py +++ b/rfs/composition/pptx.py @@ -230,6 +230,7 @@ def _align_from_text_item(item: dict): def _object_map(program: dict) -> dict[str, dict]: objects = {panel["id"]: panel for panel in program.get("panels", [])} objects.update({slot["id"]: slot for slot in program.get("slots", [])}) + objects.update({node["id"]: node for node in program.get("semantic_nodes", []) if isinstance(node, dict) and node.get("id")}) return objects @@ -307,6 +308,7 @@ def _draw_program_arrows(slide, program: dict, width_in: float, height_in: float if dashed: halo.line.dash_style = MSO_LINE_DASH_STYLE.DASH connector = slide.shapes.add_connector(connector_type, Inches(x1), Inches(y1), Inches(x2), Inches(y2)) + connector.name = f"RFS Connector {arrow.get('id') or 'unnamed'} {idx + 1}" connector.line.color.rgb = _rgb(arrow_color) connector.line.width = Pt(line_width) _apply_line_cap(connector, str(arrow.get("line_cap") or "round")) @@ -421,6 +423,42 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: rendered_arrows = _draw_program_arrows(slide, program, width_in, height_in) + # Paper semantic nodes are rendered after connectors so every edge stays + # behind the editable node shape and its exact paper label. + rendered_semantic_nodes = [] + for node in sorted(program.get("semantic_nodes", []), key=lambda item: int(item.get("z_index", 30))): + if not isinstance(node, dict) or not isinstance(node.get("bbox_percent"), dict): + continue + x, y, w, h = pct_to_inches(node["bbox_percent"], width_in, height_in) + shape_kind = str(node.get("shape_kind") or "rounded_rect").casefold() + shape_type = MSO_SHAPE.RECTANGLE if shape_kind == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE + shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) + shape.name = f"RFS Semantic Node {node.get('id') or 'unnamed'}" + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(node.get("fill_color") or "#F5FAFC")) + shape.line.color.rgb = _rgb(str(node.get("stroke_color") or "#1F6F8B")) + shape.line.width = Pt(float(node.get("stroke_width_pt") or 1.6)) + shape.shadow.inherit = False + _set_text( + shape, + str(node.get("label") or node.get("name") or node.get("id") or ""), + font_size=float(node.get("font_size_pt") or 16), + bold=bool(node.get("bold", True)), + color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), + font_family=str(node.get("font_family") or "Arial"), + ) + rendered_semantic_nodes.append({ + "node_id": node.get("id"), + "label": node.get("label") or node.get("name"), + "field": node.get("field"), + "role": node.get("role"), + "bbox_percent": node.get("bbox_percent"), + "shape_kind": shape_kind, + "editable_in": "pptx", + "render_policy": "ppt_shape_with_editable_text", + "status": "ok", + }) + # Slot image layer and editable captions. composition_items = [] caption_queue = [] @@ -585,6 +623,7 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: "slots": composition_items, "cards": rendered_cards, "arrows": rendered_arrows, + "semantic_nodes": rendered_semantic_nodes, "text": rendered_text_items, }) return pptx_path diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index b3d42a8..ec3bc28 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,7 +1,8 @@ """Paper analysis and paper-to-image workflows.""" from .inspection import inspect_paper +from .editable_ppt import build_semantic_figure_program, compile_semantic_ppt from .preparation import prepare_paper_figure_contract, run_fast_framework_prompt from .workflow import run_paper_to_image -__all__ = ["inspect_paper", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] +__all__ = ["build_semantic_figure_program", "compile_semantic_ppt", "inspect_paper", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] diff --git a/rfs/paper_to_image/editable_ppt.py b/rfs/paper_to_image/editable_ppt.py new file mode 100644 index 0000000..58b1c30 --- /dev/null +++ b/rfs/paper_to_image/editable_ppt.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..composition.pptx import compile_ppt +from ..utils import write_json +from .semantic_blueprint import compile_semantic_blueprint + + +_NODE_STYLES = { + "inputs": {"fill_color": "#EDF5FC", "stroke_color": "#2A63A1", "text_color": "#244B72"}, + "modules": {"fill_color": "#EDF9F7", "stroke_color": "#1F756F", "text_color": "#195B57"}, + "outputs": {"fill_color": "#F2F8F3", "stroke_color": "#427E50", "text_color": "#315F3B"}, + "innovations": {"fill_color": "#FCF4EB", "stroke_color": "#B15C21", "text_color": "#8C481A"}, +} + +_RELATION_STYLES = { + "feedback_loop": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "iteration": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "iterate": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "return_flow": {"stroke_color": "#B15C21", "line_pattern": "dash", "stroke_width_pt": 2.0}, + "branch": {"stroke_color": "#62529A", "stroke_width_pt": 1.8}, + "conditioning": {"stroke_color": "#1F756F", "line_pattern": "dash", "stroke_width_pt": 1.7}, + "feature_flow": {"stroke_color": "#2A63A1", "stroke_width_pt": 1.8}, + "data_flow": {"stroke_color": "#506F81", "stroke_width_pt": 1.7}, +} + + +def _ratio_value(value: str | float | int | None) -> float: + if isinstance(value, (float, int)): + return max(0.5, min(3.0, float(value))) + text = str(value or "16:9").strip().lower().replace("x", ":") + try: + if ":" in text: + left, right = text.split(":", 1) + return max(0.5, min(3.0, float(left) / max(float(right), 0.001))) + return max(0.5, min(3.0, float(text))) + except (TypeError, ValueError, ZeroDivisionError): + return 16 / 9 + + +def _font_size(label: str, bbox: dict[str, Any]) -> float: + width = float(bbox.get("w") or 0.12) + if len(label) >= 30 or width < 0.12: + return 12.0 + if len(label) >= 20: + return 14.0 + return 16.0 + + +def _relation_label_bbox(path: list[list[float]]) -> dict[str, float] | None: + if len(path) < 2: + return None + point = path[len(path) // 2] + x = max(0.02, min(0.84, float(point[0]) - 0.08)) + y = max(0.02, min(0.93, float(point[1]) - 0.022)) + return {"x": round(x, 6), "y": round(y, 6), "w": 0.16, "h": 0.044} + + +def build_semantic_figure_program( + specification: dict[str, Any], + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, +) -> dict[str, Any]: + semantic_plan = compile_semantic_blueprint(specification) + if not semantic_plan.get("applied"): + raise ValueError(f"Cannot compile editable PPT semantic blueprint: {semantic_plan.get('reason') or 'not_applicable'}") + + ratio = _ratio_value(aspect_ratio) + height_in = 7.5 + width_in = round(height_in * ratio, 3) + semantic_nodes = [] + for node in semantic_plan["nodes"]: + style = dict(_NODE_STYLES.get(str(node.get("field") or "modules"), _NODE_STYLES["modules"])) + role = str(node.get("role") or "").casefold() + if any(term in role for term in ("shared", "joint", "fusion")): + style = {"fill_color": "#F4F1FA", "stroke_color": "#62529A", "text_color": "#4B3F78"} + label = str(node.get("label") or node["id"]) + semantic_nodes.append({ + "id": node["id"], + "label": label, + "field": node.get("field"), + "role": node.get("role"), + "bbox_percent": node["bbox_percent"], + "shape_kind": "rounded_rect", + "stroke_width_pt": 1.6, + "font_size_pt": _font_size(label, node["bbox_percent"]), + "bold": True, + "editable_in": "pptx", + "z_index": 30, + **style, + }) + + arrows = [] + labels = [] + for connector in semantic_plan["connectors"]: + relation_type = str(connector.get("type") or "data_flow") + style = dict(_RELATION_STYLES.get(relation_type, _RELATION_STYLES["data_flow"])) + arrows.append({ + "id": connector["id"], + "source": connector["source"], + "target": connector["target"], + "source_id": connector["source"], + "target_id": connector["target"], + "type": relation_type, + "semantic_role": relation_type, + "control_kind": "dashed_loop" if style.get("line_pattern") == "dash" else "elbow_connector", + "route_style": connector.get("route_style"), + "path_percent": connector.get("path_percent", []), + "line_cap": "round", + "arrowhead_size": "sm", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + **style, + }) + relation_label = str(connector.get("label") or "").strip() + label_bbox = _relation_label_bbox(connector.get("path_percent", [])) if relation_label else None + if relation_label and label_bbox: + labels.append({ + "id": f"label_{connector['id']}", + "text": relation_label, + "target_id": connector["id"], + "role": "relation_label", + "bbox_percent": label_bbox, + "font_size_pt": 10, + "bold": False, + "color_hex": style["stroke_color"], + "editable_in": "pptx", + }) + + program: dict[str, Any] = { + "summary": "Editable PowerPoint figure compiled directly from the paper semantic contract.", + "canvas": {"width_in": width_in, "height_in": height_in, "ratio": round(ratio, 6), "background": "#FFFFFF"}, + "style": { + "palette": ["#2A63A1", "#1F756F", "#62529A", "#B15C21", "#427E50"], + "arrow_weight_pt": 1.7, + "font_family": "Arial", + }, + "paper_brief": {"title_guess": title}, + "panels": [], + "cards": [], + "slots": [], + "assets": [], + "semantic_nodes": semantic_nodes, + "arrows": arrows, + "labels": labels, + "groups": [], + "export_targets": [{"type": "pptx", "path": "editable_composition.pptx", "role": "main_editable_source"}], + "semantic_plan": semantic_plan, + } + if title and show_title: + program["title_block"] = { + "title": title, + "subtitle": "Editable paper framework", + "bbox_percent": {"x": 0.04, "y": 0.015, "w": 0.92, "h": 0.08}, + "title_font_size": 20, + "subtitle_font_size": 9, + } + return program + + +def compile_semantic_ppt( + specification: dict[str, Any], + out: str | Path, + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, +) -> dict[str, Any]: + root = Path(out) + root.mkdir(parents=True, exist_ok=True) + program = build_semantic_figure_program(specification, aspect_ratio=aspect_ratio, title=title, show_title=show_title) + write_json(root / "figure_program.json", program) + pptx_path = compile_ppt(program, root) + report = { + "summary": "Paper semantic contract compiled to native editable PowerPoint shapes, text, and connectors.", + "ok": pptx_path.exists(), + "pptx": str(pptx_path), + "figure_program": str(root / "figure_program.json"), + "node_count": len(program["semantic_nodes"]), + "connector_count": len(program["arrows"]), + "relation_label_count": len(program["labels"]), + "topology": program["semantic_plan"].get("topology"), + "editable_layers": ["native_shapes", "native_text", "native_connectors"], + } + write_json(root / "semantic_ppt_report.json", report) + return report diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 5f8501c..7f7d8a9 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -963,6 +963,69 @@ def _clear_unverbatim_relation_labels(spec: dict[str, Any], parsed: dict[str, An relation["label"] = "" return removed + +def _repair_isolated_input_relations(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + """Attach a grounded source input to a shared downstream module. + + Planners sometimes extract a dataset/source label as an input but omit its + edge while correctly connecting sibling modalities from the same caption. + This repair is intentionally narrow: the isolated label must appear in the + same cited evidence as an existing input-to-module edge, and the evidence + must explicitly describe data provenance (for example, "images from X"). + """ + inputs = [item for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id")] + module_ids = {str(item.get("id")) for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")} + relations = spec.get("relations", []) if isinstance(spec.get("relations"), list) else [] + connected = { + str(endpoint) + for relation in relations if isinstance(relation, dict) + for endpoint in (relation.get("source"), relation.get("target")) + if endpoint + } + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + input_ids = {str(item.get("id")) for item in inputs} + sibling_edges = [ + relation for relation in relations + if isinstance(relation, dict) + and str(relation.get("source")) in input_ids + and str(relation.get("target")) in module_ids + ] + repaired: list[str] = [] + for item in inputs: + source_id = str(item.get("id")) + if source_id in connected: + continue + label = _item_label(item).strip() + if not label: + continue + item_evidence = {str(value) for value in item.get("evidence_ids", []) if value} + for relation in sibling_edges: + shared_ids = item_evidence & {str(value) for value in relation.get("evidence_ids", []) if value} + if not shared_ids: + continue + cited_text = " ".join(evidence_by_id.get(value, "") for value in shared_ids) + label_pattern = re.escape(label).replace(r"\ ", r"\s+") + explicit_source = bool(re.search(rf"\b(?:from|using|uses?|sourced?\s+from)\b[^.;:]{{0,120}}{label_pattern}", cited_text, re.IGNORECASE)) + if not explicit_source: + continue + target_id = str(relation.get("target")) + relations.append({ + "source": source_id, + "target": target_id, + "type": "data_flow", + "label": "", + "evidence_ids": sorted(shared_ids), + }) + connected.add(source_id) + repaired.append(f"{source_id}->{target_id}") + break + spec["relations"] = relations + return repaired + def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} @@ -1004,6 +1067,7 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d completion_report["removed_duplicate_relations"] = endpoint_report["removed_duplicates"] completion_report["removed_unresolved_relations"] = endpoint_report["removed_unresolved"] completion_report["removed_self_relations"] = endpoint_report["removed_self_relations"] + completion_report["repaired_isolated_inputs"] = _repair_isolated_input_relations(spec, parsed) if endpoint_report["repaired"]: secondary_completion = augment_contract_from_evidence(spec, parsed) for key in ("added_entities", "upgraded_entities", "adopted_entities", "added_relations", "repaired_relations", "grounded_entities"): @@ -1694,9 +1758,35 @@ def prepare_paper_figure_contract( def run_fast_framework_prompt(**kwargs: Any) -> dict[str, Any]: + editable_ppt = bool(kwargs.pop("editable_ppt", False)) if not kwargs.get("planner_model"): kwargs["planner_model"] = os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" kwargs["fast_mode"] = True prepared = prepare_paper_figure_contract(**kwargs) + if editable_ppt and prepared.get("ok"): + from .editable_ppt import compile_semantic_ppt + + ppt_started = time.monotonic() + paper_summary = prepared.get("plan", {}).get("paper_summary", {}) + title = str(paper_summary.get("title") or paper_summary.get("title_guess") or "").strip() or None + ppt_result = compile_semantic_ppt( + prepared["plan"]["figure_specification"], + prepared["root"], + aspect_ratio=prepared.get("preferences", {}).get("aspect_ratio") or "16:9", + title=title, + ) + prepared.update({ + "pptx_generated": bool(ppt_result.get("ok")), + "pptx": ppt_result.get("pptx"), + "semantic_ppt": ppt_result, + }) + ppt_seconds = round(time.monotonic() - ppt_started, 3) + prepared["ppt_compile_seconds"] = ppt_seconds + prepared["elapsed_seconds"] = round(float(prepared.get("elapsed_seconds") or 0.0) + ppt_seconds, 3) + if isinstance(prepared.get("stage_timings"), dict): + prepared["stage_timings"]["ppt_compile_seconds"] = ppt_seconds + prepared["stage_timings"]["total_seconds"] = prepared["elapsed_seconds"] + prepared["artifacts"] = [*prepared.get("artifacts", []), "figure_program.json", "editable_composition.pptx", "semantic_ppt_report.json", "composition_quality_report.json"] + write_json(Path(prepared["root"]) / "run_report.json", {key: value for key, value in prepared.items() if key not in {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"}}) internal = {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"} return {key: value for key, value in prepared.items() if key not in internal} diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index c521bd9..a0bdbf2 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -1370,6 +1370,38 @@ def test_generic_completion_uses_selected_architecture_page_context_and_diagram_ self.assertNotIn("Text Encoder", ids) self.assertNotIn("Contrastive Learning", ids) + def test_isolated_dataset_input_is_connected_when_caption_explicitly_supplies_sibling_modalities(self): + caption = "Figure 1: Stage one constructs RETFound by means of SSL, using CFP and OCT from MEH-MIDAS and public datasets." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "linear", + "inputs": [ + {"id": "cfp", "name": "CFP", "evidence_ids": ["E0001"]}, + {"id": "oct", "name": "OCT", "evidence_ids": ["E0001"]}, + {"id": "meh", "name": "MEH-MIDAS", "evidence_ids": ["E0001"]}, + ], + "modules": [{"id": "ssl", "name": "SSL", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [ + {"source": "cfp", "target": "ssl", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "oct", "target": "ssl", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + }, + } + + spec = normalize_figure_contract(plan, parsed) + pairs = {(item["source"], item["target"]) for item in spec["relations"]} + + self.assertIn(("meh", "ssl"), pairs) + self.assertIn("meh->ssl", plan["contract_completion_report"]["repaired_isolated_inputs"]) + def test_generic_completion_recovers_retrieval_conditioned_generation(self): caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." result_caption = "Figure 2: Posterior for each generated token with five retrieved documents." diff --git a/tests/unit/test_semantic_ppt.py b/tests/unit/test_semantic_ppt.py new file mode 100644 index 0000000..1d8c5b0 --- /dev/null +++ b/tests/unit/test_semantic_ppt.py @@ -0,0 +1,71 @@ +import json +import tempfile +import unittest +import zipfile +from pathlib import Path + +from rfs.paper_to_image.editable_ppt import build_semantic_figure_program, compile_semantic_ppt + + +class SemanticPptTests(unittest.TestCase): + def test_compiles_contract_to_native_editable_shapes_text_and_connectors(self): + specification = { + "topology": "linear", + "required_labels": ["Input Image", "Patch Encoder", "Transformer", "Prediction"], + "inputs": [{"id": "image", "name": "Input Image"}], + "modules": [ + {"id": "patch", "name": "Patch Encoder"}, + {"id": "transformer", "name": "Transformer"}, + ], + "outputs": [{"id": "prediction", "name": "Prediction"}], + "relations": [ + {"source": "image", "target": "patch", "type": "data_flow"}, + {"source": "patch", "target": "transformer", "type": "feature_flow"}, + {"source": "transformer", "target": "prediction", "type": "data_flow"}, + ], + } + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + report = compile_semantic_ppt(specification, root, aspect_ratio="16:9") + + self.assertTrue(report["ok"]) + self.assertEqual(report["node_count"], 4) + self.assertEqual(report["connector_count"], 3) + self.assertEqual(report["editable_layers"], ["native_shapes", "native_text", "native_connectors"]) + with zipfile.ZipFile(report["pptx"]) as archive: + slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") + for label in specification["required_labels"]: + self.assertIn(label, slide_xml) + self.assertGreaterEqual(slide_xml.count(""), 3) + self.assertGreaterEqual(slide_xml.count("RFS Semantic Node"), 4) + self.assertGreaterEqual(slide_xml.count("RFS Connector"), 3) + + quality = json.loads((root / "composition_quality_report.json").read_text(encoding="utf-8")) + self.assertEqual(len(quality["semantic_nodes"]), 4) + self.assertTrue(all(item["editable_in"] == "pptx" for item in quality["semantic_nodes"])) + + def test_feedback_contract_uses_external_dashed_editable_loop(self): + specification = { + "topology": "feedback", + "required_labels": ["Sample", "Measure", "Update", "Estimate"], + "inputs": [{"id": "sample", "name": "Sample"}], + "modules": [{"id": "measure", "name": "Measure"}, {"id": "update", "name": "Update"}], + "outputs": [{"id": "estimate", "name": "Estimate"}], + "relations": [ + {"source": "sample", "target": "measure", "type": "data_flow"}, + {"source": "measure", "target": "update", "type": "data_flow"}, + {"source": "update", "target": "estimate", "type": "data_flow"}, + {"source": "estimate", "target": "measure", "type": "feedback_loop", "label": "revise"}, + ], + } + + program = build_semantic_figure_program(specification, aspect_ratio="3:2") + loop = next(item for item in program["arrows"] if item["type"] == "feedback_loop") + self.assertEqual(loop["line_pattern"], "dash") + self.assertEqual(loop["route_style"], "outer_feedback") + self.assertEqual(program["labels"][0]["text"], "revise") + + +if __name__ == "__main__": + unittest.main() From b614f68f7def2b565534a6907ef17b96a67ac401 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Fri, 24 Jul 2026 16:24:08 +0800 Subject: [PATCH 62/74] Strengthen paper role and visual enrichment review --- README.md | 4 + docs/architecture.md | 6 + docs/paper-to-image.md | 24 +- rfs/paper_to_image/critics.py | 67 +++- rfs/paper_to_image/editable_ppt.py | 4 +- rfs/paper_to_image/generator.py | 11 +- rfs/paper_to_image/planner.py | 123 ++++++- rfs/paper_to_image/preparation.py | 391 ++++++++++++++++++++--- rfs/paper_to_image/templates.py | 4 +- rfs/paper_to_image/workflow.py | 2 + tests/integration/test_paper_to_image.py | 146 ++++++++- tests/unit/test_paper_contract.py | 95 +++++- 12 files changed, 774 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index fab65f4..9b3486f 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,10 @@ paper labels and connector geometry. Self-Refine feedback and SAM-style dense multi-frame contracts retain stricter specialized layouts. It never invokes the PPTX compiler. +The contract normalizer preserves semantic roles instead of checking labels alone. Explicit provenance clauses such as `CFP and OCT from MEH-MIDAS and public datasets` compile to `data source -> modality -> method`; multi-source phrases collapse into one evidence-backed source group; explicit internal/external evaluation outputs are restored; generic labels such as `foundation model` are replaced by an explicitly named paper model when the evidence provides the alias; and named techniques such as `masked autoencoder` become required innovations. Paper-contract caching is independent of canvas ratio and display-language preferences, so changing 16:9 to 3:2 cannot select a different scientific contract. + +Production Image-2 review rejects a neat but empty text-only flowchart. The critic checks entity-role mismatches, focused provenance topology, mechanism visualization, visual information density, and publication polish. A deterministic blueprint-enrichment gate requires substantive visual content beyond gradients and shadows; configure its default `0.08` changed-pixel threshold with `RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO`. Generation prompts use a compact scientific contract so evidence, geometry, roles, and visual requirements remain explicit without repeating the full planning documents. + Offline engineering validation: ```powershell diff --git a/docs/architecture.md b/docs/architecture.md index 6b200b4..2d0f204 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -56,6 +56,7 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. |---|---| | Scientific entities and exact labels | `paper_review.json`, `figure_specification.json` | | Scientific relations and evidence | Paper semantic contract | +| Entity roles, provenance, and evaluation boundaries | Normalized evidence-backed contract completion | | Paper-to-image draft node and connector geometry | `layout_blueprint.json.semantic_plan` | | Direct paper-to-PPT node and connector geometry | `figure_program.json` compiled from the same semantic plan | | Layout, visual rhythm, palette, object style | Approved generated/user reference image | @@ -63,6 +64,11 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. | PowerPoint rendering | `ppt_compiler.py` | | Production eligibility | Candidate, semantic, visual, and PPTX validation reports | +The scientific contract cache is keyed by paper hash, planner model, compiler +version, and scientific domain profile. Canvas ratio and display-language +preferences are intentionally excluded because they are rendering concerns and +must not produce competing scientific interpretations of the same paper. + ## Stable workflow boundaries - `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index b5946ab..aa3d3ab 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -192,6 +192,23 @@ it does not copy paper figures or invent labels. It reduces topology drift by making Image2 preserve an explicit scientific source of truth instead of inferring node mapping from empty placeholder boxes. +Entity roles are part of that source of truth. Data sources, modalities, +processing modules, innovations, and outputs are reviewed separately even when +all their words are present. Explicit provenance text is normalized into a +source-to-modality-to-method chain, and multiple named sources may be collapsed +into one evidence-backed source group to avoid redundant nodes and crossed +edges. Explicit internal/external evaluation boundaries and named paper methods +or techniques are repaired deterministically when overview evidence states +them. + +Image-2 candidates must enrich the semantic blueprint rather than merely +restyling its boxes. Candidate review records `blueprint_enrichment_ratio` and +`blueprint_mean_abs_difference`; production defaults require at least `0.08` +changed-pixel enrichment. The VLM aesthetic review separately scores visual +information density, mechanism visualization, and publication polish. Sparse +text-only nodes therefore fail even when labels, arrows, spacing, and colors +are otherwise correct. + The same generic semantic plan can now bypass raster generation entirely: ```powershell @@ -205,10 +222,9 @@ This writes `figure_program.json` and `editable_composition.pptx`. Nodes are native PowerPoint shapes with exact editable labels; multi-segment routes and feedback loops are native connectors created behind the nodes. The compiler also writes `semantic_ppt_report.json` and records every editable node and -connector in `composition_quality_report.json`. A narrow evidence repair can -attach an otherwise isolated dataset/source input to the same downstream -module as sibling modalities only when the cited caption explicitly states the -shared provenance (for example, modalities "from" that dataset). +connector in `composition_quality_report.json`. The same provenance contract is +used here, so data sources feed their declared modalities instead of appearing +as peer sensor inputs. Before semantic layout, contract completion prefers true early model-overview figures over late scaling, timing, attention, or visualization figures. It also diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 37cc61c..62708f0 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -7,11 +7,11 @@ from pathlib import Path from typing import Any, Callable -from PIL import Image +from PIL import Image, ImageChops, ImageStat from ..reference_text_extractor import run_easyocr, run_paddle_ocr, run_rapidocr from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available -from .planner import collect_visible_labels, collect_visual_relations +from .planner import collect_visible_labels, collect_visual_relations, collect_visual_roles def _normalize_text(value: str) -> str: @@ -71,7 +71,10 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: Mandatory visual connector checklist: {json.dumps(collect_visual_relations(plan.get('figure_specification', {})), ensure_ascii=False, indent=2)} -Judge the visible flow against the mandatory visual connector checklist. Relations involving an evidence-supported repeatable shared component may be represented by repeated badges rather than extra implementation-level arrows. A shortcut that bypasses a checklist module is a scientific error. +Mandatory entity role and containment checklist: +{json.dumps(collect_visual_roles(plan.get('figure_specification', {})), ensure_ascii=False, indent=2)} + +Judge the visible flow against both checklists. Relations involving an evidence-supported repeatable shared component may be represented by repeated badges rather than extra implementation-level arrows. A shortcut that bypasses a checklist module is a scientific error. Merely showing every word is insufficient: a dataset/source shown as a peer modality, an output shown as an internal module, or a modality shown as a processing stage is a hard scientific role mismatch. Selected template: {json.dumps({key: template.get(key) for key in ['template_id', 'panels', 'connectors', 'visual_density', 'style']}, ensure_ascii=False, indent=2)} @@ -82,9 +85,9 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: {{ "summary": "Production candidate review.", "ocr": {{"detected_labels": [], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 0.0, "passed": false}}, - "scientific": {{"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "innovation_visible": true, "score": 0.0, "passed": false}}, + "scientific": {{"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": true, "score": 0.0, "passed": false}}, "template": {{"macro_panel_match": 0.0, "reading_order_match": 0.0, "connector_rhythm_match": 0.0, "visual_density_match": 0.0, "copied_reference_content": [], "score": 0.0, "passed": false}}, - "aesthetic": {{"hierarchy": 0.0, "balance": 0.0, "whitespace": 0.0, "color": 0.0, "icon_consistency": 0.0, "readability": 0.0, "score": 0.0, "passed": false}}, + "aesthetic": {{"hierarchy": 0.0, "balance": 0.0, "whitespace": 0.0, "color": 0.0, "icon_consistency": 0.0, "readability": 0.0, "visual_information_density": 0.0, "mechanism_visualization": 0.0, "publication_polish": 0.0, "score": 0.0, "passed": false}}, "preserve": [], "repair": [], "remove": [], @@ -94,7 +97,14 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: "overall_score": 0.0 }} -Hard failures: any missing/misspelled critical label, any duplicate label not explicitly allowed above, copied reference term, missing core module, reversed relation, invented mechanism, or template score below 0.72. Aesthetic quality cannot compensate for scientific or OCR failure. +Hard failures: any missing/misspelled critical label, any duplicate label not explicitly allowed above, copied reference term, missing core module, reversed relation, invented mechanism, semantic role/containment mismatch, or template score below 0.72. Aesthetic quality cannot compensate for scientific or OCR failure. + +Aesthetic scoring rules: +- A neat text-only flowchart is not a complete Image-2 scientific figure. Empty rounded rectangles containing only labels must score below 0.70 for mechanism_visualization and visual_information_density. +- Inspect whether inputs, core methods, innovations, and outputs contain paper-grounded visual cues, examples, mechanism details, or icons rather than unused interior space. +- icon_consistency cannot receive full credit when no icons or scientific visual metaphors are present. +- visual_density_match must reflect actual information inside nodes, not merely the number and alignment of boxes. +- Reserve scores above 0.90 for figures that are both structurally correct and visually explanatory at publication quality. """.strip() if adapter: return adapter(path, blueprint, prompt) @@ -134,6 +144,9 @@ def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Cal Mandatory directed connectors: {json.dumps(relations, ensure_ascii=False, indent=2)} +Mandatory entity roles: +{json.dumps(collect_visual_roles(specification), ensure_ascii=False, indent=2)} + Evidence-supported repeatable shared-component labels: {json.dumps(repeatable_labels, ensure_ascii=False)} @@ -141,6 +154,7 @@ def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Cal - Verify every source, target, and arrowhead direction from visible geometry. - A line that joins the outgoing side of a target or a downstream junction does not count as entering that target. - A shortcut that bypasses an intermediate module is an invented relation. +- Verify that source, modality, module, and output layers preserve their declared roles. A source -> modality -> method chain must not be flattened into peer inputs. - For refinement loops, Initial Output and Self-Feedback must both enter Refine or its explicit input-side shared-model node before Refined Output. - Refined Output must return to Feedback to close the loop. - Do not penalize repeated labels explicitly allowed by the scientific contract. @@ -185,6 +199,7 @@ def review_candidate( critic_adapter: Callable | None = None, topology_adapter: Callable | None = None, acceptable_aspect_ratios: list[str] | None = None, + require_visual_enrichment: bool = False, ) -> dict: candidate = Path(path) blueprint_path = Path(blueprint) @@ -195,6 +210,21 @@ def review_candidate( width, height = image.size with Image.open(blueprint_path) as image: bw, bh = image.size + with Image.open(candidate) as candidate_image, Image.open(blueprint_path) as blueprint_image: + candidate_rgb = candidate_image.convert("RGB") + blueprint_rgb = blueprint_image.convert("RGB").resize(candidate_rgb.size) + visual_diff = ImageChops.difference(candidate_rgb, blueprint_rgb) + threshold_mask = visual_diff.point(lambda value: 255 if value > 35 else 0).convert("L") + mask_histogram = threshold_mask.histogram() + changed_pixels = sum(mask_histogram[1:]) + total_pixels = max(1, candidate_rgb.size[0] * candidate_rgb.size[1]) + blueprint_enrichment_ratio = changed_pixels / total_pixels + blueprint_mean_abs_difference = sum(ImageStat.Stat(visual_diff).mean) / 3.0 + try: + minimum_enrichment_ratio = max(0.0, min(1.0, float(os.getenv("RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO", "0.08")))) + except ValueError: + minimum_enrichment_ratio = 0.08 + visual_enrichment_passed = not require_visual_enrichment or blueprint_enrichment_ratio >= minimum_enrichment_ratio candidate_ratio = width / max(height, 1) ratio_targets: list[tuple[str, float]] = [("blueprint", bw / max(bh, 1))] for value in acceptable_aspect_ratios or []: @@ -223,7 +253,12 @@ def review_candidate( "aspect_ratio_errors": ratio_errors, "matched_aspect_ratio": matched_ratio, "aspect_ratio_error": ratio_error, - "passed": ratio_error <= 0.08 and min(width, height) >= 512, + "visual_enrichment_required": bool(require_visual_enrichment), + "blueprint_enrichment_ratio": round(blueprint_enrichment_ratio, 5), + "blueprint_mean_abs_difference": round(blueprint_mean_abs_difference, 3), + "minimum_blueprint_enrichment_ratio": minimum_enrichment_ratio, + "visual_enrichment_passed": visual_enrichment_passed, + "passed": ratio_error <= 0.08 and min(width, height) >= 512 and visual_enrichment_passed, } local_records: list[dict] = [] @@ -236,7 +271,9 @@ def review_candidate( local_records, local_engine, local_warning = _local_ocr(candidate, requested_engine, ocr_lang, adapter=ocr_adapter) topology_name = str(plan.get("figure_specification", {}).get("topology") or "unknown") - topology_required = topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} + visual_roles = collect_visual_roles(plan.get("figure_specification", {})) + has_structured_provenance = any(str(item.get("role") or "").casefold() == "data_source" for item in visual_roles) + topology_required = topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} or has_structured_provenance vlm_raw: dict = {} topology_raw: dict = {} vlm_warning = None @@ -303,6 +340,11 @@ def review_candidate( scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") + for field in ("visual_information_density", "mechanism_visualization", "publication_polish"): + try: + aesthetic[field] = max(0.0, min(1.0, float(aesthetic.get(field, aesthetic["score"])))) + except Exception: + aesthetic[field] = aesthetic["score"] topology_review = _normalize_section(topology_raw, "Topology") if topology_required else {"summary": "Focused topology review not required for this topology.", "score": 1.0, "passed": True, "skipped": True} topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) if topology_required: @@ -311,7 +353,7 @@ def review_candidate( if ocr_issues == 0: ocr["score"] = 1.0 ocr["passed"] = ocr["score"] >= 0.999 and ocr_issues == 0 - scientific_issues = sum(len(scientific.get(field, [])) for field in ["missing_modules", "missing_relations", "reversed_relations", "invented_items"]) + scientific_issues = sum(len(scientific.get(field, [])) for field in ["missing_modules", "missing_relations", "reversed_relations", "invented_items", "role_mismatches", "containment_mismatches"]) has_innovations = bool(plan.get("figure_specification", {}).get("innovations")) innovation_ok = not has_innovations or bool(scientific.get("innovation_visible", False)) if scientific_issues == 0 and innovation_ok: @@ -319,7 +361,8 @@ def review_candidate( scientific["passed"] = scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok copied = template_review.get("copied_reference_content", []) template_review["passed"] = template_review["score"] >= 0.70 and not copied - aesthetic["passed"] = aesthetic["score"] >= 0.70 + aesthetic_floor = min(aesthetic["visual_information_density"], aesthetic["mechanism_visualization"], aesthetic["publication_polish"]) + aesthetic["passed"] = aesthetic["score"] >= 0.78 and aesthetic_floor >= 0.70 and visual_enrichment_passed if mode != "vlm": template_review.update({"score": 1.0 if basic["passed"] else 0.0, "passed": basic["passed"], "engineering_only": True}) aesthetic.update({"score": 0.0, "passed": False, "engineering_only": True}) @@ -349,6 +392,10 @@ def review_candidate( repair.extend(f"Remove unexpected visible label: {value}" for value in unexpected_labels) remove = list(vlm_raw.get("remove", [])) remove.extend(unexpected_labels) + if require_visual_enrichment and not visual_enrichment_passed: + repair.append("Enrich node interiors with paper-grounded visual examples, mechanism cues, and consistent scientific icons while preserving all node boxes and connectors.") + hard_errors.append(f"Image-2 visual enrichment ratio {blueprint_enrichment_ratio:.3f} is below required {minimum_enrichment_ratio:.3f}; candidate is too close to the bare layout blueprint") + hard_errors = list(dict.fromkeys(hard_errors)) production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and topology_review["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) score = 0.25 * scientific["score"] + 0.20 * ocr["score"] + 0.20 * topology_review["score"] + 0.20 * template_review["score"] + 0.15 * aesthetic["score"] return { diff --git a/rfs/paper_to_image/editable_ppt.py b/rfs/paper_to_image/editable_ppt.py index 58b1c30..baac987 100644 --- a/rfs/paper_to_image/editable_ppt.py +++ b/rfs/paper_to_image/editable_ppt.py @@ -76,7 +76,9 @@ def build_semantic_figure_program( for node in semantic_plan["nodes"]: style = dict(_NODE_STYLES.get(str(node.get("field") or "modules"), _NODE_STYLES["modules"])) role = str(node.get("role") or "").casefold() - if any(term in role for term in ("shared", "joint", "fusion")): + if "data_source" in role or role in {"dataset", "source dataset"}: + style = {"fill_color": "#F8F4EC", "stroke_color": "#896538", "text_color": "#684B28"} + elif any(term in role for term in ("shared", "joint", "fusion")): style = {"fill_color": "#F4F1FA", "stroke_color": "#62529A", "text_color": "#4B3F78"} label = str(node.get("label") or node["id"]) semantic_nodes.append({ diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index 9e47a9a..2acadc0 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -123,6 +123,8 @@ def _candidate_failure_modes(record: dict) -> list[str]: modes.append("bypassed_relations") if scientific.get("invented_items") or topology.get("invented_relations"): modes.append("invented_content") + if scientific.get("role_mismatches") or scientific.get("containment_mismatches"): + modes.append("semantic_role_mismatch") if not template.get("passed", False): modes.append("template_alignment") if not aesthetic.get("passed", False): @@ -267,7 +269,7 @@ def _repair_prompt(review: dict, base_prompt: str, plan: dict) -> str: Mandatory directed connector checklist after repair: {connector_checklist} -Preserve every currently correct connector. Add or redirect only the connectors named by the repair findings. The repaired image must contain every checklist edge exactly in the stated direction. Never draw a shortcut that bypasses Generate, Feedback, Self-Feedback, or Refine. +Preserve every currently correct connector. Add or redirect only the connectors named by the repair findings. The repaired image must contain every checklist edge exactly in the stated direction. Never draw a shortcut that bypasses any declared intermediate module. Original scientific and visual contract: {base_prompt} @@ -294,6 +296,7 @@ def generate_and_select( critic_adapter: Callable | None = None, topology_adapter: Callable | None = None, resume_candidates: bool = False, + require_visual_enrichment: bool = True, ) -> dict: root = ensure_dir(out_dir) candidate_dir = ensure_dir(root / "candidates") @@ -335,7 +338,7 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) review_started = time.monotonic() - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) review_seconds = round(time.monotonic() - review_started, 3) return { "record": { @@ -367,7 +370,7 @@ def generate_candidate(index: int) -> dict[str, Any]: prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" write_text(prompt_path, prompt) review_started = time.monotonic() - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) review_seconds = round(time.monotonic() - review_started, 3) generation = {"mode": "existing_candidate", "source": str(repair_source_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} records.append({ @@ -448,7 +451,7 @@ def generate_candidate(index: int) -> dict[str, Any]: generation_seconds = round(time.monotonic() - generation_started, 3) requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) review_started = time.monotonic() - review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios) + review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) review_seconds = round(time.monotonic() - review_started, 3) repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"], "timings": {"generation_seconds": generation_seconds, "review_seconds": review_seconds, "total_seconds": round(time.monotonic() - repair_started, 3)}} repair_records.append(repaired) diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index cd231cb..a4bb19d 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -530,6 +530,9 @@ def add(value: object) -> None: for field in ("inputs", "modules", "outputs"): for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: add(item) + for item in spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []: + if isinstance(item, dict) and (item.get("visible_label_required") or item.get("must_appear_in_figure")): + add(item) for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: if isinstance(relation, dict): add(relation.get("label")) @@ -578,6 +581,43 @@ def collect_visual_relations(spec: dict) -> list[dict[str, str]]: return result +def collect_visual_roles(spec: dict) -> list[dict[str, str]]: + result: list[dict[str, str]] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict) or not item.get("id"): + continue + role = str(item.get("role") or field.rstrip("s")) + normalized_role = re.sub(r"[^a-z0-9]+", "", role.casefold()) + if "datasource" in normalized_role or role.casefold() in {"dataset", "source dataset"}: + constraint = "Show as an upstream data source that provides declared modalities; never show it as a peer modality or processing module." + visual_cue = "Layered dataset cards, database/archive symbols, or source-record thumbnails grounded in the named source." + elif "modality" in normalized_role: + constraint = "Show as an observed input modality downstream of any declared data sources and upstream of the processing method." + visual_cue = "A representative sample or measurement thumbnail for the named modality, not an empty text card." + elif field == "inputs": + constraint = "Show at the input boundary and connect only to its declared downstream target." + visual_cue = "A compact paper-grounded input example or input-object icon." + elif field == "outputs": + constraint = "Show at the output/evaluation boundary, not as an upstream input or internal method module." + visual_cue = "A compact outcome, prediction, or evaluation visual that does not introduce new metrics or numbers." + elif field == "innovations": + constraint = "Show as a highlighted paper innovation attached to its supported method context, not as an unrelated pipeline stage." + visual_cue = "A small mechanism inset that visually explains the named innovation." + else: + constraint = "Show as an internal method operation or component between its declared incoming and outgoing relations." + visual_cue = "A mechanism-level mini diagram or scientific icon showing what the named operation does." + result.append({ + "id": str(item.get("id")), + "label": str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")), + "field": field, + "role": role, + "visual_constraint": constraint, + "visual_cue": visual_cue, + }) + return result + + def _topology_specific_prompt_rules(spec: dict) -> str: topology = str(spec.get("topology") or "unknown") normalized_labels = { @@ -647,11 +687,66 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = for index, label in enumerate(labels, start=1) ) visual_relations = collect_visual_relations(spec) + visual_roles = collect_visual_roles(spec) connector_checklist = "\n".join( f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + (f" label: {item['label']}" if item["label"] else "") for index, item in enumerate(visual_relations, start=1) ) topology_specific_rules = _topology_specific_prompt_rules(spec) + compact_roles = [ + {"id": item["id"], "label": item["label"], "role": item["role"], "visual_cue": item["visual_cue"]} + for item in visual_roles + ] + compact_contract = { + "figure_goal": spec.get("figure_goal"), + "central_claim": spec.get("central_claim", {}).get("text") if isinstance(spec.get("central_claim"), dict) else spec.get("central_claim"), + "topology": spec.get("topology"), + "training_flow": spec.get("training_flow", []), + "inference_flow": spec.get("inference_flow", []), + "forbidden_inventions": spec.get("forbidden_inventions", []), + "uncertainties": spec.get("uncertainties", []), + } + compact_style = { + "medium": style.get("medium"), + "palette": style.get("palette", []), + "viewpoint": style.get("viewpoint"), + "line_and_shadow": style.get("line_and_shadow"), + "visual_density": style.get("visual_density"), + "background": style.get("background"), + "negative_reference_rules": style.get("negative_reference_rules", []), + "template_style": style.get("template_style", {}), + } + compact_template = { + "template_id": template.get("template_id"), + "visual_density": template.get("visual_density"), + "style": template.get("style", {}), + "panels": [] if semantic_plan.get("applied") else template.get("panels", []), + "connectors": [] if semantic_plan.get("applied") else template.get("connectors", []), + } + compact_semantic_plan = { + "applied": bool(semantic_plan.get("applied")), + "topology": semantic_plan.get("topology"), + "nodes": [ + { + "id": item.get("id"), + "label": item.get("label"), + "rank": item.get("rank"), + "bbox_percent": item.get("bbox_percent"), + } + for item in semantic_plan.get("nodes", []) if isinstance(item, dict) + ], + "connectors": [ + { + "source": item.get("source"), + "target": item.get("target"), + "type": item.get("type"), + "label": item.get("label"), + "path_percent": item.get("path_percent"), + "route_style": item.get("route_style"), + } + for item in semantic_plan.get("connectors", []) if isinstance(item, dict) + ], + } return f""" # Summary @@ -662,26 +757,17 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Target canvas ratio: {preferences.get('aspect_ratio', '16:9')}. Text language: {preferences.get('language', 'English')}. -Scientific contract: -{json.dumps(spec, ensure_ascii=False, indent=2)} - -Information narrative: -{json.dumps(design, ensure_ascii=False, indent=2)} - -Layout intent: -{json.dumps(layout, ensure_ascii=False, indent=2)} - -Concrete visual metaphors: -{json.dumps(metaphors, ensure_ascii=False, indent=2)} +Compact scientific contract: +{json.dumps(compact_contract, ensure_ascii=False, separators=(',', ':'))} Style contract: -{json.dumps(style, ensure_ascii=False, indent=2)} +{json.dumps(compact_style, ensure_ascii=False, separators=(',', ':'))} Reference-derived content-free template contract: -{json.dumps({key: template.get(key) for key in ['template_id', 'panels', 'connectors', 'visual_density', 'style', 'selection']}, ensure_ascii=False, indent=2)} +{json.dumps(compact_template, ensure_ascii=False, separators=(',', ':'))} Paper-grounded semantic blueprint geometry: -{json.dumps(semantic_plan if semantic_plan.get('applied') else {"applied": False}, ensure_ascii=False, indent=2)} +{json.dumps(compact_semantic_plan, ensure_ascii=False, separators=(',', ':'))} Exact visible label whitelist (render these exactly and do not invent alternatives): {json.dumps(labels, ensure_ascii=False)} @@ -696,6 +782,11 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows. +Mandatory entity role and containment checklist: +{json.dumps(compact_roles, ensure_ascii=False, separators=(',', ':'))} + +Preserve these semantic roles in the visible composition. A data source, modality, processing module, innovation, and output are not interchangeable even when all labels are present. In particular, do not flatten a source -> modality -> method chain into peer inputs or place a dataset beside modalities as though it were another sensor type. + Topology-specific rendering rules: {topology_specific_rules} @@ -708,6 +799,10 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = - Make arrows unambiguous and follow the specified source-to-target direction. - Emphasize the paper's actual innovation rather than generic AI decoration. - Use a clean academic composition with dense but readable information hierarchy. +- Enrich every input, core method, innovation, and output node with a paper-grounded visual cue from the role checklist. Do not return a bare text-only flowchart. +- Use roughly 35-60% of each node interior for a representative example, mechanism inset, scientific icon, or outcome cue while keeping the exact label clearly readable. +- For modality nodes, show a representative sample of that modality; for data sources, show dataset/archive cues; for method nodes, show the internal operation; for outputs, show a compact outcome/evaluation cue. +- Preserve the supplied geometry and connectors while adding visual explanation inside the nodes; do not merely add gradients or shadows to the blueprint boxes. - Keep all major objects complete and inside the canvas; avoid large blank margins. - Do not invent datasets, mechanisms, formulas, metrics, modules, or results. - Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 7f7d8a9..9c4a6a3 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -964,66 +964,343 @@ def _clear_unverbatim_relation_labels(spec: dict[str, Any], parsed: dict[str, An return removed -def _repair_isolated_input_relations(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: - """Attach a grounded source input to a shared downstream module. - - Planners sometimes extract a dataset/source label as an input but omit its - edge while correctly connecting sibling modalities from the same caption. - This repair is intentionally narrow: the isolated label must appear in the - same cited evidence as an existing input-to-module edge, and the evidence - must explicitly describe data provenance (for example, "images from X"). +def _repair_isolated_input_relations(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, list[str]]: + """Recover explicit data-source -> modality provenance from overview text. + + A common planner error is to flatten ``CFP and OCT from MEH-MIDAS and + public datasets`` into three peer inputs. That preserves all words while + changing the scientific meaning. This repair only fires for an explicit + ``using/with ... from ...`` clause in cited evidence. It reclassifies the + right-hand entities as data sources, creates a missing generic dataset + source when it is named verbatim, and connects sources to the left-hand + modalities before the existing modality-to-method flow. """ inputs = [item for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id")] - module_ids = {str(item.get("id")) for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")} relations = spec.get("relations", []) if isinstance(spec.get("relations"), list) else [] - connected = { - str(endpoint) - for relation in relations if isinstance(relation, dict) - for endpoint in (relation.get("source"), relation.get("target")) - if endpoint + by_normalized = {_normalized_label(_item_label(item)): item for item in inputs if _normalized_label(_item_label(item))} + repaired: list[str] = [] + added_sources: list[str] = [] + reclassified: list[str] = [] + removed_shortcuts: list[str] = [] + removed_orphan_sources: list[str] = [] + collapsed_source_entities: list[str] = [] + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + provenance_pattern = re.compile(r"\b(?:using|uses?|with)\s+(.{1,140}?)\s+from\s+(.{1,140}?)(?=[.;]|\bstage\s+(?:two|2)\b|$)", re.IGNORECASE) + source_term_pattern = re.compile(r"\b(?:datasets?|data|corpus|corpora|cohort|database|registry|biobank|repository)\b", re.IGNORECASE) + + for evidence in parsed.get("evidence", []): + if not isinstance(evidence, dict) or not evidence.get("id"): + continue + text = str(evidence.get("text") or "") + match = provenance_pattern.search(text) + if not match: + continue + modality_phrase, source_phrase = match.groups() + normalized_modality_phrase = _normalized_label(modality_phrase) + normalized_source_phrase = _normalized_label(source_phrase) + modalities = [ + item for item in inputs + if _normalized_label(_item_label(item)) + and _normalized_label(_item_label(item)) in normalized_modality_phrase + ] + if not modalities: + continue + sources = [ + item for item in inputs + if _normalized_label(_item_label(item)) + and _normalized_label(_item_label(item)) in normalized_source_phrase + ] + component_norms: list[str] = [] + component_labels: list[str] = [] + for raw_chunk in re.split(r"\s*(?:,|\band\b|\+)\s*", source_phrase, flags=re.IGNORECASE): + chunk = re.sub(r"^(?:the|a|an)\s+", "", raw_chunk.strip(" ()"), flags=re.IGNORECASE).strip() + normalized_chunk = _normalized_label(chunk) + if not chunk or not normalized_chunk: + continue + component_norms.append(normalized_chunk) + component_labels.append(chunk) + exact_existing = by_normalized.get(normalized_chunk) + if exact_existing: + if exact_existing not in sources: + sources.append(exact_existing) + continue + acronym_source = bool(re.fullmatch(r"[A-Z][A-Z0-9-]{2,}", chunk)) + if (not source_term_pattern.search(chunk) and not acronym_source) or len(chunk) > 50: + continue + source = { + "id": _stable_id("source", chunk, len(inputs)), + "name": chunk, + "role": "data_source", + "evidence_ids": [str(evidence["id"])], + } + inputs.append(source) + spec.setdefault("inputs", []).append(source) + by_normalized[normalized_chunk] = source + sources.append(source) + added_sources.append(str(source["id"])) + component_norms = list(dict.fromkeys(component_norms)) + if len(component_norms) >= 2: + composite = next( + ( + item for item in inputs + if isinstance(item, dict) + and _normalized_label(_item_label(item)) not in component_norms + and all(value in _normalized_label(_item_label(item)) for value in component_norms) + ), + None, + ) + if not composite: + composite_label = " + ".join(component_labels) + composite = { + "id": _stable_id("source", composite_label, len(inputs)), + "name": composite_label, + "role": "data_source", + "evidence_ids": [str(evidence["id"])], + } + inputs.append(composite) + spec.setdefault("inputs", []).append(composite) + by_normalized[_normalized_label(composite_label)] = composite + added_sources.append(str(composite["id"])) + component_ids = { + str(item.get("id")) + for item in inputs + if isinstance(item, dict) and _normalized_label(_item_label(item)) in component_norms + } + if component_ids: + collapsed_source_entities.extend(sorted(component_ids)) + inputs = [item for item in inputs if str(item.get("id")) not in component_ids] + spec["inputs"] = [item for item in spec.get("inputs", []) if not (isinstance(item, dict) and str(item.get("id")) in component_ids)] + relations = [ + item for item in relations + if not ( + isinstance(item, dict) + and (str(item.get("source")) in component_ids or str(item.get("target")) in component_ids) + ) + ] + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + added_sources = [value for value in added_sources if value not in component_ids] + for value in component_norms: + by_normalized.pop(value, None) + sources = [composite] + if not sources: + continue + + evidence_id = str(evidence["id"]) + modality_ids = {str(item.get("id")) for item in modalities} + source_ids = {str(item.get("id")) for item in sources} + downstream_targets = { + str(relation.get("target")) + for relation in relations + if isinstance(relation, dict) and str(relation.get("source")) in modality_ids + } + kept_relations = [] + for relation in relations: + if not isinstance(relation, dict): + kept_relations.append(relation) + continue + pair = (str(relation.get("source")), str(relation.get("target"))) + if pair[0] in source_ids and pair[1] in downstream_targets: + removed_shortcuts.append(f"{pair[0]}->{pair[1]}") + existing_pairs.discard(pair) + continue + kept_relations.append(relation) + relations = kept_relations + + for modality in modalities: + if str(modality.get("role") or "").casefold() != "input modality": + modality["role"] = "input modality" + reclassified.append(str(modality.get("id"))) + for source in sources: + if str(source.get("role") or "").casefold() != "data_source": + source["role"] = "data_source" + reclassified.append(str(source.get("id"))) + source["evidence_ids"] = list(dict.fromkeys([*source.get("evidence_ids", []), evidence_id])) + for modality in modalities: + pair = (str(source.get("id")), str(modality.get("id"))) + if pair in existing_pairs: + continue + relations.append({ + "source": pair[0], + "target": pair[1], + "type": "data_source", + "label": "", + "evidence_ids": [evidence_id], + }) + existing_pairs.add(pair) + repaired.append(f"{pair[0]}->{pair[1]}") + connected_sources = { + str(item.get("source")) + for item in relations + if isinstance(item, dict) and str(item.get("type") or "").casefold() == "data_source" + } + retained_inputs = [] + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + item_id = str(item.get("id") or "") if isinstance(item, dict) else "" + role = str(item.get("role") or "").casefold() if isinstance(item, dict) else "" + if item_id.startswith("source_") and role == "data_source" and item_id not in connected_sources: + removed_orphan_sources.append(item_id) + continue + retained_inputs.append(item) + spec["inputs"] = retained_inputs + spec["relations"] = relations + return { + "repaired_relations": list(dict.fromkeys(repaired)), + "added_source_entities": list(dict.fromkeys(added_sources)), + "reclassified_entities": list(dict.fromkeys(reclassified)), + "removed_source_shortcuts": list(dict.fromkeys(removed_shortcuts)), + "removed_orphan_sources": list(dict.fromkeys(removed_orphan_sources)), + "collapsed_source_entities": list(dict.fromkeys(collapsed_source_entities)), + } + + +def _repair_explicit_evaluation_outputs(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, list[str]]: + modules = [item for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")] + if not modules: + return {"added_outputs": [], "added_relations": []} + source = next( + ( + item for item in reversed(modules) + if any(term in _item_label(item).casefold() for term in ("supervised", "fine-tun", "adapt")) + ), + modules[-1], + ) + outputs = spec.setdefault("outputs", []) + by_normalized = { + _normalized_label(_item_label(item)): item + for item in outputs if isinstance(item, dict) and _normalized_label(_item_label(item)) + } + relations = spec.setdefault("relations", []) + existing_pairs = {(str(item.get("source")), str(item.get("target"))) for item in relations if isinstance(item, dict)} + added_outputs: list[str] = [] + added_relations: list[str] = [] + for evidence in parsed.get("evidence", []): + if not isinstance(evidence, dict) or not evidence.get("id"): + continue + text = str(evidence.get("text") or "") + section = str(evidence.get("section_hint") or "").casefold() + if str(evidence.get("kind") or "").casefold() != "caption" and "figure" not in section: + continue + if not re.search(r"\binternal\s+and\s+external\s+evaluation\b", text, re.IGNORECASE): + continue + evidence_id = str(evidence["id"]) + for label in ("internal evaluation", "external evaluation"): + normalized = _normalized_label(label) + output = by_normalized.get(normalized) + if not output: + output = { + "id": _stable_id("output", label, len(outputs)), + "name": label, + "role": "evaluation output", + "evidence_ids": [evidence_id], + } + outputs.append(output) + by_normalized[normalized] = output + added_outputs.append(str(output["id"])) + else: + output["evidence_ids"] = list(dict.fromkeys([*output.get("evidence_ids", []), evidence_id])) + pair = (str(source.get("id")), str(output.get("id"))) + if pair not in existing_pairs: + relations.append({"source": pair[0], "target": pair[1], "type": "evaluation", "label": "", "evidence_ids": [evidence_id]}) + existing_pairs.add(pair) + added_relations.append(f"{pair[0]}->{pair[1]}") + break + return {"added_outputs": added_outputs, "added_relations": added_relations} + + +def _repair_generic_named_modules(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + generic_names = { + "model", "foundationmodel", "proposedmodel", "ourmodel", "method", "proposedmethod", "framework", "proposedframework", } evidence_by_id = { - str(item.get("id")): str(item.get("text") or "") + str(item.get("id")): item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id") } - input_ids = {str(item.get("id")) for item in inputs} - sibling_edges = [ - relation for relation in relations - if isinstance(relation, dict) - and str(relation.get("source")) in input_ids - and str(relation.get("target")) in module_ids - ] repaired: list[str] = [] - for item in inputs: - source_id = str(item.get("id")) - if source_id in connected: - continue - label = _item_label(item).strip() - if not label: + patterns = ( + re.compile(r"\bfoundation models?\s*\(\s*([A-Za-z][A-Za-z0-9_-]{2,40})\s*\)", re.IGNORECASE), + re.compile(r"\b(?:model|method|framework)\s+(?:called|named|denoted as)\s+([A-Za-z][A-Za-z0-9_-]{2,40})\b", re.IGNORECASE), + ) + for item in spec.get("modules", []) if isinstance(spec.get("modules"), list) else []: + if not isinstance(item, dict) or _normalized_label(_item_label(item)) not in generic_names: continue - item_evidence = {str(value) for value in item.get("evidence_ids", []) if value} - for relation in sibling_edges: - shared_ids = item_evidence & {str(value) for value in relation.get("evidence_ids", []) if value} - if not shared_ids: + cited = [evidence_by_id.get(str(value)) for value in item.get("evidence_ids", [])] + candidates = [record for record in cited if record] + candidates.extend( + record for record in parsed.get("evidence", []) + if isinstance(record, dict) + and record not in candidates + and (str(record.get("kind") or "").casefold() == "caption" or int(record.get("page") or 9999) <= 3) + ) + replacement = None + replacement_evidence = None + for record in candidates: + text = str(record.get("text") or "") + match = next((pattern.search(text) for pattern in patterns if pattern.search(text)), None) + if not match: continue - cited_text = " ".join(evidence_by_id.get(value, "") for value in shared_ids) - label_pattern = re.escape(label).replace(r"\ ", r"\s+") - explicit_source = bool(re.search(rf"\b(?:from|using|uses?|sourced?\s+from)\b[^.;:]{{0,120}}{label_pattern}", cited_text, re.IGNORECASE)) - if not explicit_source: + candidate = match.group(1).strip() + if _normalized_label(candidate) in {"ssl", "cnn", "transformer", "network"}: continue - target_id = str(relation.get("target")) - relations.append({ - "source": source_id, - "target": target_id, - "type": "data_flow", - "label": "", - "evidence_ids": sorted(shared_ids), - }) - connected.add(source_id) - repaired.append(f"{source_id}->{target_id}") + replacement = candidate + replacement_evidence = str(record.get("id") or "") break - spec["relations"] = relations + if not replacement: + continue + old_label = _item_label(item) + item["name"] = replacement + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} + for key, value in list(terminology.items()): + if _normalized_label(value) == _normalized_label(old_label) and _normalized_label(key) == _normalized_label(replacement): + terminology[key] = replacement + elif _normalized_label(key) == _normalized_label(old_label): + terminology.pop(key, None) + terminology[replacement] = replacement + if replacement_evidence: + item["evidence_ids"] = list(dict.fromkeys([*item.get("evidence_ids", []), replacement_evidence])) + repaired.append(f"{old_label}->{replacement}") + return repaired + + +def _repair_explicit_named_techniques(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + existing = { + _normalized_label(_item_label(item)): item + for item in spec.get("innovations", []) if isinstance(item, dict) and _normalized_label(_item_label(item)) + } + modules = [item for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")] + ssl_module = next((item for item in modules if _normalized_label(_item_label(item)) in {"ssl", "selfsupervisedlearning", "sslpretraining"}), None) + if not ssl_module: + return [] + repaired: list[str] = [] + pattern = re.compile(r"\b(?:ssl\s+)?(?:technique|method|approach)\s*\(\s*([A-Za-z][A-Za-z -]{2,50}?)\s*(?:\d+)?\s*\)", re.IGNORECASE) + for evidence in parsed.get("evidence", []): + if not isinstance(evidence, dict) or not evidence.get("id"): + continue + match = pattern.search(str(evidence.get("text") or "")) + if not match: + continue + label = re.sub(r"\s+", " ", match.group(1)).strip() + normalized = _normalized_label(label) + if not normalized or normalized in {"ssl", "learning", "training"}: + continue + innovation = existing.get(normalized) + if innovation: + innovation["evidence_ids"] = list(dict.fromkeys([*innovation.get("evidence_ids", []), str(evidence["id"])])) + innovation["must_appear_in_figure"] = True + innovation["visible_label_required"] = True + continue + innovation = { + "id": _stable_id("innovation", label, len(existing)), + "name": label, + "role": f"technique used by {_item_label(ssl_module)}", + "evidence_ids": [str(evidence["id"])], + "must_appear_in_figure": True, + "visible_label_required": True, + } + spec.setdefault("innovations", []).append(innovation) + existing[normalized] = innovation + repaired.append(str(innovation["id"])) + break return repaired def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: @@ -1067,7 +1344,19 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d completion_report["removed_duplicate_relations"] = endpoint_report["removed_duplicates"] completion_report["removed_unresolved_relations"] = endpoint_report["removed_unresolved"] completion_report["removed_self_relations"] = endpoint_report["removed_self_relations"] - completion_report["repaired_isolated_inputs"] = _repair_isolated_input_relations(spec, parsed) + provenance_report = _repair_isolated_input_relations(spec, parsed) + completion_report["repaired_isolated_inputs"] = provenance_report["repaired_relations"] + completion_report["repaired_data_provenance"] = provenance_report["repaired_relations"] + completion_report["added_data_sources"] = provenance_report["added_source_entities"] + completion_report["reclassified_data_entities"] = provenance_report["reclassified_entities"] + completion_report["removed_data_source_shortcuts"] = provenance_report["removed_source_shortcuts"] + completion_report["removed_orphan_data_sources"] = provenance_report["removed_orphan_sources"] + completion_report["collapsed_data_source_entities"] = provenance_report["collapsed_source_entities"] + evaluation_report = _repair_explicit_evaluation_outputs(spec, parsed) + completion_report["added_explicit_evaluation_outputs"] = evaluation_report["added_outputs"] + completion_report["added_explicit_evaluation_relations"] = evaluation_report["added_relations"] + completion_report["repaired_generic_named_modules"] = _repair_generic_named_modules(spec, parsed) + completion_report["added_explicit_named_techniques"] = _repair_explicit_named_techniques(spec, parsed) if endpoint_report["repaired"]: secondary_completion = augment_contract_from_evidence(spec, parsed) for key in ("added_entities", "upgraded_entities", "adopted_entities", "added_relations", "repaired_relations", "grounded_entities"): @@ -1470,8 +1759,8 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: } -def _fast_cache_path(parsed: dict[str, Any], model: str, preferences: dict[str, Any]) -> Path: - signature = json.dumps({"version": 29, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "aspect_ratio": preferences.get("aspect_ratio"), "language": preferences.get("language")}, sort_keys=True).encode("utf-8") +def _fast_cache_path(parsed: dict[str, Any], model: str, domain_profile: str) -> Path: + signature = json.dumps({"version": 30, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "domain_profile": domain_profile}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -1559,7 +1848,7 @@ def prepare_paper_figure_contract( if fast_mode: remaining = max(0, int(provider_deadline_at - time.monotonic())) effective_mode = planner_mode if remaining >= 25 else "heuristic" - cache_path = _fast_cache_path(parsed, str(planner_model or ""), preferences) + cache_path = _fast_cache_path(parsed, str(planner_model or ""), str(selected_domain.get("id") or "general")) if cache_path.exists() and effective_mode == "vlm" and _semantic_cache_safe(parsed): plan = read_json(cache_path) planner_metadata = {"requested_mode": "vlm", "mode": "vlm", "model": planner_model, "warning": None, "prompt": "", "cached": True} diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index a6af9d1..259c309 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -608,7 +608,9 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) ) field = str(node.get("field") or "modules") role = str(node.get("role") or "").casefold() - if field == "inputs": + if "data_source" in role or role in {"dataset", "source dataset"}: + fill, outline = (248, 244, 236), (137, 101, 56) + elif field == "inputs": fill, outline = (237, 245, 252), (42, 99, 161) elif field == "outputs": fill, outline = (242, 248, 243), (66, 126, 80) diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 4f428ff..ba9ee7e 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -39,6 +39,7 @@ def run_paper_to_image( critic_adapter=None, topology_adapter=None, resume_candidates: bool = False, + require_visual_enrichment: bool = True, ) -> dict: started = time.time() effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" @@ -173,6 +174,7 @@ def run_paper_to_image( critic_adapter=critic_adapter, topology_adapter=topology_adapter, resume_candidates=resume_candidates, + require_visual_enrichment=require_visual_enrichment, ) elapsed = round(time.time() - started, 3) run_summary: dict[str, Any] = { diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 5ff2f42..a51ab4a 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -16,8 +16,9 @@ from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import required_labels, review_candidate from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio -from rfs.paper_to_image.planner import collect_visual_relations, compile_image_prompt +from rfs.paper_to_image.planner import collect_visual_relations, collect_visual_roles, compile_image_prompt from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage +from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template @@ -149,6 +150,66 @@ def test_prompt_and_critic_share_complete_visible_label_contract(self): self.assertIn("Evidence Grounding", prompt) self.assertIn("never replace an output label with an icon-only node", prompt) + def test_evidence_required_innovation_label_is_added_to_visible_contract(self): + plan = _plan() + plan["figure_specification"]["innovations"] = [{ + "id": "innovation_masked_autoencoder", + "name": "masked autoencoder", + "visible_label_required": True, + "must_appear_in_figure": True, + }] + + labels = required_labels(plan) + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("masked autoencoder", labels) + self.assertIn("masked autoencoder", prompt) + + def test_semantic_generation_prompt_stays_compact_for_nine_node_figure(self): + plan = _plan() + spec = { + "topology": "linear", + "figure_goal": "Show source, modalities, training stages, and evaluation outputs.", + "central_claim": {"text": "A named foundation model is pretrained then fine-tuned."}, + "inputs": [ + {"id": "source", "name": "Dataset + public data", "role": "data_source"}, + {"id": "image_a", "name": "Image A", "role": "input modality"}, + {"id": "image_b", "name": "Image B", "role": "input modality"}, + ], + "modules": [ + {"id": "ssl", "name": "SSL", "role": "training_objective"}, + {"id": "model", "name": "Named Model", "role": "module"}, + {"id": "tune", "name": "Supervised fine-tuning", "role": "training_objective"}, + ], + "outputs": [ + {"id": "diagnosis", "name": "Diagnosis", "role": "output"}, + {"id": "internal", "name": "internal evaluation", "role": "evaluation output"}, + {"id": "external", "name": "external evaluation", "role": "evaluation output"}, + ], + "innovations": [{"id": "masked", "name": "masked autoencoder", "role": "technique used by SSL", "visible_label_required": True}], + "required_labels": ["Dataset + public data", "Image A", "Image B", "SSL", "Named Model", "Supervised fine-tuning", "Diagnosis", "internal evaluation", "external evaluation"], + "relations": [ + {"source": "source", "target": "image_a", "type": "data_source"}, + {"source": "source", "target": "image_b", "type": "data_source"}, + {"source": "image_a", "target": "ssl", "type": "data_flow"}, + {"source": "image_b", "target": "ssl", "type": "data_flow"}, + {"source": "ssl", "target": "model", "type": "encoding"}, + {"source": "model", "target": "tune", "type": "conditioning"}, + {"source": "tune", "target": "diagnosis", "type": "prediction"}, + {"source": "tune", "target": "internal", "type": "evaluation"}, + {"source": "tune", "target": "external", "type": "evaluation"}, + ], + "forbidden_inventions": ["Do not add unsupported layers."], + } + plan["figure_specification"] = spec + semantic_plan = compile_semantic_blueprint(spec) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}, selected_template={"template_id": "linear", "semantic_plan": semantic_plan}) + + self.assertLess(len(prompt), 14000) + self.assertIn("masked autoencoder", prompt) + self.assertIn('"route_style":"straight"', prompt) + def test_visual_relation_checklist_deduplicates_edges_and_skips_shared_model_wiring(self): spec = { "inputs": [{"id": "input", "name": "Input"}], @@ -210,6 +271,30 @@ def test_candidate_review_accepts_requested_ratio_when_image2_ignores_blueprint_ self.assertTrue(review["basic"]["passed"]) self.assertEqual(review["basic"]["matched_aspect_ratio"], "16:9") + def test_image2_candidate_must_visually_enrich_the_bare_blueprint(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + review = review_candidate( + candidate, + blueprint, + _plan(), + {"template_id": "linear", "forbidden_copy_terms": []}, + mode="vlm", + ocr_engine="off", + critic_adapter=_passing_critic, + require_visual_enrichment=True, + ) + + self.assertFalse(review["basic"]["visual_enrichment_passed"]) + self.assertFalse(review["production_pass"]) + self.assertTrue(any("too close to the bare layout blueprint" in value for value in review["hard_errors"])) + self.assertTrue(any("Enrich node interiors" in value for value in review["repair"])) + def test_candidate_review_allows_evidence_supported_shared_label_reuse(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -252,6 +337,46 @@ def critic(path, template, prompt): self.assertIn("Remove unexpected visible label: Contrastive Loss", review["repair"]) self.assertIn("Contrastive Loss", review["remove"]) + def test_candidate_review_rejects_dataset_drawn_as_peer_modality(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"].update({ + "inputs": [ + {"id": "dataset", "name": "Source Dataset", "role": "data_source"}, + {"id": "image", "name": "Image", "role": "input modality"}, + ], + "relations": [ + {"source": "dataset", "target": "image", "type": "data_source"}, + {"source": "image", "target": "parser", "type": "data_flow"}, + {"source": "parser", "target": "graph", "type": "data_flow"}, + ], + "required_labels": ["Source Dataset", "Image", "Document Parser", "Evidence Graph Builder"], + }) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"] = ["Source Dataset", "Image", "Document Parser", "Evidence Graph Builder"] + result["scientific"]["role_mismatches"] = ["Source Dataset is shown as a peer input modality instead of the source of Image"] + return result + + def topology(_path, _prompt): + return {"summary": "Topology pass.", "relations": [], "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], "repair": [], "repair_regions": [], "score": 1.0, "passed": True} + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic, topology_adapter=topology) + + self.assertFalse(review["production_pass"]) + self.assertFalse(review["scientific"]["passed"]) + self.assertEqual(len(review["scientific"]["role_mismatches"]), 1) + roles = collect_visual_roles(plan["figure_specification"]) + self.assertIn("never show it as a peer modality", next(item["visual_constraint"] for item in roles if item["id"] == "dataset")) + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + self.assertIn("source -> modality -> method chain", prompt) + def test_visible_label_whitelist_includes_declared_connector_labels(self): plan = _plan() plan["figure_specification"]["relations"][0]["label"] = "iterate" @@ -420,7 +545,7 @@ def test_generic_linear_blueprint_renders_contract_nodes_and_connectors(self): self.assertTrue(Path(report["path"]).exists()) prompt = compile_image_prompt(_plan(), {"aspect_ratio": "3:2", "language": "English"}, selected_template={**linear, "semantic_plan": report["semantic_plan"]}) self.assertIn("Paper-grounded semantic blueprint geometry", prompt) - self.assertIn('"route_style": "straight"', prompt) + self.assertIn('"route_style":"straight"', prompt) self.assertIn("preserve every node bounding box", prompt) def test_generic_branch_multimodal_and_feedback_blueprints_render_without_paper_specific_rules(self): @@ -560,7 +685,7 @@ def test_issue_free_ten_point_scale_review_is_normalized_by_findings(self): def critic(path, template, prompt): result = _passing_critic(path, template, prompt) for section in ("ocr", "scientific", "template", "aesthetic"): - result[section]["score"] = 7.0 + result[section]["score"] = 8.0 result[section]["passed"] = False return result @@ -570,8 +695,8 @@ def critic(path, template, prompt): self.assertTrue(review["production_pass"]) self.assertEqual(review["ocr"]["score"], 1.0) self.assertEqual(review["scientific"]["score"], 1.0) - self.assertEqual(review["template"]["score"], 0.7) - self.assertEqual(review["aesthetic"]["score"], 0.7) + self.assertEqual(review["template"]["score"], 0.8) + self.assertEqual(review["aesthetic"]["score"], 0.8) def test_inspect_pdf_cli_defaults(self): parser = build_parser() @@ -815,7 +940,7 @@ def test_mock_image2_generates_three_candidates_and_safe_manifest(self, _post): root = Path(temp) template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") - result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic) + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic, require_visual_enrichment=False) self.assertTrue(result["selected_passed_all_checks"]) self.assertEqual(result["successful_candidates"], 3) self.assertTrue((root / "selected_image.png").exists()) @@ -871,7 +996,7 @@ def response(*_args, **_kwargs): template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") - result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic) + result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic, require_visual_enrichment=False) self.assertEqual(post.call_count, 4) self.assertEqual(result["stability"]["production_pass_rate"], 1.0) @@ -888,7 +1013,7 @@ def test_resume_candidates_reuses_existing_images_and_generates_only_missing_see Image.new("RGB", (1536, 1024), "white").save(candidate_dir / "candidate_01.png") Image.new("RGB", (1536, 1024), "white").save(candidate_dir / "candidate_02.png") - result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic, resume_candidates=True) + result = generate_and_select(_plan(), {"aspect_ratio": "3:2"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, image_retries=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", critic_adapter=_passing_critic, resume_candidates=True, require_visual_enrichment=False) self.assertEqual(post.call_count, 1) modes = {item["candidate_id"]: item["generation"]["mode"] for item in result["candidates"]} @@ -918,6 +1043,7 @@ def test_repair_source_skips_fresh_candidate_generation_when_source_passes(self, repair_rounds=1, repair_source=source, critic_adapter=_passing_critic, + require_visual_enrichment=False, ) post.assert_not_called() @@ -942,7 +1068,7 @@ def critic(path, blueprint, prompt): root = Path(temp) template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") - result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=critic) + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=3, review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=critic, require_visual_enrichment=False) self.assertEqual(result["repair_candidates"], 1) self.assertEqual(result["selected_candidate_id"], "repair_01") repair = next(item for item in result["candidates"] if item["candidate_id"] == "repair_01") @@ -960,7 +1086,7 @@ def test_mock_full_production_workflow(self, planner_call, _coverage, _post): paper = root / "paper.md" paper.write_text(PAPER_TEXT, encoding="utf-8") out = root / "run" - result = run_paper_to_image(paper=paper, out=out, planner_mode="vlm", domain_profile="general", template="linear", asset_mode="image2", candidates=3, aspect_ratio="1.5:1", review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic) + result = run_paper_to_image(paper=paper, out=out, planner_mode="vlm", domain_profile="general", template="linear", asset_mode="image2", candidates=3, aspect_ratio="1.5:1", review_mode="vlm", repair_rounds=1, ocr_engine="vlm", critic_adapter=_passing_critic, require_visual_enrichment=False) self.assertTrue(result["ok"]) self.assertEqual(result["paper_review_mode"], "derived_from_vlm_plan") self.assertEqual(result["template_id"], "linear") diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index a0bdbf2..7ff8bdf 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -2,7 +2,7 @@ import unittest from rfs.paper_to_image.planner import validate_plan_grounding -from rfs.paper_to_image.preparation import build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract +from rfs.paper_to_image.preparation import _fast_cache_path, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract from rfs.paper_to_image.contract_completion import _overview_candidates @@ -1370,12 +1370,15 @@ def test_generic_completion_uses_selected_architecture_page_context_and_diagram_ self.assertNotIn("Text Encoder", ids) self.assertNotIn("Contrastive Learning", ids) - def test_isolated_dataset_input_is_connected_when_caption_explicitly_supplies_sibling_modalities(self): - caption = "Figure 1: Stage one constructs RETFound by means of SSL, using CFP and OCT from MEH-MIDAS and public datasets." + def test_dataset_sources_are_separated_from_modalities_when_caption_states_provenance(self): + caption = "Figure 1: Stage one constructs RETFound by means of SSL, using CFP and OCT from MEH-MIDAS and public datasets. Stage two adapts RETFound by supervised fine-tuning for internal and external evaluation." parsed = { "page_count": 4, "document_index": {"figures": [{"page": 2, "caption": caption}]}, - "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + "evidence": [ + {"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}, + {"id": "E0002", "page": 3, "kind": "paragraph", "text": "Models were adapted with labelled training data and internally evaluated on hold-out test data from the MEH-AlzEye dataset.", "section_hint": "Evaluation", "confidence": 1.0}, + ], } plan = { "paper_summary": {"unknowns": []}, @@ -1384,23 +1387,99 @@ def test_isolated_dataset_input_is_connected_when_caption_explicitly_supplies_si "inputs": [ {"id": "cfp", "name": "CFP", "evidence_ids": ["E0001"]}, {"id": "oct", "name": "OCT", "evidence_ids": ["E0001"]}, - {"id": "meh", "name": "MEH-MIDAS", "evidence_ids": ["E0001"]}, ], - "modules": [{"id": "ssl", "name": "SSL", "evidence_ids": ["E0001"]}], + "modules": [ + {"id": "ssl", "name": "SSL", "evidence_ids": ["E0001"]}, + {"id": "supervised", "name": "supervised fine-tuning", "evidence_ids": ["E0001"]}, + ], "outputs": [], "innovations": [], "relations": [ {"source": "cfp", "target": "ssl", "type": "data_flow", "evidence_ids": ["E0001"]}, {"source": "oct", "target": "ssl", "type": "data_flow", "evidence_ids": ["E0001"]}, + {"source": "ssl", "target": "supervised", "type": "data_flow", "evidence_ids": ["E0001"]}, ], }, } spec = normalize_figure_contract(plan, parsed) pairs = {(item["source"], item["target"]) for item in spec["relations"]} + by_name = {item["name"]: item for item in spec["inputs"]} + source_id = by_name["MEH-MIDAS + public datasets"]["id"] + + self.assertEqual(by_name["MEH-MIDAS + public datasets"]["role"], "data_source") + self.assertEqual(by_name["CFP"]["role"], "input modality") + self.assertEqual(by_name["OCT"]["role"], "input modality") + self.assertIn((source_id, "cfp"), pairs) + self.assertIn((source_id, "oct"), pairs) + self.assertNotIn((source_id, "ssl"), pairs) + self.assertIn(source_id, plan["contract_completion_report"]["added_data_sources"]) + self.assertNotIn("MEH-MIDAS", by_name) + self.assertNotIn("public datasets", by_name) + self.assertNotIn("MEH-AlzEye dataset", by_name) + output_names = {item["name"]: item["id"] for item in spec["outputs"]} + self.assertIn(("supervised", output_names["internal evaluation"]), pairs) + self.assertIn(("supervised", output_names["external evaluation"]), pairs) + + def test_fast_contract_cache_is_scoped_by_scientific_domain_not_visual_preferences(self): + parsed = {"source_sha256": "abc123"} + + first = _fast_cache_path(parsed, "planner-model", "ai-ml-method") + second = _fast_cache_path(parsed, "planner-model", "ai-ml-method") + + self.assertEqual(first, second) + self.assertNotEqual(first, _fast_cache_path(parsed, "planner-model", "empirical-science")) + + def test_generic_foundation_model_label_is_replaced_by_explicit_paper_name(self): + caption = "Figure 1: Schematic of development and evaluation of the foundation models (RETFound)." + parsed = { + "page_count": 4, + "document_index": {"figures": [{"page": 2, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 2, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "linear", + "inputs": [], + "modules": [{"id": "model", "name": "foundation model", "role": "module", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [], + "terminology": {"RETFound": "foundation model"}, + }, + } + + spec = normalize_figure_contract(plan, parsed) + + self.assertEqual(spec["modules"][0]["name"], "RETFound") + self.assertEqual(spec["terminology"], {"RETFound": "RETFound"}) + self.assertIn("foundation model->RETFound", plan["contract_completion_report"]["repaired_generic_named_modules"]) + + def test_explicit_named_ssl_technique_is_restored_as_required_innovation(self): + parsed = { + "page_count": 4, + "document_index": {"figures": []}, + "evidence": [{"id": "E0001", "page": 2, "kind": "paragraph", "text": "We use an advanced SSL technique (masked autoencoder 15) on retinal images.", "section_hint": "Method", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "linear", + "inputs": [], + "modules": [{"id": "ssl", "name": "SSL", "role": "training_objective", "evidence_ids": ["E0001"]}], + "outputs": [], + "innovations": [], + "relations": [], + }, + } + + spec = normalize_figure_contract(plan, parsed) - self.assertIn(("meh", "ssl"), pairs) - self.assertIn("meh->ssl", plan["contract_completion_report"]["repaired_isolated_inputs"]) + self.assertEqual(spec["innovations"][0]["name"], "masked autoencoder") + self.assertTrue(spec["innovations"][0]["visible_label_required"]) + self.assertTrue(spec["innovations"][0]["must_appear_in_figure"]) + self.assertTrue(plan["contract_completion_report"]["added_explicit_named_techniques"]) def test_generic_completion_recovers_retrieval_conditioned_generation(self): caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." From 883506e21e0e3348c08d3bdbf44fc2f716277068 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 00:20:18 +0800 Subject: [PATCH 63/74] Normalize branch heads and feedback enrichment --- README.md | 2 + docs/architecture.md | 5 + docs/paper-to-image.md | 15 ++- rfs/paper_to_image/critics.py | 27 +++- rfs/paper_to_image/planner.py | 15 ++- rfs/paper_to_image/preparation.py | 155 ++++++++++++++++++++++- rfs/paper_to_image/templates.py | 5 +- tests/integration/test_paper_to_image.py | 54 +++++++- tests/unit/test_paper_contract.py | 41 ++++++ 9 files changed, 310 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9b3486f..99abdef 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,8 @@ The contract normalizer preserves semantic roles instead of checking labels alon Production Image-2 review rejects a neat but empty text-only flowchart. The critic checks entity-role mismatches, focused provenance topology, mechanism visualization, visual information density, and publication polish. A deterministic blueprint-enrichment gate requires substantive visual content beyond gradients and shadows; configure its default `0.08` changed-pixel threshold with `RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO`. Generation prompts use a compact scientific contract so evidence, geometry, roles, and visual requirements remain explicit without repeating the full planning documents. +Branch contracts now promote evidence-backed classification, regression, mask, prediction, and similar parallel leaf heads to one consistent output role, remove ancestor shortcuts that bypass the shared branch point, and merge duplicate innovation labels into the existing node instead of drawing a second node. Feedback blueprints anchor nested feedback connectors at the actual inner artifact rather than the outer container. Visual enrichment is measured inside the blueprint's active figure region while retaining the same `0.08` threshold, so large white margins do not reject a genuinely illustrated figure and sparse box-only diagrams still fail. + Offline engineering validation: ```powershell diff --git a/docs/architecture.md b/docs/architecture.md index 2d0f204..da6acdf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,6 +69,11 @@ version, and scientific domain profile. Canvas ratio and display-language preferences are intentionally excluded because they are rendering concerns and must not produce competing scientific interpretations of the same paper. +Topology normalization also owns role-consistent branch heads and shortcut +removal, while deterministic template rendering owns exact nested connector +anchors. Candidate review measures raster enrichment inside the detected active +blueprint region and records both active-region and whole-canvas ratios. + ## Stable workflow boundaries - `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index aa3d3ab..9db7048 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -204,11 +204,24 @@ them. Image-2 candidates must enrich the semantic blueprint rather than merely restyling its boxes. Candidate review records `blueprint_enrichment_ratio` and `blueprint_mean_abs_difference`; production defaults require at least `0.08` -changed-pixel enrichment. The VLM aesthetic review separately scores visual +changed-pixel enrichment inside the blueprint's detected active figure region. +Reports also retain `blueprint_global_enrichment_ratio`, +`blueprint_active_region_bbox`, and `blueprint_active_region_fraction` for +diagnosis. This keeps the threshold strict without penalizing feedback or +branch layouts that intentionally leave large white margins. The VLM aesthetic review separately scores visual information density, mechanism visualization, and publication polish. Sparse text-only nodes therefore fail even when labels, arrows, spacing, and colors are otherwise correct. +For branch figures, evidence-backed parallel leaf heads are normalized to the +same output-head role. Relations from an upstream ancestor directly into one +head are removed when they bypass the shared branch point already feeding that +head. If an innovation label is identical to an existing module or output, the +role contract highlights the existing node rather than requesting a duplicate. +For nested feedback layouts, the deterministic blueprint starts the feedback-to- +refinement connector at the inner feedback artifact, not the outer container +boundary. + The same generic semantic plan can now bypass raster generation entirely: ```powershell diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 62708f0..16f0d33 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -218,7 +218,28 @@ def review_candidate( mask_histogram = threshold_mask.histogram() changed_pixels = sum(mask_histogram[1:]) total_pixels = max(1, candidate_rgb.size[0] * candidate_rgb.size[1]) - blueprint_enrichment_ratio = changed_pixels / total_pixels + blueprint_global_enrichment_ratio = changed_pixels / total_pixels + corner_pixels = [ + blueprint_rgb.getpixel((0, 0)), + blueprint_rgb.getpixel((blueprint_rgb.width - 1, 0)), + blueprint_rgb.getpixel((0, blueprint_rgb.height - 1)), + blueprint_rgb.getpixel((blueprint_rgb.width - 1, blueprint_rgb.height - 1)), + ] + background_color = tuple(sorted(pixel[channel] for pixel in corner_pixels)[len(corner_pixels) // 2] for channel in range(3)) + background = Image.new("RGB", blueprint_rgb.size, background_color) + foreground_mask = ImageChops.difference(blueprint_rgb, background).point(lambda value: 255 if value > 8 else 0).convert("L") + active_bbox = foreground_mask.getbbox() or (0, 0, blueprint_rgb.width, blueprint_rgb.height) + padding = 24 + active_bbox = ( + max(0, active_bbox[0] - padding), + max(0, active_bbox[1] - padding), + min(blueprint_rgb.width, active_bbox[2] + padding), + min(blueprint_rgb.height, active_bbox[3] + padding), + ) + active_diff = threshold_mask.crop(active_bbox) + active_pixels = max(1, active_diff.width * active_diff.height) + blueprint_active_enrichment_ratio = sum(active_diff.histogram()[1:]) / active_pixels + blueprint_enrichment_ratio = blueprint_active_enrichment_ratio blueprint_mean_abs_difference = sum(ImageStat.Stat(visual_diff).mean) / 3.0 try: minimum_enrichment_ratio = max(0.0, min(1.0, float(os.getenv("RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO", "0.08")))) @@ -255,6 +276,10 @@ def review_candidate( "aspect_ratio_error": ratio_error, "visual_enrichment_required": bool(require_visual_enrichment), "blueprint_enrichment_ratio": round(blueprint_enrichment_ratio, 5), + "blueprint_global_enrichment_ratio": round(blueprint_global_enrichment_ratio, 5), + "blueprint_active_enrichment_ratio": round(blueprint_active_enrichment_ratio, 5), + "blueprint_active_region_bbox": list(active_bbox), + "blueprint_active_region_fraction": round(active_pixels / total_pixels, 5), "blueprint_mean_abs_difference": round(blueprint_mean_abs_difference, 3), "minimum_blueprint_enrichment_ratio": minimum_enrichment_ratio, "visual_enrichment_passed": visual_enrichment_passed, diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index a4bb19d..b277a94 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -583,10 +583,19 @@ def collect_visual_relations(spec: dict) -> list[dict[str, str]]: def collect_visual_roles(spec: dict) -> list[dict[str, str]]: result: list[dict[str, str]] = [] + label_to_index: dict[str, int] = {} for field in ("inputs", "modules", "outputs", "innovations"): for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: if not isinstance(item, dict) or not item.get("id"): continue + label = str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")) + normalized_label = re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) + if field == "innovations" and normalized_label in label_to_index: + existing = result[label_to_index[normalized_label]] + existing["visual_constraint"] += " Also highlight this entity as an evidence-backed paper innovation without duplicating the node." + existing["visual_cue"] += " Add a compact innovation emphasis or mechanism inset inside the same entity." + existing["innovation"] = "true" + continue role = str(item.get("role") or field.rstrip("s")) normalized_role = re.sub(r"[^a-z0-9]+", "", role.casefold()) if "datasource" in normalized_role or role.casefold() in {"dataset", "source dataset"}: @@ -598,7 +607,7 @@ def collect_visual_roles(spec: dict) -> list[dict[str, str]]: elif field == "inputs": constraint = "Show at the input boundary and connect only to its declared downstream target." visual_cue = "A compact paper-grounded input example or input-object icon." - elif field == "outputs": + elif field == "outputs" or any(term in normalized_role for term in ("output", "prediction", "evaluation", "classification", "regressionhead")): constraint = "Show at the output/evaluation boundary, not as an upstream input or internal method module." visual_cue = "A compact outcome, prediction, or evaluation visual that does not introduce new metrics or numbers." elif field == "innovations": @@ -609,12 +618,14 @@ def collect_visual_roles(spec: dict) -> list[dict[str, str]]: visual_cue = "A mechanism-level mini diagram or scientific icon showing what the named operation does." result.append({ "id": str(item.get("id")), - "label": str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")), + "label": label, "field": field, "role": role, "visual_constraint": constraint, "visual_cue": visual_cue, }) + if normalized_label: + label_to_index[normalized_label] = len(result) - 1 return result diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 9c4a6a3..2c4260b 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -457,7 +457,7 @@ def _deduplicate_contract_entities(spec: dict[str, Any]) -> dict[str, list[str]] def _simplify_branch_contract(spec: dict[str, Any]) -> dict[str, list[str]]: if str(spec.get("topology") or "") != "branch": - return {"merged_entities": [], "removed_cross_branch_relations": []} + return {"merged_entities": [], "removed_cross_branch_relations": [], "removed_branch_shortcuts": []} def group_for(item: dict[str, Any]) -> str | None: label = _normalized_label(_item_label(item)) @@ -545,8 +545,71 @@ def rank(record: tuple[str, dict[str, Any]]) -> tuple[int, int]: continue seen_relations[key] = relation deduplicated.append(relation) + branch_head_ids = { + str(item.get("id")) + for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) + if isinstance(item, dict) + and item.get("id") + and (group_for(item) in branch_groups or "outputhead" in _normalized_label(str(item.get("role") or ""))) + } + source_head_counts: dict[str, set[str]] = {} + for relation in deduplicated: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if target in branch_head_ids: + source_head_counts.setdefault(source, set()).add(target) + branch_source = next( + ( + source for source, targets in sorted(source_head_counts.items(), key=lambda item: (-len(item[1]), item[0])) + if len(targets) >= 2 + ), + None, + ) + removed_branch_shortcuts: list[str] = [] + if branch_source: + adjacency: dict[str, set[str]] = {} + for relation in deduplicated: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if source and target and target not in branch_head_ids: + adjacency.setdefault(source, set()).add(target) + + def reaches_branch_source(start: str) -> bool: + pending = [start] + visited: set[str] = set() + while pending: + current = pending.pop() + if current == branch_source: + return True + if current in visited: + continue + visited.add(current) + pending.extend(adjacency.get(current, set()) - visited) + return False + + direct_branch_pairs = { + (str(item.get("source") or ""), str(item.get("target") or "")) + for item in deduplicated + } + kept_relations: list[dict[str, Any]] = [] + for relation in deduplicated: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + bypasses_branch_point = ( + target in branch_head_ids + and source != branch_source + and (branch_source, target) in direct_branch_pairs + and reaches_branch_source(source) + ) + if bypasses_branch_point: + removed_branch_shortcuts.append(f"{source}->{target}") + continue + kept_relations.append(relation) + deduplicated = kept_relations + spec["relations"] = deduplicated - return {"merged_entities": merged_entities, "removed_cross_branch_relations": removed_cross_branch_relations} + return { + "merged_entities": merged_entities, + "removed_cross_branch_relations": removed_cross_branch_relations, + "removed_branch_shortcuts": list(dict.fromkeys(removed_branch_shortcuts)), + } def _simplify_multimodal_contract(spec: dict[str, Any]) -> dict[str, Any]: @@ -1303,6 +1366,90 @@ def _repair_explicit_named_techniques(spec: dict[str, Any], parsed: dict[str, An break return repaired + +def _repair_parallel_output_heads(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, list[str]]: + """Promote evidence-backed leaf branches to output-head roles. + + Branch planners often place classification, regression, mask, score, or + prediction heads in ``modules`` while placing only one sibling in + ``outputs``. That makes visually equivalent parallel heads receive + incompatible role instructions. Promotion is limited to leaf entities + sharing one upstream branch point, with at least two output-like siblings + and explicit branch/head/parallel evidence. + """ + if str(spec.get("topology") or "").casefold() != "branch": + return {"promoted_outputs": [], "reclassified_outputs": []} + modules = [item for item in spec.get("modules", []) if isinstance(item, dict) and item.get("id")] + outputs = [item for item in spec.get("outputs", []) if isinstance(item, dict) and item.get("id")] + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + if not modules or not relations: + return {"promoted_outputs": [], "reclassified_outputs": []} + + entity_by_id = {str(item["id"]): item for item in [*modules, *outputs]} + outgoing = {str(item.get("source")) for item in relations if item.get("source")} + incoming_by_source: dict[str, set[str]] = {} + relation_evidence: dict[tuple[str, str], list[str]] = {} + for relation in relations: + source, target = str(relation.get("source") or ""), str(relation.get("target") or "") + if not source or not target: + continue + incoming_by_source.setdefault(source, set()).add(target) + relation_evidence.setdefault((source, target), []).extend(str(value) for value in relation.get("evidence_ids", []) if value) + + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + output_name_pattern = re.compile( + r"\b(?:class(?:ification|ifier)?|regression|mask|segmentation|detection|prediction|score|label|output|head)\b", + re.IGNORECASE, + ) + branch_evidence_pattern = re.compile(r"\b(?:parallel|branch(?:es)?|heads?|multi[- ]?task)\b", re.IGNORECASE) + promoted: list[str] = [] + reclassified: list[str] = [] + + for source_id, target_ids in incoming_by_source.items(): + leaf_targets = [ + entity_by_id[target_id] + for target_id in target_ids + if target_id in entity_by_id and target_id not in outgoing + ] + output_like = [item for item in leaf_targets if output_name_pattern.search(_item_label(item))] + if len(output_like) < 2: + continue + evidence_ids = list(dict.fromkeys( + value + for item in output_like + for value in [ + *item.get("evidence_ids", []), + *relation_evidence.get((source_id, str(item.get("id"))), []), + ] + if value + )) + evidence_text = " ".join(evidence_by_id.get(str(value), "") for value in evidence_ids) + if not branch_evidence_pattern.search(evidence_text): + continue + output_ids = {str(item.get("id")) for item in outputs} + for item in output_like: + item_id = str(item.get("id")) + if str(item.get("role") or "").casefold() != "output head": + item["role"] = "output head" + reclassified.append(item_id) + if item_id not in output_ids: + outputs.append(item) + output_ids.add(item_id) + promoted.append(item_id) + + if promoted: + promoted_set = set(promoted) + spec["modules"] = [item for item in modules if str(item.get("id")) not in promoted_set] + spec["outputs"] = outputs + return { + "promoted_outputs": list(dict.fromkeys(promoted)), + "reclassified_outputs": list(dict.fromkeys(reclassified)), + } + def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: summary = plan.get("paper_summary") if isinstance(plan.get("paper_summary"), dict) else {} spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} @@ -1357,6 +1504,9 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d completion_report["added_explicit_evaluation_relations"] = evaluation_report["added_relations"] completion_report["repaired_generic_named_modules"] = _repair_generic_named_modules(spec, parsed) completion_report["added_explicit_named_techniques"] = _repair_explicit_named_techniques(spec, parsed) + output_head_report = _repair_parallel_output_heads(spec, parsed) + completion_report["promoted_parallel_output_heads"] = output_head_report["promoted_outputs"] + completion_report["reclassified_parallel_output_heads"] = output_head_report["reclassified_outputs"] if endpoint_report["repaired"]: secondary_completion = augment_contract_from_evidence(spec, parsed) for key in ("added_entities", "upgraded_entities", "adopted_entities", "added_relations", "repaired_relations", "grounded_entities"): @@ -1507,6 +1657,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: branch_report = _simplify_branch_contract(spec) completion_report["branch_merged_entities"] = branch_report["merged_entities"] completion_report["removed_cross_branch_relations"] = branch_report["removed_cross_branch_relations"] + completion_report["removed_branch_shortcuts"] = branch_report["removed_branch_shortcuts"] spec.setdefault("training_flow", summary.get("training_flow") if isinstance(summary.get("training_flow"), list) else []) spec.setdefault("inference_flow", summary.get("inference_flow") if isinstance(summary.get("inference_flow"), list) else []) spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 259c309..92d5326 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -506,7 +506,8 @@ def bottom_at(box, fraction): return (round(box[0] + fraction * (box[2] - box[0] initial_start, initial_end = bottom_at(initial_box, 0.45), top_at(refinement_box, 0.35) initial_mid_y = round((initial_start[1] + initial_end[1]) / 2) draw_arrow_path([initial_start, (initial_start[0], initial_mid_y), (initial_end[0], initial_mid_y), initial_end], (31, 117, 126)) - feedback_start, feedback_end = bottom_at(feedback_box, 0.50), top_at(refinement_box, 0.72) + feedback_source_box = region(feedback_box, 0.13, 0.60, 0.87, 0.88) if semantic_feedback else feedback_box + feedback_start, feedback_end = bottom_at(feedback_source_box, 0.50), top_at(refinement_box, 0.72) feedback_mid_y = round((feedback_start[1] + feedback_end[1]) / 2) draw_arrow_path([feedback_start, (feedback_start[0], feedback_mid_y), (feedback_end[0], feedback_mid_y), feedback_end], (31, 117, 126)) draw_arrow_path([right(refinement_box), left(refined_box)], (31, 117, 126)) @@ -533,7 +534,7 @@ def bottom_at(box, fraction): return (round(box[0] + fraction * (box[2] - box[0] draw.rounded_rectangle(badge_box, radius=10, fill=(235, 243, 248), outline=(126, 157, 177), width=2) centered_text(badge_box, exact("Model M"), badge_size, bold=False) - self_feedback_box = region(feedback_box, 0.13, 0.60, 0.87, 0.88) + self_feedback_box = feedback_source_box draw.rounded_rectangle(self_feedback_box, radius=10, fill=(238, 249, 248), outline=(31, 117, 126), width=2) centered_text(self_feedback_box, exact("Self-Feedback"), badge_size) internal_start = (round((feedback_box[0] + feedback_box[2]) / 2), region(feedback_box, 0, 0.47, 1, 0.47)[1]) diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index a51ab4a..c28d747 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -10,7 +10,7 @@ import fitz import requests -from PIL import Image +from PIL import Image, ImageDraw from rfs.cli import _doctor, _probe_executable, build_parser from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image @@ -295,6 +295,34 @@ def test_image2_candidate_must_visually_enrich_the_bare_blueprint(self): self.assertTrue(any("too close to the bare layout blueprint" in value for value in review["hard_errors"])) self.assertTrue(any("Enrich node interiors" in value for value in review["repair"])) + def test_visual_enrichment_is_measured_inside_the_blueprint_active_region(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + base = Image.new("RGB", (1000, 1000), "white") + draw = ImageDraw.Draw(base) + draw.rectangle((100, 100, 900, 900), outline="#1F4E79", width=4) + base.save(blueprint) + enriched = base.copy() + ImageDraw.Draw(enriched).rectangle((350, 350, 649, 549), fill="#3A9D9A") + enriched.save(candidate) + + review = review_candidate( + candidate, + blueprint, + _plan(), + {"template_id": "linear", "forbidden_copy_terms": []}, + mode="vlm", + ocr_engine="off", + critic_adapter=_passing_critic, + require_visual_enrichment=True, + ) + + self.assertLess(review["basic"]["blueprint_global_enrichment_ratio"], 0.08) + self.assertGreater(review["basic"]["blueprint_active_enrichment_ratio"], 0.08) + self.assertTrue(review["basic"]["visual_enrichment_passed"]) + def test_candidate_review_allows_evidence_supported_shared_label_reuse(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -377,6 +405,24 @@ def topology(_path, _prompt): prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) self.assertIn("source -> modality -> method chain", prompt) + def test_visual_role_contract_merges_duplicate_innovation_with_existing_node(self): + spec = { + "inputs": [{"id": "image", "name": "Input Image", "role": "input"}], + "modules": [{"id": "roi", "name": "RoIAlign", "role": "module"}], + "outputs": [{"id": "mask", "name": "mask branch", "role": "output head"}], + "innovations": [ + {"id": "innovation_roi", "name": "RoIAlign"}, + {"id": "innovation_mask", "name": "mask branch"}, + ], + } + + roles = collect_visual_roles(spec) + + self.assertEqual([item["label"] for item in roles].count("RoIAlign"), 1) + self.assertEqual([item["label"] for item in roles].count("mask branch"), 1) + self.assertIn("without duplicating the node", next(item["visual_constraint"] for item in roles if item["label"] == "RoIAlign")) + self.assertIn("output/evaluation boundary", next(item["visual_constraint"] for item in roles if item["label"] == "mask branch")) + def test_visible_label_whitelist_includes_declared_connector_labels(self): plan = _plan() plan["figure_specification"]["relations"][0]["label"] = "iterate" @@ -454,6 +500,12 @@ def test_feedback_blueprint_embeds_exact_semantic_labels(self): self.assertTrue(report["contains_paper_semantic_labels"]) self.assertEqual(report["semantic_labels"][-1], "iterate") self.assertTrue(Path(report["path"]).exists()) + with Image.open(report["path"]).convert("RGB") as rendered: + feedback_box = next(item["bbox_percent"] for item in feedback["panels"] if item["id"] == "feedback") + connector_x = round((feedback_box["x"] + 0.50 * feedback_box["w"]) * rendered.width) + self_feedback_bottom = round((feedback_box["y"] + 0.88 * feedback_box["h"]) * rendered.height) + connector_pixel = rendered.getpixel((connector_x, self_feedback_bottom + 5)) + self.assertLess(sum(abs(value - expected) for value, expected in zip(connector_pixel, (31, 117, 126))), 30) def test_generic_feedback_contract_does_not_receive_self_refine_blueprint_or_prompt_terms(self): with tempfile.TemporaryDirectory() as temp: diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 7ff8bdf..936022b 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -1481,6 +1481,47 @@ def test_explicit_named_ssl_technique_is_restored_as_required_innovation(self): self.assertTrue(spec["innovations"][0]["must_appear_in_figure"]) self.assertTrue(plan["contract_completion_report"]["added_explicit_named_techniques"]) + def test_parallel_leaf_heads_are_promoted_to_consistent_output_roles(self): + caption = "The mask branch is in parallel with the classification and bounding-box regression heads." + parsed = { + "page_count": 3, + "document_index": {"figures": [{"page": 1, "caption": caption}]}, + "evidence": [{"id": "E0001", "page": 1, "kind": "caption", "text": caption, "section_hint": "Figure Captions", "confidence": 1.0}], + } + plan = { + "paper_summary": {"unknowns": []}, + "figure_specification": { + "topology": "branch", + "inputs": [], + "modules": [ + {"id": "roialign", "name": "RoIAlign", "role": "module", "evidence_ids": ["E0001"]}, + {"id": "box", "name": "bounding-box regression", "role": "module", "evidence_ids": ["E0001"]}, + {"id": "mask", "name": "mask branch", "role": "module", "evidence_ids": ["E0001"]}, + ], + "outputs": [{"id": "classification", "name": "Classification", "role": "output", "evidence_ids": ["E0001"]}], + "innovations": [{"id": "innovation_mask", "name": "mask branch", "evidence_ids": ["E0001"]}], + "relations": [ + {"source": "proposal", "target": "roialign", "type": "conditioning", "evidence_ids": ["E0001"]}, + {"source": "roialign", "target": "classification", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roialign", "target": "box", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "roialign", "target": "mask", "type": "branch", "evidence_ids": ["E0001"]}, + {"source": "proposal", "target": "mask", "type": "data_flow", "evidence_ids": ["E0001"]}, + ], + }, + } + plan["figure_specification"]["modules"].insert(0, {"id": "proposal", "name": "Region Proposals", "role": "module", "evidence_ids": ["E0001"]}) + + spec = normalize_figure_contract(plan, parsed) + output_by_id = {item["id"]: item for item in spec["outputs"]} + + self.assertEqual(set(output_by_id), {"classification", "box", "mask"}) + self.assertNotIn("box", {item["id"] for item in spec["modules"]}) + self.assertNotIn("mask", {item["id"] for item in spec["modules"]}) + self.assertTrue(all(output_by_id[item_id]["role"] == "output head" for item_id in ("classification", "box", "mask"))) + self.assertEqual(set(plan["contract_completion_report"]["promoted_parallel_output_heads"]), {"box", "mask"}) + self.assertNotIn(("proposal", "mask"), {(item["source"], item["target"]) for item in spec["relations"]}) + self.assertIn("proposal->mask", plan["contract_completion_report"]["removed_branch_shortcuts"]) + def test_generic_completion_recovers_retrieval_conditioned_generation(self): caption = "Figure 1: Overview of our approach. We combine a pre-trained retriever (Query Encoder + Document Index) with a pre-trained seq2seq model (Generator). For query x, we use MIPS to find the top-K documents. For final prediction y, the generator produces the output sequence." result_caption = "Figure 2: Posterior for each generated token with five retrieved documents." From c36be06798dd0c179ad1094c2633458a10bfba99 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 01:14:12 +0800 Subject: [PATCH 64/74] Stabilize dense overview normalization --- README.md | 2 ++ docs/paper-to-image.md | 7 +++++++ rfs/paper_to_image/preparation.py | 6 +++--- tests/unit/test_paper_contract.py | 12 ++++++++++-- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 99abdef..ae33b49 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,8 @@ Production Image-2 review rejects a neat but empty text-only flowchart. The crit Branch contracts now promote evidence-backed classification, regression, mask, prediction, and similar parallel leaf heads to one consistent output role, remove ancestor shortcuts that bypass the shared branch point, and merge duplicate innovation labels into the existing node instead of drawing a second node. Feedback blueprints anchor nested feedback connectors at the actual inner artifact rather than the outer container. Visual enrichment is measured inside the blueprint's active figure region while retaining the same `0.08` threshold, so large white margins do not reject a genuinely illustrated figure and sparse box-only diagrams still fail. +Dense multi-frame overview normalization tolerates descriptive paper wording such as `heavyweight image encoder` and `lightweight mask decoder`. Those phrases still map to the canonical Figure 1 nodes `Image Encoder` and `Mask Decoder`, allowing the overview compiler to remove lower-level implementation details instead of misclassifying the whole contract as a generic branch graph. + Offline engineering validation: ```powershell diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 9db7048..77c11c0 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -222,6 +222,13 @@ For nested feedback layouts, the deterministic blueprint starts the feedback-to- refinement connector at the inner feedback artifact, not the outer container boundary. +Dense multi-frame overview matching uses scientific head terms rather than +requiring exact bare labels. Descriptive variants such as `heavyweight image +encoder`, `prompt encoder`, and `lightweight mask decoder` are normalized to +their canonical overview nodes before the 13-entity SAM-style contract is +rebuilt. This prevents detailed implementation entities from changing the +intended Figure 1 topology. + The same generic semantic plan can now bypass raster generation entirely: ```powershell diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 2c4260b..fc03963 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -739,9 +739,9 @@ def candidates(predicate: Callable[[str, dict[str, Any]], bool]) -> list[tuple[s data_engine = candidates(lambda label, item: label == "dataengine") image = candidates(lambda label, item: label in {"image", "images", "inputimage"} and item in spec.get("inputs", [])) prompt = candidates(lambda label, item: label in {"prompt", "inputprompts", "segmentationprompt", "sparseprompts", "denseprompts"} and item in spec.get("inputs", [])) - image_encoder = candidates(lambda label, item: label == "imageencoder") - prompt_encoder = candidates(lambda label, item: label == "promptencoder") - mask_decoder = candidates(lambda label, item: label == "maskdecoder") + image_encoder = candidates(lambda label, item: label.endswith("imageencoder") and "prompt" not in label) + prompt_encoder = candidates(lambda label, item: label.endswith("promptencoder")) + mask_decoder = candidates(lambda label, item: label.endswith("maskdecoder")) valid_mask = candidates(lambda label, item: label in {"validsegmentationmask", "validmask", "validmasks"}) assisted = candidates(lambda label, item: label == "assistedmanual") semi = candidates(lambda label, item: label == "semiautomatic") diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 936022b..ce1e961 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -949,10 +949,13 @@ def test_dense_multiframe_contract_keeps_task_model_and_data_engine_panels(self) {"id": "task", "name": "Prompt-able segmentation task", "role": "overview_panel", "evidence_ids": ["E0001"]}, {"id": "sam", "name": "Segmentation model (SAM)", "role": "overview_panel", "evidence_ids": ["E0001"]}, {"id": "engine", "name": "Data engine", "role": "overview_panel", "evidence_ids": ["E0001"]}, - {"id": "image_encoder", "name": "image encoder", "evidence_ids": ["E0001"]}, + {"id": "image_encoder", "name": "heavyweight image encoder", "evidence_ids": ["E0001"]}, {"id": "prompt_encoder", "name": "prompt encoder", "evidence_ids": ["E0001"]}, - {"id": "mask_decoder", "name": "mask decoder", "evidence_ids": ["E0001"]}, + {"id": "mask_decoder", "name": "lightweight mask decoder", "evidence_ids": ["E0001"]}, + {"id": "image_embeddings", "name": "Image Embeddings", "evidence_ids": ["E0001"]}, {"id": "transformer", "name": "Transformer Encoder", "evidence_ids": ["E0001"]}, + {"id": "position", "name": "Positional Encoding", "evidence_ids": ["E0001"]}, + {"id": "modality_group", "name": "Modality Encoders", "evidence_ids": ["E0001"]}, {"id": "assisted", "name": "Assisted-Manual", "evidence_ids": ["E0001"]}, {"id": "semi", "name": "Semi-Automatic", "evidence_ids": ["E0001"]}, {"id": "fully", "name": "Fully Automatic", "evidence_ids": ["E0001"]}, @@ -978,7 +981,12 @@ def test_dense_multiframe_contract_keeps_task_model_and_data_engine_panels(self) self.assertIn("promptablesegmentationtask", labels) self.assertIn("segmentanythingmodel", labels) self.assertIn("dataengine", labels) + self.assertIn("imageencoder", labels) + self.assertIn("maskdecoder", labels) + self.assertNotIn("imageembeddings", labels) self.assertNotIn("transformerencoder", labels) + self.assertNotIn("positionalencoding", labels) + self.assertNotIn("modalityencoders", labels) self.assertNotIn("confidencescores", labels) def test_overview_caption_and_stage_list_complete_missing_panels(self): From 0d9706ab869097995ad2dbefe1ad01f39ed6e2d5 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 02:21:53 +0800 Subject: [PATCH 65/74] Validate unseen paper contracts and PPT round trips --- README.md | 2 + rfs/cli.py | 2 + rfs/paper_to_image/editable_ppt.py | 39 +++- rfs/paper_to_image/planner.py | 12 +- rfs/paper_to_image/ppt_roundtrip.py | 123 ++++++++++++ rfs/paper_to_image/preparation.py | 246 ++++++++++++++++++++++- rfs/paper_to_image/semantic_blueprint.py | 7 + tests/integration/test_paper_to_image.py | 12 ++ tests/unit/test_paper_contract.py | 64 +++++- tests/unit/test_ppt_roundtrip.py | 38 ++++ tests/unit/test_semantic_blueprint.py | 22 ++ tests/unit/test_semantic_ppt.py | 21 ++ 12 files changed, 572 insertions(+), 16 deletions(-) create mode 100644 rfs/paper_to_image/ppt_roundtrip.py create mode 100644 tests/unit/test_ppt_roundtrip.py diff --git a/README.md b/README.md index ae33b49..aee3976 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ This writes `paper.md`, `document_model.json`, `extraction_report.json`, `sectio Add `--editable-ppt` to compile the same evidence-backed semantic contract directly into `editable_composition.pptx` without image generation. Every visible module is a native PowerPoint shape with editable text, every relation is a native connector, and `figure_program.json`, `semantic_ppt_report.json`, and `composition_quality_report.json` remain available for auditing. This direct route is intended for fast structural drafts; generated icons or richer visual assets can be added later without changing the scientific graph. +Add `--verify-ppt-roundtrip` when the editable draft must be checked end to end. It implies `--editable-ppt`, exports the deck through PowerPoint, and writes `ppt_roundtrip_report.json` with editable label/node/connector coverage, flattened-picture area, render status, and canvas-ratio drift. The default fast path leaves this off because launching PowerPoint usually adds roughly 10-20 seconds. + Use `rfs inspect-pdf` for parser-only diagnostics. Native PDF blocks are normalized into displayed-page coordinates on rotated pages, and the reading-order pass detects one-, two-, and three-column body layouts while keeping cross-column headings in sequence. The same column pass is applied to OCR lines; when a title fragment creates an invalid three-column hypothesis, detection retries a two-column layout before falling back to one column. Native extraction preserves font size and bold metadata so unnumbered or split-number headings such as `Model Architecture` and `Encoder and Decoder Stacks` still create correct section boundaries. Text quality uses Unicode letters and numbers, and section detection includes common English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean headings plus localized Figure/Table captions. Cross-parser gating compares supported lexical content rather than raw string order, using CJK character bigrams when whitespace segmentation differs, so normal two-column reordering, equation layout, hidden accessibility text, and native CJK PDFs do not trigger unnecessary OCR. `extraction_report.json` records section, typographic-heading, caption, column, rotation, OCR render DPI, and spacing-repair metrics so assumptions remain auditable. `--ocr-engine auto` prefers RapidOCR when installed, then EasyOCR/PaddleOCR. RapidOCR processes up to three selected pages concurrently at 84 DPI and filters short vertical margin artifacts. The base installation includes `wordninja` for conservative English spacing recovery, while requiring multiple English anchors so non-English Latin words are not split accidentally. Set `RFS_OCR_WORKERS` to override the page-worker count. OCR page selection prioritizes semantic pages and spreads the remaining budget across the document. Long fully scanned papers can continue as explicit `sampled_pages_only` engineering contracts when the OCR sample has high confidence and covers Abstract plus Method; these results always remain `production_ready: false`. Long-paper evidence budgets preserve page-wide coverage before selecting topology definitions and priority sections. OCR model downloads are disabled by default; set `RFS_OCR_ALLOW_DOWNLOAD=1` only for an explicit one-time EasyOCR download. `rfs doctor --json` reports OCR package, worker policy, spacing-repair support, and model readiness. Run a repeatable fast suite with provider, cache, recall, and timing aggregation: diff --git a/rfs/cli.py b/rfs/cli.py index f85f624..4e8dd20 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -164,6 +164,7 @@ def build_parser() -> argparse.ArgumentParser: fast_prompt.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") fast_prompt.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") fast_prompt.add_argument("--editable-ppt", action="store_true", help="Also compile the semantic contract into native editable PowerPoint shapes, text, and connectors.") + fast_prompt.add_argument("--verify-ppt-roundtrip", action="store_true", help="Compile the editable PPTX, render it through PowerPoint, and write ppt_roundtrip_report.json.") fast_prompt.add_argument("--json", action="store_true", help="Emit JSON.") make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") @@ -415,6 +416,7 @@ def main(argv: list[str] | None = None) -> int: language=args.language, domain_profile=args.domain_profile, editable_ppt=args.editable_ppt, + verify_ppt_roundtrip=args.verify_ppt_roundtrip, ) elif args.command == "make-framework": result = make_framework( diff --git a/rfs/paper_to_image/editable_ppt.py b/rfs/paper_to_image/editable_ppt.py index baac987..ea73697 100644 --- a/rfs/paper_to_image/editable_ppt.py +++ b/rfs/paper_to_image/editable_ppt.py @@ -42,20 +42,45 @@ def _ratio_value(value: str | float | int | None) -> float: def _font_size(label: str, bbox: dict[str, Any]) -> float: width = float(bbox.get("w") or 0.12) + longest_token = max((len(token) for token in label.replace("/", " ").replace("-", " ").split()), default=0) + if longest_token >= 13: + return 10.5 if len(label) >= 30 or width < 0.12: return 12.0 + if len(label) >= 24: + return 12.0 if len(label) >= 20: - return 14.0 + return 13.0 return 16.0 -def _relation_label_bbox(path: list[list[float]]) -> dict[str, float] | None: +def _relation_label_bbox(path: list[list[float]], label: str) -> dict[str, float] | None: if len(path) < 2: return None - point = path[len(path) // 2] - x = max(0.02, min(0.84, float(point[0]) - 0.08)) - y = max(0.02, min(0.93, float(point[1]) - 0.022)) - return {"x": round(x, 6), "y": round(y, 6), "w": 0.16, "h": 0.044} + segments: list[tuple[list[float], list[float], float]] = [] + total = 0.0 + for start, end in zip(path[:-1], path[1:]): + length = ((float(end[0]) - float(start[0])) ** 2 + (float(end[1]) - float(start[1])) ** 2) ** 0.5 + segments.append((start, end, length)) + total += length + if total <= 0: + return None + halfway = total / 2 + traversed = 0.0 + midpoint = path[0] + for start, end, length in segments: + if traversed + length >= halfway: + fraction = (halfway - traversed) / max(length, 1e-9) + midpoint = [ + float(start[0]) + (float(end[0]) - float(start[0])) * fraction, + float(start[1]) + (float(end[1]) - float(start[1])) * fraction, + ] + break + traversed += length + width = max(0.07, min(0.13, 0.052 + len(label) * 0.006)) + x = max(0.01, min(0.99 - width, float(midpoint[0]) - width / 2)) + y = max(0.01, min(0.95, float(midpoint[1]) - 0.052)) + return {"x": round(x, 6), "y": round(y, 6), "w": round(width, 6), "h": 0.038} def build_semantic_figure_program( @@ -119,7 +144,7 @@ def build_semantic_figure_program( **style, }) relation_label = str(connector.get("label") or "").strip() - label_bbox = _relation_label_bbox(connector.get("path_percent", [])) if relation_label else None + label_bbox = _relation_label_bbox(connector.get("path_percent", []), relation_label) if relation_label else None if relation_label and label_bbox: labels.append({ "id": f"label_{connector['id']}", diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index b277a94..b3d57aa 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -716,6 +716,14 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = "inference_flow": spec.get("inference_flow", []), "forbidden_inventions": spec.get("forbidden_inventions", []), "uncertainties": spec.get("uncertainties", []), + "supporting_details": [ + { + "label": str(item.get("name") or item.get("label") or item.get("statement") or ""), + "role": item.get("role"), + "visibility": item.get("visibility"), + } + for item in spec.get("supporting_details", []) if isinstance(item, dict) + ], } compact_style = { "medium": style.get("medium"), @@ -796,6 +804,8 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Mandatory entity role and containment checklist: {json.dumps(compact_roles, ensure_ascii=False, separators=(',', ':'))} +Panel-local supporting details in the compact contract are evidence-backed visual cues only. Use them as unlabeled internal texture or mechanism detail when helpful; do not promote them to peer top-level nodes, do not add them to the visible label whitelist, and do not invent connectors for them. + Preserve these semantic roles in the visible composition. A data source, modality, processing module, innovation, and output are not interchangeable even when all labels are present. In particular, do not flatten a source -> modality -> method chain into peer inputs or place a dataset beside modalities as though it were another sensor type. Topology-specific rendering rules: @@ -891,7 +901,7 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * label_cjk, label_latin = script_counts(value) source_cjk, source_latin = script_counts(source) changed_from_cjk = source_cjk >= 2 and label_cjk == 0 and label_latin >= 2 - changed_from_latin = source_latin >= 4 and source_cjk == 0 and label_cjk >= 2 and label_latin == 0 + changed_from_latin = source_latin >= 4 and source_cjk == 0 and label_cjk >= 1 if changed_from_cjk or changed_from_latin: errors.append(f"{location} visible label changes the source writing system and is not verbatim evidence: {value}") return diff --git a/rfs/paper_to_image/ppt_roundtrip.py b/rfs/paper_to_image/ppt_roundtrip.py new file mode 100644 index 0000000..996c8ea --- /dev/null +++ b/rfs/paper_to_image/ppt_roundtrip.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import zipfile +from pathlib import Path +from xml.etree import ElementTree + +from PIL import Image + +from ..exporter import export_outputs +from ..utils import write_json + + +_NS = { + "a": "http://schemas.openxmlformats.org/drawingml/2006/main", + "p": "http://schemas.openxmlformats.org/presentationml/2006/main", +} + + +def _slide_tree(pptx_path: Path) -> ElementTree.Element: + with zipfile.ZipFile(pptx_path) as archive: + return ElementTree.fromstring(archive.read("ppt/slides/slide1.xml")) + + +def _shape_names(root: ElementTree.Element) -> list[str]: + return [str(item.get("name") or "") for item in root.findall(".//p:cNvPr", _NS)] + + +def _visible_text(root: ElementTree.Element) -> str: + return "\n".join(str(item.text or "") for item in root.findall(".//a:t", _NS)) + + +def _flattened_picture_area_ratio(root: ElementTree.Element, slide_width_emu: int, slide_height_emu: int) -> float: + area = 0 + for picture in root.findall(".//p:pic", _NS): + ext = picture.find(".//a:xfrm/a:ext", _NS) + if ext is None: + continue + try: + area += max(0, int(ext.get("cx") or 0)) * max(0, int(ext.get("cy") or 0)) + except (TypeError, ValueError): + continue + slide_area = max(1, int(slide_width_emu) * int(slide_height_emu)) + return round(min(1.0, area / slide_area), 6) + + +def evaluate_ppt_roundtrip( + program: dict, + pptx: str | Path, + out: str | Path, + *, + render: bool = True, +) -> dict: + pptx_path = Path(pptx) + root_dir = Path(out) + root_dir.mkdir(parents=True, exist_ok=True) + slide = _slide_tree(pptx_path) + names = _shape_names(slide) + text = _visible_text(slide) + + expected_nodes = [item for item in program.get("semantic_nodes", []) if isinstance(item, dict) and item.get("id")] + expected_arrows = [item for item in program.get("arrows", []) if isinstance(item, dict) and item.get("id")] + expected_labels = [str(item.get("label") or "").strip() for item in expected_nodes if str(item.get("label") or "").strip()] + present_labels = [label for label in expected_labels if label in text] + present_nodes = [item for item in expected_nodes if any(name.startswith(f"RFS Semantic Node {item['id']}") for name in names)] + present_arrows = [item for item in expected_arrows if any(name.startswith(f"RFS Connector {item['id']} ") for name in names)] + + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + width_in = float(canvas.get("width_in") or 13.333) + height_in = float(canvas.get("height_in") or 7.5) + emu_per_inch = 914400 + flattened_ratio = _flattened_picture_area_ratio( + slide, + round(width_in * emu_per_inch), + round(height_in * emu_per_inch), + ) + + export = {"pdf": None, "png": None, "status": "not_requested"} + rendered_ratio_error = None + rendered_size = None + if render: + export = export_outputs(pptx_path, root_dir) + png_path = Path(export["png"]) if export.get("png") else None + if png_path and png_path.exists(): + with Image.open(png_path) as image: + rendered_size = {"width": image.width, "height": image.height} + expected_ratio = width_in / max(height_in, 0.001) + actual_ratio = image.width / max(image.height, 1) + rendered_ratio_error = round(abs(actual_ratio - expected_ratio) / expected_ratio, 6) + + label_coverage = len(present_labels) / max(1, len(expected_labels)) + node_coverage = len(present_nodes) / max(1, len(expected_nodes)) + connector_coverage = len(present_arrows) / max(1, len(expected_arrows)) + structure_ok = label_coverage == 1.0 and node_coverage == 1.0 and connector_coverage == 1.0 and flattened_ratio <= 0.35 + render_ok = not render or (export.get("status") == "ok" and rendered_ratio_error is not None and rendered_ratio_error <= 0.01) + report = { + "summary": "Editable PowerPoint round-trip report comparing the semantic program, native PPT object tree, and optional PowerPoint render.", + "ok": bool(structure_ok and render_ok), + "pptx": str(pptx_path), + "structure": { + "expected_node_count": len(expected_nodes), + "native_editable_node_count": len(present_nodes), + "node_editability_ratio": round(node_coverage, 6), + "expected_connector_count": len(expected_arrows), + "native_editable_connector_count": len(present_arrows), + "connector_editability_ratio": round(connector_coverage, 6), + "expected_label_count": len(expected_labels), + "editable_label_count": len(present_labels), + "editable_label_ratio": round(label_coverage, 6), + "missing_labels": [label for label in expected_labels if label not in present_labels], + "missing_node_ids": [str(item["id"]) for item in expected_nodes if item not in present_nodes], + "missing_connector_ids": [str(item["id"]) for item in expected_arrows if item not in present_arrows], + "flattened_picture_area_ratio": flattened_ratio, + "max_flattened_picture_area_ratio": 0.35, + }, + "render": { + **export, + "rendered_size": rendered_size, + "canvas_ratio_relative_error": rendered_ratio_error, + "max_canvas_ratio_relative_error": 0.01, + }, + } + write_json(root_dir / "ppt_roundtrip_report.json", report) + return report diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index fc03963..5a882ee 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -150,6 +150,40 @@ def _repair_cross_script_terminology(spec: dict[str, Any], parsed: dict[str, Any spec["terminology"] = repaired +def _repair_cross_script_entity_labels(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + evidence_by_id = { + str(item.get("id")): str(item.get("text") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + repaired: list[str] = [] + for field in ("inputs", "modules", "outputs", "innovations"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + label = _item_label(item) + label_cjk, label_latin = _script_counts(label) + records = [evidence_by_id[str(value)] for value in item.get("evidence_ids", []) if str(value) in evidence_by_id] + source = " ".join(records) + source_cjk, source_latin = _script_counts(source) + if not (label_cjk >= 1 and label_latin >= 2 and source_cjk == 0 and source_latin >= 4): + continue + tokens = re.findall(r"[A-Za-z]+|\d+(?:\.\d+)?", label) + if len(tokens) < 2: + continue + pattern = r"\b" + r"[^A-Za-z0-9]+".join(re.escape(token) for token in tokens) + r"\b" + match = re.search(pattern, source, re.IGNORECASE) + if not match: + continue + exact = re.sub(r"\s+", " ", match.group(0)).strip(" .,:;") + if not exact: + continue + item["name"] = exact + item.pop("visible_label", None) + repaired.append(f"{item.get('id')}:{label}->{exact}") + return repaired + + def _ground_exact_visible_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict) and item.get("id") and item.get("text")] novelty_pattern = re.compile(r"\b(?:we\s+(?:introduce|propose|present|develop)|novel|new\s+(?:method|model|system|framework)|contribution)\b|\u672c\u6587\u63d0\u51fa|\u6211\u4eec\u63d0\u51fa|\u9996\u6b21|\u521b\u65b0", re.IGNORECASE) @@ -328,6 +362,171 @@ def _infer_topology(spec: dict[str, Any]) -> str: return "linear" if relations else "unknown" +def _detect_dense_overview_panels(parsed: dict[str, Any]) -> dict[str, Any]: + figures = parsed.get("document_index", {}).get("figures", []) + overview_pattern = re.compile(r"\b(?:overview|architecture|framework|pipeline|model)\b", re.IGNORECASE) + figure = next( + ( + item + for item in figures + if isinstance(item, dict) + and re.match(r"^(?:figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) + and overview_pattern.search(str(item.get("caption") or "")) + ), + None, + ) + if not figure: + return {"detected": False, "page": None, "panel_labels": [], "panel_count": 0} + + page_number = int(figure.get("page") or 0) + page = next( + ( + item + for item in parsed.get("pages", []) + if isinstance(item, dict) and int(item.get("page") or 0) == page_number + ), + None, + ) + if not page: + return {"detected": False, "page": page_number, "panel_labels": [], "panel_count": 0} + + labels: set[str] = set() + for block in page.get("blocks", []): + if not isinstance(block, dict): + continue + text = re.sub(r"\s+", " ", str(block.get("text") or "")).strip() + marker = re.fullmatch(r"([a-h])", text, re.IGNORECASE) + description = re.match(r"^([a-h])\s*[,.:]\s+", text, re.IGNORECASE) + if marker: + labels.add(marker.group(1).casefold()) + if description: + labels.add(description.group(1).casefold()) + caption = re.sub(r"\s+", " ", str(figure.get("caption") or "")) + labels.update(match.group(1).casefold() for match in re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s+", caption, re.IGNORECASE)) + ordered = sorted(labels) + return { + "detected": len(ordered) >= 3, + "page": page_number, + "panel_labels": ordered, + "panel_count": len(ordered), + } + + +def _merge_isolated_architecture_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + modules = [item for item in spec.get("modules", []) if isinstance(item, dict)] + relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] + connected_ids = { + str(value) + for relation in relations + for value in (relation.get("source"), relation.get("target")) + if str(value or "").strip() + } + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + overview_pages = { + int(item.get("page") or 0) + for item in parsed.get("document_index", {}).get("figures", []) + if isinstance(item, dict) and re.search(r"\b(?:overview|architecture|framework|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE) + } + removed_ids: set[str] = set() + merged: list[str] = [] + for isolated in modules: + isolated_id = str(isolated.get("id") or "") + isolated_label = _item_label(isolated) + isolated_normalized = _normalized_label(isolated_label) + if not isolated_id or isolated_id in connected_ids or len(isolated_normalized) < 3: + continue + for target in modules: + target_id = str(target.get("id") or "") + target_label = _item_label(target) + target_normalized = _normalized_label(target_label) + if not target_id or target_id not in connected_ids or target_id == isolated_id or len(target_normalized) < 3: + continue + shared_ids = set(str(value) for value in isolated.get("evidence_ids", []) if value) & set(str(value) for value in target.get("evidence_ids", []) if value) + records = [evidence_by_id[value] for value in shared_ids if value in evidence_by_id] + relation_record = next( + ( + record + for record in records + if isolated_normalized in _normalized_label(str(record.get("text") or "")) + and target_normalized in _normalized_label(str(record.get("text") or "")) + and re.search(r"\b(?:based on|built on|using|uses|powered by|adapts?)\b", str(record.get("text") or ""), re.IGNORECASE) + ), + None, + ) + if not relation_record: + continue + exact_label_candidates = [ + record + for record in parsed.get("evidence", []) + if isinstance(record, dict) + and int(record.get("page") or 0) in overview_pages + and 3 <= len(str(record.get("text") or "").strip()) <= 90 + and isolated_normalized in _normalized_label(str(record.get("text") or "")) + and target_normalized in _normalized_label(str(record.get("text") or "")) + ] + exact_label_record = min( + exact_label_candidates, + key=lambda record: len(str(record.get("text") or "").strip()), + default=None, + ) + if exact_label_record: + target["name"] = re.sub(r"\s+", " ", str(exact_label_record.get("text") or "")).strip(" .") + target["evidence_ids"] = list(dict.fromkeys([*target.get("evidence_ids", []), str(exact_label_record.get("id"))])) + else: + target["name"] = f"{target_label} ({isolated_label})" + target["evidence_ids"] = list(dict.fromkeys([*target.get("evidence_ids", []), *isolated.get("evidence_ids", [])])) + removed_ids.add(isolated_id) + merged.append(f"{isolated_id}->{target_id}") + break + if removed_ids: + spec["modules"] = [item for item in modules if str(item.get("id") or "") not in removed_ids] + for relation in relations: + if str(relation.get("source") or "") in removed_ids: + relation["source"] = next(value.split("->", 1)[1] for value in merged if value.startswith(f"{relation.get('source')}->")) + if str(relation.get("target") or "") in removed_ids: + relation["target"] = next(value.split("->", 1)[1] for value in merged if value.startswith(f"{relation.get('target')}->")) + spec["relations"] = relations + return merged + + +def _demote_unconnected_completion_entities(spec: dict[str, Any], completion_report: dict[str, Any]) -> list[str]: + added_ids = {str(value) for value in completion_report.get("added_entities", []) if str(value)} + if not added_ids: + return [] + connected_ids = { + str(value) + for relation in spec.get("relations", []) if isinstance(relation, dict) + for value in (relation.get("source"), relation.get("target")) + if str(value or "").strip() + } + demoted: list[str] = [] + supporting = [item for item in spec.get("supporting_details", []) if isinstance(item, dict)] + for field in ("inputs", "modules", "outputs", "innovations"): + kept = [] + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "") + if item_id in added_ids and item_id not in connected_ids: + supporting.append({ + **item, + "original_field": field, + "original_role": item.get("role"), + "role": "panel_local_supporting_detail", + "visibility": "unlabeled_visual_detail_not_global_node", + }) + demoted.append(item_id) + continue + kept.append(item) + spec[field] = kept + spec["supporting_details"] = supporting + return demoted + + def _complete_from_overview_caption(spec: dict[str, Any], parsed: dict[str, Any]) -> None: figures = parsed.get("document_index", {}).get("figures", []) figure = next((item for item in figures if re.match(r"^(figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("caption") or ""), re.IGNORECASE)), None) @@ -1623,18 +1822,20 @@ def choose_input_target(label: str) -> dict[str, Any] | None: outgoing.add(source) incoming.add(pair[1]) module_sources = {str(item.get("source")) for item in relations if isinstance(item, dict)} - sink_modules = [item for item in modules if str(item.get("id")) not in module_sources] + completion_added_ids = {str(value) for value in completion_report.get("added_entities", []) if str(value)} + eligible_output_modules = [item for item in modules if str(item.get("id") or "") not in completion_added_ids] + sink_modules = [item for item in eligible_output_modules if str(item.get("id")) not in module_sources] for item in spec.get("outputs", []) if isinstance(spec.get("outputs"), list) else []: target_id = str(item.get("id")) if not target_id or target_id in incoming: continue output_label = _item_label(item).casefold() - source = next((module for module in reversed(modules) if any(term in _item_label(module).casefold() for term in ("head", "decoder", "refine", "classifier", "predict"))), None) + source = next((module for module in reversed(eligible_output_modules) if any(term in _item_label(module).casefold() for term in ("head", "decoder", "refine", "classifier", "predict"))), None) if "mask" in output_label: - source = next((module for module in modules if "mask decoder" in _item_label(module).casefold() or "mask branch" in _item_label(module).casefold()), source) + source = next((module for module in eligible_output_modules if "mask decoder" in _item_label(module).casefold() or "mask branch" in _item_label(module).casefold()), source) elif "box" in output_label or "bounding" in output_label or "class" in output_label or "category" in output_label: - source = next((module for module in modules if "box branch" in _item_label(module).casefold() or "classification head" in _item_label(module).casefold()), source) - source = source or (sink_modules[-1] if sink_modules else modules[-1] if modules else None) + source = next((module for module in eligible_output_modules if "box branch" in _item_label(module).casefold() or "classification head" in _item_label(module).casefold()), source) + source = source or (sink_modules[-1] if sink_modules else eligible_output_modules[-1] if eligible_output_modules else None) if source: pair = (str(source.get("id")), target_id) if pair not in existing_pairs: @@ -1642,6 +1843,8 @@ def choose_input_target(label: str) -> dict[str, Any] | None: existing_pairs.add(pair) incoming.add(target_id) spec["relations"] = relations + completion_report["merged_architecture_entities"] = _merge_isolated_architecture_entities(spec, parsed) + completion_report["demoted_unconnected_completion_entities"] = _demote_unconnected_completion_entities(spec, completion_report) feedback_report = _simplify_feedback_contract(spec) completion_report["feedback_simplification_applied"] = feedback_report["applied"] completion_report["feedback_removed_entities"] = feedback_report["removed_entities"] @@ -1663,13 +1866,18 @@ def choose_input_target(label: str) -> dict[str, Any] | None: spec.setdefault("feedback_loops", [item for item in spec.get("relations", []) if isinstance(item, dict) and "feedback" in str(item.get("type") or "").casefold()]) allowed_topologies = {"linear", "branch", "feedback", "multimodal", "dense_multiframe", "unknown"} inferred_topology = _infer_topology(spec) - if str(spec.get("topology") or "") not in allowed_topologies or inferred_topology in {"feedback", "multimodal"}: + dense_overview = _detect_dense_overview_panels(parsed) + completion_report["dense_overview_detection"] = dense_overview + if dense_overview["detected"] and inferred_topology not in {"feedback", "multimodal"}: + inferred_topology = "dense_multiframe" + if str(spec.get("topology") or "") not in allowed_topologies or inferred_topology in {"feedback", "multimodal", "dense_multiframe"}: spec["topology"] = inferred_topology completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) completion_report["removed_out_of_scope_entities"] = _prune_out_of_scope_entities(spec, parsed, completion_report) completion_report["removed_ungrounded_innovations"] = _remove_ungrounded_innovations(spec) completion_report["removed_input_shortcuts"] = _remove_input_shortcuts_through_intermediates(spec) _repair_cross_script_terminology(spec, parsed) + completion_report["repaired_cross_script_entity_labels"] = _repair_cross_script_entity_labels(spec, parsed) completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) labels = [] normalized_labels: set[str] = set() @@ -1911,7 +2119,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, domain_profile: str) -> Path: - signature = json.dumps({"version": 30, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "domain_profile": domain_profile}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 32, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "domain_profile": domain_profile}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" @@ -2199,6 +2407,8 @@ def prepare_paper_figure_contract( def run_fast_framework_prompt(**kwargs: Any) -> dict[str, Any]: editable_ppt = bool(kwargs.pop("editable_ppt", False)) + verify_ppt_roundtrip = bool(kwargs.pop("verify_ppt_roundtrip", False)) + editable_ppt = editable_ppt or verify_ppt_roundtrip if not kwargs.get("planner_model"): kwargs["planner_model"] = os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" kwargs["fast_mode"] = True @@ -2227,6 +2437,28 @@ def run_fast_framework_prompt(**kwargs: Any) -> dict[str, Any]: prepared["stage_timings"]["ppt_compile_seconds"] = ppt_seconds prepared["stage_timings"]["total_seconds"] = prepared["elapsed_seconds"] prepared["artifacts"] = [*prepared.get("artifacts", []), "figure_program.json", "editable_composition.pptx", "semantic_ppt_report.json", "composition_quality_report.json"] + if verify_ppt_roundtrip and ppt_result.get("ok"): + from .ppt_roundtrip import evaluate_ppt_roundtrip + + roundtrip_started = time.monotonic() + roundtrip = evaluate_ppt_roundtrip( + read_json(Path(prepared["root"]) / "figure_program.json"), + str(ppt_result["pptx"]), + prepared["root"], + render=True, + ) + roundtrip_seconds = round(time.monotonic() - roundtrip_started, 3) + prepared["ppt_roundtrip"] = roundtrip + prepared["ppt_roundtrip_seconds"] = roundtrip_seconds + prepared["ok"] = bool(prepared.get("ok") and roundtrip.get("ok")) + prepared["production_ready"] = bool(prepared.get("production_ready") and roundtrip.get("ok")) + if not roundtrip.get("ok"): + prepared["status"] = "completed_with_warnings" + prepared["elapsed_seconds"] = round(float(prepared.get("elapsed_seconds") or 0.0) + roundtrip_seconds, 3) + if isinstance(prepared.get("stage_timings"), dict): + prepared["stage_timings"]["ppt_roundtrip_seconds"] = roundtrip_seconds + prepared["stage_timings"]["total_seconds"] = prepared["elapsed_seconds"] + prepared["artifacts"] = [*prepared.get("artifacts", []), "ppt_roundtrip_report.json", "review.pdf", "final_600dpi.png"] write_json(Path(prepared["root"]) / "run_report.json", {key: value for key, value in prepared.items() if key not in {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"}}) internal = {"root", "parsed", "preferences", "paper_review", "review_metadata", "selected_domain", "plan", "planner_metadata", "archived_positive", "archived_negative", "planning_validation"} return {key: value for key, value in prepared.items() if key not in internal} diff --git a/rfs/paper_to_image/semantic_blueprint.py b/rfs/paper_to_image/semantic_blueprint.py index 3ec1b88..d0bc2ff 100644 --- a/rfs/paper_to_image/semantic_blueprint.py +++ b/rfs/paper_to_image/semantic_blueprint.py @@ -256,6 +256,12 @@ def compile_semantic_blueprint( if not relations: return {"summary": "Generic semantic blueprint requires at least one declared relation.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "no_relations"} + connected_ids = {value for item in relations for value in (item["source"], item["target"])} + isolated_node_ids = [item["id"] for item in nodes if item["id"] not in connected_ids] + connected_nodes = [item for item in nodes if item["id"] in connected_ids] + if len(connected_nodes) >= 2: + nodes = connected_nodes + ranks = _rank_nodes(nodes, relations) positioned = _layout_nodes(nodes, ranks) by_id = {item["id"]: item for item in positioned} @@ -291,6 +297,7 @@ def compile_semantic_blueprint( "topology": topology, "node_count": len(positioned), "connector_count": len(connectors), + "isolated_node_ids": isolated_node_ids, "nodes": positioned, "connectors": connectors, "source": "figure_specification", diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index c28d747..74db08e 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -870,6 +870,18 @@ def test_fast_framework_cli_defaults(self): self.assertEqual(args.command, "fast-framework-prompt") self.assertEqual(args.deadline, 180) self.assertEqual(args.planner_mode, "vlm") + self.assertFalse(args.verify_ppt_roundtrip) + + def test_fast_framework_cli_accepts_ppt_roundtrip_verification(self): + parser = build_parser() + args = parser.parse_args([ + "fast-framework-prompt", + "--paper", "paper.pdf", + "--out", "output/fast", + "--verify-ppt-roundtrip", + ]) + + self.assertTrue(args.verify_ppt_roundtrip) def test_benchmark_fast_cli_defaults(self): parser = build_parser() diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index ce1e961..27446d2 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -2,11 +2,73 @@ import unittest from rfs.paper_to_image.planner import validate_plan_grounding -from rfs.paper_to_image.preparation import _fast_cache_path, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract +from rfs.paper_to_image.preparation import _demote_unconnected_completion_entities, _detect_dense_overview_panels, _fast_cache_path, _merge_isolated_architecture_entities, _repair_cross_script_entity_labels, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract from rfs.paper_to_image.contract_completion import _overview_candidates class PaperContractTests(unittest.TestCase): + def test_dense_overview_panels_and_local_details_are_not_flattened_into_peer_nodes(self): + parsed = { + "document_index": { + "figures": [{"page": 3, "caption": "Fig. 1 | Overview of Example. a, Model architecture."}], + }, + "pages": [{ + "page": 3, + "blocks": [ + {"text": "a"}, + {"text": "b"}, + {"text": "c"}, + {"text": "b, Tile-level pretraining."}, + {"text": "c, Slide-level pretraining."}, + ], + }], + "evidence": [ + {"id": "E_MAIN", "page": 3, "text": "Slide Encoder based on the LongNet architecture"}, + {"id": "E_EXACT", "page": 3, "text": "Slide Encoder (LongNet)"}, + {"id": "E_DETAIL", "page": 3, "text": "Classification token"}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E_MAIN"]}], + "modules": [ + {"id": "encoder", "name": "Slide Encoder", "evidence_ids": ["E_MAIN"]}, + {"id": "longnet", "name": "LongNet", "evidence_ids": ["E_MAIN"]}, + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_DETAIL"]}, + ], + "outputs": [{"id": "output", "name": "Output", "evidence_ids": ["E_MAIN"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "evidence_ids": ["E_MAIN"]}, + {"source": "encoder", "target": "output", "type": "data_flow", "evidence_ids": ["E_MAIN"]}, + ], + } + completion = {"added_entities": ["class_token"]} + + dense = _detect_dense_overview_panels(parsed) + merged = _merge_isolated_architecture_entities(spec, parsed) + demoted = _demote_unconnected_completion_entities(spec, completion) + + self.assertTrue(dense["detected"]) + self.assertEqual(dense["panel_labels"], ["a", "b", "c"]) + self.assertEqual(merged, ["longnet->encoder"]) + self.assertEqual(spec["modules"][0]["name"], "Slide Encoder (LongNet)") + self.assertEqual(demoted, ["class_token"]) + self.assertEqual(spec["supporting_details"][0]["visibility"], "unlabeled_visual_detail_not_global_node") + + def test_cross_script_symbol_hallucination_is_repaired_from_exact_evidence_span(self): + spec = { + "inputs": [{"id": "tiles", "name": "256 脳 256 image tiles", "evidence_ids": ["E1"]}], + "modules": [], + "outputs": [], + "innovations": [], + } + parsed = {"evidence": [{"id": "E1", "text": "serializes each WSI into a sequence of 256 × 256 image tiles in row-major order"}]} + + repaired = _repair_cross_script_entity_labels(spec, parsed) + + self.assertEqual(spec["inputs"][0]["name"], "256 × 256 image tiles") + self.assertEqual(repaired, ["tiles:256 脳 256 image tiles->256 × 256 image tiles"]) + def test_overview_selection_rejects_late_scaling_figure_when_figure_one_is_a_true_model_overview(self): parsed = { "document_index": { diff --git a/tests/unit/test_ppt_roundtrip.py b/tests/unit/test_ppt_roundtrip.py new file mode 100644 index 0000000..23c4b86 --- /dev/null +++ b/tests/unit/test_ppt_roundtrip.py @@ -0,0 +1,38 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.paper_to_image.editable_ppt import compile_semantic_ppt +from rfs.paper_to_image.ppt_roundtrip import evaluate_ppt_roundtrip +from rfs.utils import read_json + + +class PptRoundtripTests(unittest.TestCase): + def test_semantic_ppt_object_tree_preserves_labels_nodes_and_connectors(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Encoder", "Output"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + compiled = compile_semantic_ppt(specification, root) + program = read_json(root / "figure_program.json") + + report = evaluate_ppt_roundtrip(program, compiled["pptx"], root, render=False) + + self.assertTrue(report["ok"]) + self.assertEqual(report["structure"]["editable_label_ratio"], 1.0) + self.assertEqual(report["structure"]["node_editability_ratio"], 1.0) + self.assertEqual(report["structure"]["connector_editability_ratio"], 1.0) + self.assertEqual(report["structure"]["flattened_picture_area_ratio"], 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_semantic_blueprint.py b/tests/unit/test_semantic_blueprint.py index e346108..d2a56da 100644 --- a/tests/unit/test_semantic_blueprint.py +++ b/tests/unit/test_semantic_blueprint.py @@ -4,6 +4,28 @@ class SemanticBlueprintTests(unittest.TestCase): + def test_isolated_required_nodes_are_not_rendered_as_peer_graph_nodes(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Encoder", "Output", "Local Detail"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [ + {"id": "encoder", "name": "Encoder"}, + {"id": "detail", "name": "Local Detail"}, + ], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + + report = compile_semantic_blueprint(specification) + + self.assertTrue(report["applied"]) + self.assertEqual(report["isolated_node_ids"], ["detail"]) + self.assertNotIn("detail", {item["id"] for item in report["nodes"]}) + def test_linear_contract_uses_graph_ranks_and_keeps_parallel_outputs_in_one_layer(self): specification = { "topology": "linear", diff --git a/tests/unit/test_semantic_ppt.py b/tests/unit/test_semantic_ppt.py index 1d8c5b0..70bf75a 100644 --- a/tests/unit/test_semantic_ppt.py +++ b/tests/unit/test_semantic_ppt.py @@ -66,6 +66,27 @@ def test_feedback_contract_uses_external_dashed_editable_loop(self): self.assertEqual(loop["route_style"], "outer_feedback") self.assertEqual(program["labels"][0]["text"], "revise") + def test_long_words_and_relation_labels_are_compiled_for_readable_rendering(self): + specification = { + "topology": "linear", + "required_labels": ["Input", "Encoder", "contextualized embeddings"], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "output", "name": "contextualized embeddings"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow", "label": "serialize"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + + program = build_semantic_figure_program(specification) + output = next(item for item in program["semantic_nodes"] if item["id"] == "output") + relation_label = program["labels"][0] + + self.assertLessEqual(output["font_size_pt"], 10.5) + self.assertLess(relation_label["bbox_percent"]["w"], 0.13) + self.assertLess(relation_label["bbox_percent"]["y"], 0.5) + if __name__ == "__main__": unittest.main() From 58cd7fc1d02b830e5f8449a498c59c626e75c132 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 03:31:44 +0800 Subject: [PATCH 66/74] Stabilize dense paper figures and editable resume --- rfs/contracts/semantic.py | 118 +++++++-- rfs/editable_rebuild.py | 10 +- rfs/paper_to_image/critics.py | 15 +- rfs/paper_to_image/planner.py | 61 ++++- rfs/paper_to_image/preparation.py | 277 ++++++++++++++++++++- rfs/paper_to_image/semantic_blueprint.py | 117 ++++++++- rfs/paper_to_image/templates.py | 45 +++- tests/integration/test_paper_to_image.py | 69 +++++ tests/integration/test_rebuild_editable.py | 31 +++ tests/unit/test_paper_contract.py | 68 ++++- tests/unit/test_semantic_contract.py | 50 ++++ 11 files changed, 823 insertions(+), 38 deletions(-) diff --git a/rfs/contracts/semantic.py b/rfs/contracts/semantic.py index fbd973e..cbd6894 100644 --- a/rfs/contracts/semantic.py +++ b/rfs/contracts/semantic.py @@ -31,6 +31,11 @@ def _text(item: dict, fallback: str = "") -> str: def _entities(contract: dict) -> list[dict]: spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + required_labels = { + re.sub(r"[^a-z0-9]+", "", str(value).casefold()) + for value in spec.get("required_labels", []) + if str(value).strip() + } entities: list[dict] = [] for field, role in (("inputs", "input"), ("modules", "module"), ("outputs", "output"), ("innovations", "innovation")): values = spec.get(field) if isinstance(spec.get(field), list) else [] @@ -40,6 +45,11 @@ def _entities(contract: dict) -> list[dict]: label = _text(raw) if not label: continue + if role == "innovation": + visible_required = bool(raw.get("visible_label_required") or raw.get("must_appear_in_figure")) + normalized_label = re.sub(r"[^a-z0-9]+", "", label.casefold()) + if not visible_required and normalized_label not in required_labels: + continue entity_id = str(raw.get("id") or raw.get("name") or f"{role}_{index:02d}").strip() entities.append({ "id": entity_id, @@ -65,14 +75,35 @@ def _center(obj: dict) -> tuple[float, float]: def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dict], list[dict]]: - objects = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] - if len(objects) < len(entities): - seen = {str(item.get("id")) for item in objects} - objects.extend( - item for item in program.get("cards", []) + program.get("panels", []) - if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and str(item.get("id")) not in seen - ) - objects.sort(key=lambda item: (_center(item)[0], _center(item)[1], str(item.get("id") or ""))) + slots = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + cards = [item for item in program.get("cards", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + panels = [item for item in program.get("panels", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + objects = [*cards, *slots, *panels] + seen: set[str] = set() + objects = [item for item in objects if not (str(item.get("id")) in seen or seen.add(str(item.get("id"))))] + text_items = [ + item for item in (program.get("text_program", {}).get("items", []) if isinstance(program.get("text_program"), dict) else []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) + and not str(item.get("id") or "").startswith("semantic_text_") + and not item.get("semantic_entity_id") + ] + + def object_search_text(obj: dict) -> str: + values = [_object_text(obj)] + box = obj.get("bbox_percent") or {} + for text_item in text_items: + if str(text_item.get("target_id") or "") == str(obj.get("id") or "") or _inside(text_item.get("bbox_percent") or {}, box): + values.append(str(text_item.get("text") or "")) + return " ".join(values) + + def object_priority(obj: dict) -> int: + if obj in cards: + return 2 + if obj in slots: + return 1 + return 0 + + objects.sort(key=lambda item: (_center(item)[1], _center(item)[0], str(item.get("id") or ""))) available = list(objects) mapping: dict[str, dict] = {} records: list[dict] = [] @@ -80,14 +111,19 @@ def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dic entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) scored = [] for obj in available: - obj_tokens = _tokens(_object_text(obj)) + searchable = object_search_text(obj) + obj_tokens = _tokens(searchable) score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) - scored.append((score, -_center(obj)[0], obj)) + entity_compact = re.sub(r"[^a-z0-9]+", "", str(entity.get("label") or "").casefold()) + object_compact = re.sub(r"[^a-z0-9]+", "", searchable.casefold()) + if entity_compact and entity_compact in object_compact: + score += 1.0 + scored.append((score, object_priority(obj), -_center(obj)[1], -_center(obj)[0], obj)) if not scored: records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) continue - scored.sort(key=lambda value: (value[0], value[1]), reverse=True) - chosen = scored[0][2] if scored[0][0] > 0 else available[0] + scored.sort(key=lambda value: (value[0], value[1], value[2], value[3]), reverse=True) + chosen = scored[0][4] if scored[0][0] > 0 else available[0] available.remove(chosen) mapping[entity["id"]] = chosen records.append({ @@ -96,7 +132,7 @@ def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dic "role": entity["role"], "object_id": chosen.get("id"), "match_score": round(max(0.0, scored[0][0]), 4), - "status": "mapped", + "status": "mapped" if scored[0][0] > 0 else "mapped_spatial_fallback", }) return mapping, records @@ -136,13 +172,56 @@ def _anchor(source: dict, target: dict) -> list[list[float]]: return clean +def _preserve_local_reference_arrows(program: dict, mapping: dict[str, dict]) -> tuple[list[dict], list[str]]: + mapped_objects = list(mapping.values()) + all_objects = { + str(item.get("id")): item + for field in ("slots", "cards", "panels") + for item in program.get(field, []) + if isinstance(item, dict) and item.get("id") and isinstance(item.get("bbox_percent"), dict) + } + + def mapped_container(obj: dict | None) -> str | None: + if not obj: + return None + center_box = obj.get("bbox_percent") or {} + for mapped in mapped_objects: + if str(mapped.get("id")) == str(obj.get("id")) or _inside(center_box, mapped.get("bbox_percent") or {}): + return str(mapped.get("id")) + return None + + preserved: list[dict] = [] + replaced: list[str] = [] + for arrow in program.get("arrows", []) if isinstance(program.get("arrows"), list) else []: + if not isinstance(arrow, dict): + continue + source_id = str(arrow.get("source_id") or arrow.get("source") or "") + target_id = str(arrow.get("target_id") or arrow.get("target") or "") + arrow_id = str(arrow.get("id") or "reference_arrow") + if not source_id or not target_id: + replaced.append(arrow_id) + continue + source_container = mapped_container(all_objects.get(source_id)) + target_container = mapped_container(all_objects.get(target_id)) + if source_container and target_container and source_container != target_container: + replaced.append(arrow_id) + continue + preserved.append(arrow) + return preserved, replaced + + def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: entities = _entities(contract) mapping, mapping_records = _assign_entities(entities, program) mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] - preserved = [item for item in existing if not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes)] + preserved = [ + item for item in existing + if not str(item.get("id") or "").startswith("semantic_text_") + and not item.get("semantic_entity_id") + and not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes) + ] semantic_text = [] for entity in entities: obj = mapping.get(entity["id"]) @@ -208,19 +287,24 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "arrowhead": "triangle", "editable_in": "pptx", }) - if semantic_arrows: - program["arrows"] = semantic_arrows + preserved_reference_arrows, replaced_reference_arrow_ids = _preserve_local_reference_arrows(program, mapping) + if mapping: + program["arrows"] = [*preserved_reference_arrows, *semantic_arrows] + low_confidence_mappings = [item for item in mapping_records if float(item.get("match_score") or 0) <= 0] report = { "summary": "Paper semantic contract application report.", - "status": "pass" if len(mapping) == len(entities) and not skipped_relations else "warning", + "status": "pass" if len(mapping) == len(entities) and not skipped_relations and not low_confidence_mappings else "warning", "semantic_authority": "paper_ground_truth", "visual_authority": "generated_reference_image", "entity_count": len(entities), "mapped_entity_count": len(mapping), "relation_count": len(spec.get("relations") or []), "mapped_relation_count": len(semantic_arrows), + "preserved_reference_arrow_count": len(preserved_reference_arrows), + "replaced_reference_arrow_ids": replaced_reference_arrow_ids, "mappings": mapping_records, + "low_confidence_mappings": low_confidence_mappings, "skipped_relations": skipped_relations, "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", } diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index 05aa942..e2a388b 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -1004,7 +1004,14 @@ def rebuild_editable( program["arrows"] = reference_controls.get("arrows", []) write_json(out_path / "reference_controls.json", reference_controls) - if text_mode != "off": + reuse_text_layer = bool( + skip_analysis + and (out_path / "reference_text_geometry.json").exists() + and (out_path / "text_program.json").exists() + ) + if reuse_text_layer: + pass + elif text_mode != "off": extractor_mode = "ocr" if text_mode == "ocr" else "heuristic" build_text_layer( archived_reference, @@ -1108,6 +1115,7 @@ def rebuild_editable( "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, "connector_count": len(program.get("arrows", [])), "text_mode": text_mode, + "text_layer_reused": reuse_text_layer, "control_mode": control_mode, "layout_mode": layout_mode, "rebuild_visual_quality_status": visual_quality_report.get("status"), diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 16f0d33..51826ba 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -50,7 +50,7 @@ def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = No return [], engine, "local OCR disabled" -def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], repeatable_labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: +def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], repeatable_labels: list[str], optional_visible_labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: prompt = f""" # Summary @@ -62,6 +62,9 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: Evidence-supported labels that may repeat to show reuse of the same component: {json.dumps(repeatable_labels, ensure_ascii=False)} +Evidence-supported optional labels that may appear but must not be reported missing: +{json.dumps(optional_visible_labels, ensure_ascii=False)} + Forbidden copied reference labels: {json.dumps(forbidden_labels, ensure_ascii=False)} @@ -205,6 +208,7 @@ def review_candidate( blueprint_path = Path(blueprint) labels = required_labels(plan) repeatable_labels = [str(value).strip() for value in plan.get("figure_specification", {}).get("repeatable_labels", []) if str(value).strip()] + optional_visible_labels = [str(value).strip() for value in plan.get("figure_specification", {}).get("optional_visible_labels", []) if str(value).strip()] forbidden = [str(value) for value in template.get("forbidden_copy_terms", []) if str(value).strip()] with Image.open(candidate) as image: width, height = image.size @@ -314,6 +318,7 @@ def review_candidate( template, labels, repeatable_labels, + optional_visible_labels, forbidden, model, critic_adapter, @@ -351,18 +356,24 @@ def review_candidate( ocr.setdefault("local_engine", local_engine) ocr.setdefault("local_detected_text", []) repeatable_normalized = {_normalize_text(value) for value in repeatable_labels if _normalize_text(value)} + optional_normalized = {_normalize_text(value) for value in optional_visible_labels if _normalize_text(value)} required_normalized = {_normalize_text(value) for value in labels if _normalize_text(value)} duplicate_labels = [str(value) for value in ocr.get("duplicate_labels", [])] allowed_duplicate_labels = [value for value in duplicate_labels if _normalize_text(value) in repeatable_normalized] ocr["allowed_duplicate_labels"] = allowed_duplicate_labels ocr["duplicate_labels"] = [value for value in duplicate_labels if _normalize_text(value) not in repeatable_normalized] detected_labels = [str(value).strip() for value in ocr.get("detected_labels", []) if str(value).strip()] - allowed_visible_labels = required_normalized | repeatable_normalized + allowed_visible_labels = required_normalized | repeatable_normalized | optional_normalized ocr["unexpected_labels"] = [ value for value in detected_labels if _normalize_text(value) and _normalize_text(value) not in allowed_visible_labels ] scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") + unexpected_visible_labels = [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] + scientific["unexpected_visible_labels"] = unexpected_visible_labels + if unexpected_visible_labels: + invented_items = [str(value) for value in scientific.get("invented_items", []) if str(value).strip()] + scientific["invented_items"] = list(dict.fromkeys([*invented_items, *unexpected_visible_labels])) template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") for field in ("visual_information_density", "mechanism_visualization", "publication_polish"): diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index b3d57aa..af30556 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -89,7 +89,9 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = "training_flow": [], "inference_flow": [], "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", + "overview_panels": [{{"id": "panel_a", "panel_label": "a", "label": "exact evidence-backed panel title", "description": "...", "evidence_ids": ["E0001"], "entity_ids": []}}], "required_labels": [], + "optional_visible_labels": [], "visual_priorities": [], "terminology": {{}}, "forbidden_inventions": [], @@ -673,11 +675,24 @@ def _topology_specific_prompt_rules(spec: dict) -> str: - Forbidden connectors: Prompt -> Image Encoder; Image -> Prompt Encoder; Image Encoder -> Prompt Encoder; Prompt Encoder -> Image Encoder; Fully Automatic -> Valid Segmentation Mask; Mask Decoder -> Data Engine as a replacement for the required container-level support arrow. - Do not let any Data Engine stage enter Valid Segmentation Mask, and do not let the model inference path enter SA-1B.""" if topology == "dense_multiframe": - return """Dense multi-frame topology hard rules: + panels = [item for item in spec.get("overview_panels", []) if isinstance(item, dict) and str(item.get("label") or "").strip()] + panel_lines = "\n".join( + f" {str(item.get('panel_label') or index + 1)}. {str(item.get('label') or '').strip()} — {str(item.get('description') or '').strip()}" + for index, item in enumerate(panels) + ) + panel_rule = ( + f"- Preserve exactly {len(panels)} macro panels with these evidence-backed meanings:\n{panel_lines}" + if panels else + "- Preserve the declared macro-panel grouping and every directed cross-panel relation." + ) + return f"""Dense multi-frame topology hard rules: +{panel_rule} - Preserve the declared macro-panel grouping and every directed cross-panel relation. - Keep independent input branches separate until their declared convergence node. - Keep panel-local output paths separate from dataset, training, or data-engine paths. - Draw high-level container-to-container relations separately from internal module arrows. +- Do not replace evidence-backed pretraining, data, or evaluation panels with an invented architecture-detail inset. +- Do not add a titled internal inset, legend, or architecture breakdown unless its title is in the exact visible label whitelist. - Do not introduce SAM-specific encoders, masks, data-engine stages, or dataset labels unless they are present in the contract.""" return "No additional topology-specific rules. Follow the directed connector checklist exactly." @@ -692,6 +707,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = semantic_plan = template.get("semantic_plan", {}) if isinstance(template.get("semantic_plan"), dict) else {} labels = collect_visible_labels(spec) repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] + optional_visible_labels = [str(value).strip() for value in spec.get("optional_visible_labels", []) if str(value).strip()] if isinstance(spec.get("optional_visible_labels"), list) else [] repeatable_normalized = {re.sub(r"[^\w\u4e00-\u9fff]+", "", value.casefold()) for value in repeatable_labels} numbered_labels = "\n".join( f"{index}. {label}" + (" (may repeat to show evidence-supported component reuse)" if re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) in repeatable_normalized else "") @@ -716,11 +732,20 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = "inference_flow": spec.get("inference_flow", []), "forbidden_inventions": spec.get("forbidden_inventions", []), "uncertainties": spec.get("uncertainties", []), - "supporting_details": [ + "overview_panels": [ + { + "id": item.get("id"), + "panel_label": item.get("panel_label"), + "label": item.get("label"), + "description": item.get("description"), + "entity_ids": item.get("entity_ids", []), + } + for item in spec.get("overview_panels", []) if isinstance(item, dict) + ], + "supporting_visual_cues": [ { - "label": str(item.get("name") or item.get("label") or item.get("statement") or ""), - "role": item.get("role"), - "visibility": item.get("visibility"), + "visual_cue": item.get("visual_cue") or "one unlabeled panel-local scientific mechanism detail", + "must_be_unlabeled": True, } for item in spec.get("supporting_details", []) if isinstance(item, dict) ], @@ -745,6 +770,15 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = compact_semantic_plan = { "applied": bool(semantic_plan.get("applied")), "topology": semantic_plan.get("topology"), + "panels": [ + { + "id": item.get("id"), + "panel_label": item.get("panel_label"), + "label": item.get("label"), + "bbox_percent": item.get("bbox_percent"), + } + for item in semantic_plan.get("panels", []) if isinstance(item, dict) + ], "nodes": [ { "id": item.get("id"), @@ -791,6 +825,9 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Exact visible label whitelist (render these exactly and do not invent alternatives): {json.dumps(labels, ensure_ascii=False)} +Evidence-backed optional visible labels (allowed if useful, but not required): +{json.dumps(optional_visible_labels, ensure_ascii=False)} + Mandatory visible label checklist: {numbered_labels} @@ -804,7 +841,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Mandatory entity role and containment checklist: {json.dumps(compact_roles, ensure_ascii=False, separators=(',', ':'))} -Panel-local supporting details in the compact contract are evidence-backed visual cues only. Use them as unlabeled internal texture or mechanism detail when helpful; do not promote them to peer top-level nodes, do not add them to the visible label whitelist, and do not invent connectors for them. +Panel-local supporting visual cues in the compact contract are evidence-backed planning metadata only. Use them as unlabeled internal texture or mechanism detail when helpful. Never render the cue wording itself, never turn the cues into a legend, never give them titles, never promote them to peer top-level nodes, and never invent connectors for them. Preserve these semantic roles in the visible composition. A data source, modality, processing module, innovation, and output are not interchangeable even when all labels are present. In particular, do not flatten a source -> modality -> method chain into peer inputs or place a dataset beside modalities as though it were another sensor type. @@ -832,6 +869,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = - When semantic blueprint geometry is supplied, preserve every node bounding box, layer assignment, connector path, and arrow direction; visual decoration may change but graph geometry may not. - Template panels are macro spatial regions, not an exact limit on the number of scientific content nodes inside those regions. - Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. +- The exact visible label whitelist is the only permitted visible text. Do not add a legend, inset title, process verb, architecture submodule, or explanatory heading outside that whitelist. - Render every mandatory visible label at least once; repeat only labels explicitly marked as evidence-supported shared components, and never replace an output label with an icon-only node. - Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. - Include explicit input-boundary and output-boundary connectors. @@ -949,6 +987,17 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * errors.append(f"{field}[{index}] references unknown evidence ids: {invalid}") validate_visible_label(f"{field}[{index}]", str(item.get("name") or item.get("visible_label") or item.get("text") or ""), evidence_ids) + for index, panel in enumerate(spec.get("overview_panels") if isinstance(spec.get("overview_panels"), list) else []): + if not isinstance(panel, dict): + continue + evidence_ids = panel.get("evidence_ids") if isinstance(panel.get("evidence_ids"), list) else [] + if not evidence_ids: + errors.append(f"overview_panels[{index}] has no evidence_ids") + invalid = [value for value in evidence_ids if value not in valid_evidence] + if invalid: + errors.append(f"overview_panels[{index}] references unknown evidence ids: {invalid}") + validate_visible_label(f"overview_panels[{index}]", str(panel.get("label") or ""), evidence_ids) + relations = spec.get("relations") if isinstance(spec.get("relations"), list) else [] for index, relation in enumerate(relations): if not isinstance(relation, dict): diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 5a882ee..a778186 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -412,6 +412,143 @@ def _detect_dense_overview_panels(parsed: dict[str, Any]) -> dict[str, Any]: } +def _overview_caption_blocks(parsed: dict[str, Any], figure: dict[str, Any]) -> list[dict[str, Any]]: + """Recover a caption that continues across columns on the same page.""" + page_number = int(figure.get("page") or 0) + page = next( + ( + item + for item in parsed.get("pages", []) + if isinstance(item, dict) and int(item.get("page") or 0) == page_number + ), + None, + ) + if not page: + return [] + blocks = [item for item in page.get("blocks", []) if isinstance(item, dict)] + start_id = str(figure.get("block_id") or "") + start = next((item for item in blocks if str(item.get("id") or "") == start_id), None) + if not start: + return [] + start_bbox = start.get("bbox") if isinstance(start.get("bbox"), list) else [0, 0, 0, 0] + start_y = float(start_bbox[1] or 0) + page_height = max(1.0, float(page.get("height") or 1)) + max_y = start_y + max(42.0, page_height * 0.09) + start_font = float(start.get("font_size") or 0) + start_order = int(start.get("reading_order") or 0) + caption_blocks = [] + for block in blocks: + bbox = block.get("bbox") if isinstance(block.get("bbox"), list) else [0, 0, 0, 0] + block_y = float(bbox[1] or 0) + block_font = float(block.get("font_size") or 0) + if int(block.get("reading_order") or 0) < start_order: + continue + if not (start_y - 2.0 <= block_y <= max_y): + continue + if start_font and block_font and abs(block_font - start_font) > 0.8: + continue + caption_blocks.append(block) + return sorted(caption_blocks, key=lambda item: int(item.get("reading_order") or 0)) + + +def _clean_overview_panel_label(description: str) -> str: + first_sentence = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", description).strip())[0].strip(" .") + first_sentence = re.sub( + r"^(?:flow\s+chart|diagram|schematic|illustration)\s+(?:showing|of)\s+(?:the\s+)?", + "", + first_sentence, + flags=re.IGNORECASE, + ).strip() + if len(first_sentence) > 110: + first_sentence = re.split(r"\s+(?:that|which|where|by|to)\s+", first_sentence, maxsplit=1, flags=re.IGNORECASE)[0].strip() + return first_sentence[:1].upper() + first_sentence[1:] if first_sentence else "" + + +def _extract_overview_panels(parsed: dict[str, Any], spec: dict[str, Any]) -> list[dict[str, Any]]: + figures = parsed.get("document_index", {}).get("figures", []) + figure = next( + ( + item + for item in figures + if isinstance(item, dict) + and re.match(r"^(?:figure|fig\.)\s*1\b", str(item.get("caption") or ""), re.IGNORECASE) + and re.search(r"\b(?:overview|architecture|framework|pipeline|model)\b", str(item.get("caption") or ""), re.IGNORECASE) + ), + None, + ) + if not figure: + return [] + blocks = _overview_caption_blocks(parsed, figure) + if not blocks: + return [] + evidence_by_block = { + str(item.get("block_id") or ""): str(item.get("id") or "") + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("block_id") and item.get("id") + } + marker_pattern = re.compile(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", re.IGNORECASE) + descriptions: dict[str, list[str]] = {} + evidence_ids: dict[str, list[str]] = {} + current: str | None = None + for block in blocks: + text = re.sub(r"\s+", " ", str(block.get("text") or "")).strip() + if not text: + continue + matches = list(marker_pattern.finditer(text)) + cursor = 0 + for match in matches: + prefix = text[cursor:match.start()].strip(" ;,") + if prefix and match.group(0).lstrip().startswith((".", ";")) and not prefix.endswith((".", ";")): + prefix += match.group(0).lstrip()[0] + if current and prefix: + descriptions.setdefault(current, []).append(prefix) + evidence_id = evidence_by_block.get(str(block.get("id") or "")) + if evidence_id: + evidence_ids.setdefault(current, []).append(evidence_id) + current = match.group(1).casefold() + descriptions.setdefault(current, []) + evidence_ids.setdefault(current, []) + cursor = match.end() + suffix = text[cursor:].strip(" ;,") + if current and suffix: + descriptions.setdefault(current, []).append(suffix) + evidence_id = evidence_by_block.get(str(block.get("id") or "")) + if evidence_id: + evidence_ids.setdefault(current, []).append(evidence_id) + if len(descriptions) < 3: + return [] + + entities = [ + item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + ] + panels: list[dict[str, Any]] = [] + for label in sorted(descriptions): + description = re.sub(r"\s+", " ", " ".join(descriptions[label])).strip(" .") + visible_label = _clean_overview_panel_label(description) + if not visible_label: + continue + panel_evidence = list(dict.fromkeys(evidence_ids.get(label, []))) + panel_evidence_set = set(panel_evidence) + entity_ids = [ + str(item.get("id")) + for item in entities + if panel_evidence_set.intersection(str(value) for value in item.get("evidence_ids", []) if value) + ] + panels.append({ + "id": f"panel_{label}", + "panel_label": label, + "label": visible_label, + "description": description, + "page": int(figure.get("page") or 0), + "evidence_ids": panel_evidence, + "entity_ids": list(dict.fromkeys(entity_ids)), + }) + return panels if len(panels) >= 3 else [] + + def _merge_isolated_architecture_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: modules = [item for item in spec.get("modules", []) if isinstance(item, dict)] relations = [item for item in spec.get("relations", []) if isinstance(item, dict)] @@ -493,7 +630,70 @@ def _merge_isolated_architecture_entities(spec: dict[str, Any], parsed: dict[str return merged -def _demote_unconnected_completion_entities(spec: dict[str, Any], completion_report: dict[str, Any]) -> list[str]: +def _supporting_visual_cue(label: str) -> str: + normalized = _normalized_label(label) + if "contrastive" in normalized: + return "paired augmented views converging on an unlabeled comparison objective" + if "mask" in normalized and ("embedding" in normalized or "token" in normalized): + return "a compact sequence of feature blocks with several visibly masked positions" + if "classificationtoken" in normalized or "classtoken" in normalized or normalized in {"cls", "clstoken"}: + return "one distinct highlighted token block among repeated token blocks" + if "patch" in normalized or "tile" in normalized or "crop" in normalized: + return "a compact group of image patches or multi-scale views" + if "embedding" in normalized: + return "a compact sequence of feature blocks" + if "loss" in normalized or "objective" in normalized: + return "an unlabeled objective or comparison cue" + return "one unlabeled panel-local scientific mechanism detail" + + +def _label_like_evidence_text(value: str) -> str | None: + text = re.sub(r"\s+", " ", str(value or "")).strip().strip(".;:") + if not 2 <= len(text) <= 64 or len(text.split()) > 9: + return None + if re.match(r"^(?:to|we|our|the|this|these|those|it|they)\b", text, re.IGNORECASE): + return None + if re.search(r"\b(?:which|that|can|could|would|should|serve as|results? in)\b", text, re.IGNORECASE): + return None + return text + + +def _defined_symbol_for_supporting_detail( + label: str, + evidence_ids: list[str], + parsed: dict[str, Any] | None, +) -> str | None: + if not parsed: + return None + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict)] + page_by_id = {str(item.get("id")): int(item.get("page") or 0) for item in evidence if item.get("id")} + pages = {page_by_id.get(str(value), 0) for value in evidence_ids} + pages.discard(0) + normalized_label = _normalized_label(label) + if not pages or not normalized_label: + return None + for page in sorted(pages): + page_text = " ".join( + re.sub(r"\s+", " ", str(item.get("text") or "")).strip() + for item in evidence + if int(item.get("page") or 0) == page + ) + for match in re.finditer( + r"(?P\[[A-Za-z0-9_+\-]{1,16}\]|<[A-Za-z0-9_+\-]{1,16}>)\s+(?:is|denotes|represents)\s+(?:the\s+|an?\s+)?(?P[^.;]{2,80})", + page_text, + re.IGNORECASE, + ): + definition = _normalized_label(match.group("definition")) + if normalized_label in definition or definition in normalized_label: + return match.group("symbol") + return None + + +def _demote_unconnected_completion_entities( + spec: dict[str, Any], + completion_report: dict[str, Any], + parsed: dict[str, Any] | None = None, +) -> list[str]: added_ids = {str(value) for value in completion_report.get("added_entities", []) if str(value)} if not added_ids: return [] @@ -504,7 +704,12 @@ def _demote_unconnected_completion_entities(spec: dict[str, Any], completion_rep if str(value or "").strip() } demoted: list[str] = [] - supporting = [item for item in spec.get("supporting_details", []) if isinstance(item, dict)] + evidence_by_id = { + str(item.get("id")): item + for item in (parsed or {}).get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + supporting = [dict(item) for item in spec.get("supporting_details", []) if isinstance(item, dict)] for field in ("inputs", "modules", "outputs", "innovations"): kept = [] for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: @@ -523,7 +728,45 @@ def _demote_unconnected_completion_entities(spec: dict[str, Any], completion_rep continue kept.append(item) spec[field] = kept - spec["supporting_details"] = supporting + normalized_supporting: list[dict[str, Any]] = [] + by_key: dict[str, dict[str, Any]] = {} + for item in supporting: + exact_candidates = [ + _label_like_evidence_text(str(evidence_by_id[evidence_id].get("text") or "")) + for evidence_id in (str(value) for value in item.get("evidence_ids", []) if value) + if evidence_id in evidence_by_id + ] + exact_candidates = [value for value in exact_candidates if value] + if evidence_by_id and not exact_candidates: + continue + if exact_candidates: + item["name"] = min(exact_candidates, key=len) + label = _item_label(item) + if not label: + continue + item["visual_cue"] = _supporting_visual_cue(label) + item["visibility"] = "unlabeled_visual_detail_not_global_node" + optional_symbol = _defined_symbol_for_supporting_detail( + label, + [str(value) for value in item.get("evidence_ids", []) if value], + parsed, + ) + if optional_symbol: + item["optional_visible_label"] = optional_symbol + key = str(item.get("id") or "").strip() or _normalized_label(label) + label_key = _normalized_label(label) + existing = by_key.get(key) or by_key.get(label_key) + if existing: + existing["evidence_ids"] = list(dict.fromkeys([ + *existing.get("evidence_ids", []), + *item.get("evidence_ids", []), + ])) + continue + normalized_supporting.append(item) + by_key[key] = item + if label_key: + by_key[label_key] = item + spec["supporting_details"] = normalized_supporting return demoted @@ -1844,7 +2087,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: incoming.add(target_id) spec["relations"] = relations completion_report["merged_architecture_entities"] = _merge_isolated_architecture_entities(spec, parsed) - completion_report["demoted_unconnected_completion_entities"] = _demote_unconnected_completion_entities(spec, completion_report) + completion_report["demoted_unconnected_completion_entities"] = _demote_unconnected_completion_entities(spec, completion_report, parsed) feedback_report = _simplify_feedback_contract(spec) completion_report["feedback_simplification_applied"] = feedback_report["applied"] completion_report["feedback_removed_entities"] = feedback_report["removed_entities"] @@ -1879,6 +2122,29 @@ def choose_input_target(label: str) -> dict[str, Any] | None: _repair_cross_script_terminology(spec, parsed) completion_report["repaired_cross_script_entity_labels"] = _repair_cross_script_entity_labels(spec, parsed) completion_report["removed_unverbatim_relation_labels"] = _clear_unverbatim_relation_labels(spec, parsed) + overview_panels = _extract_overview_panels(parsed, spec) + if overview_panels: + spec["overview_panels"] = overview_panels + elif not isinstance(spec.get("overview_panels"), list): + spec["overview_panels"] = [] + completion_report["overview_panels"] = spec["overview_panels"] + optional_labels: list[str] = [] + optional_normalized: set[str] = set() + def add_optional_label(value: object) -> None: + label = str(value or "").strip() + normalized = _normalized_label(label) + if label and normalized and normalized not in optional_normalized: + optional_labels.append(label) + optional_normalized.add(normalized) + for value in spec.get("optional_visible_labels", []) if isinstance(spec.get("optional_visible_labels"), list) else []: + add_optional_label(value) + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if isinstance(panel, dict): + add_optional_label(panel.get("panel_label")) + for item in spec.get("supporting_details", []) if isinstance(spec.get("supporting_details"), list) else []: + if isinstance(item, dict): + add_optional_label(item.get("optional_visible_label")) + spec["optional_visible_labels"] = optional_labels labels = [] normalized_labels: set[str] = set() def add_required_label(value: object) -> None: @@ -1892,6 +2158,9 @@ def add_required_label(value: object) -> None: for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: if isinstance(item, dict) and _item_label(item): add_required_label(_item_label(item)) + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if isinstance(panel, dict): + add_required_label(panel.get("label")) for value in terminology.values(): add_required_label(value) spec["required_labels"] = labels diff --git a/rfs/paper_to_image/semantic_blueprint.py b/rfs/paper_to_image/semantic_blueprint.py index d0bc2ff..2f480de 100644 --- a/rfs/paper_to_image/semantic_blueprint.py +++ b/rfs/paper_to_image/semantic_blueprint.py @@ -46,11 +46,119 @@ def _visible_entities(specification: dict[str, Any]) -> list[dict[str, Any]]: "label": label, "field": field, "role": str(item.get("role") or "").strip(), + "evidence_ids": [str(value) for value in item.get("evidence_ids", []) if value], "order": len(entities), }) return entities +def _dense_overview_panels(specification: dict[str, Any]) -> list[dict[str, Any]]: + values = specification.get("overview_panels", []) + if not isinstance(values, list): + return [] + panels: list[dict[str, Any]] = [] + for index, item in enumerate(values): + if not isinstance(item, dict): + continue + label = str(item.get("label") or item.get("title") or "").strip() + panel_id = str(item.get("id") or f"panel_{index + 1}").strip() + if not label or not panel_id: + continue + panels.append({ + "id": panel_id, + "panel_label": str(item.get("panel_label") or chr(ord("a") + index)).strip(), + "label": label, + "description": str(item.get("description") or "").strip(), + "evidence_ids": [str(value) for value in item.get("evidence_ids", []) if value], + "entity_ids": [str(value) for value in item.get("entity_ids", []) if value], + "order": index, + }) + return panels + + +def _layout_dense_panels(panels: list[dict[str, Any]]) -> list[dict[str, Any]]: + count = len(panels) + positioned: list[dict[str, Any]] = [] + if count == 3: + boxes = [ + {"x": 0.04, "y": 0.05, "w": 0.92, "h": 0.42}, + {"x": 0.04, "y": 0.53, "w": 0.44, "h": 0.42}, + {"x": 0.52, "y": 0.53, "w": 0.44, "h": 0.42}, + ] + else: + columns = 2 if count <= 4 else 3 + rows = (count + columns - 1) // columns + gap_x, gap_y = 0.04, 0.06 + width = (0.92 - gap_x * (columns - 1)) / columns + height = (0.90 - gap_y * (rows - 1)) / rows + boxes = [ + { + "x": round(0.04 + (index % columns) * (width + gap_x), 6), + "y": round(0.05 + (index // columns) * (height + gap_y), 6), + "w": round(width, 6), + "h": round(height, 6), + } + for index in range(count) + ] + for panel, box in zip(panels, boxes): + positioned.append({**panel, "bbox_percent": box}) + return positioned + + +def _layout_nodes_in_dense_panels( + nodes: list[dict[str, Any]], + relations: list[dict[str, str]], + panels: list[dict[str, Any]], +) -> list[dict[str, Any]]: + if not panels: + return _layout_nodes(nodes, _rank_nodes(nodes, relations)) + panel_by_entity = { + entity_id: panel["id"] + for panel in panels + for entity_id in panel.get("entity_ids", []) + } + panel_by_evidence = { + evidence_id: panel["id"] + for panel in panels + for evidence_id in panel.get("evidence_ids", []) + } + default_panel = panels[0]["id"] + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for node in nodes: + panel_id = panel_by_entity.get(node["id"]) + if not panel_id: + panel_id = next((panel_by_evidence[value] for value in node.get("evidence_ids", []) if value in panel_by_evidence), None) + grouped[panel_id or default_panel].append(node) + by_panel = {panel["id"]: panel for panel in panels} + positioned: list[dict[str, Any]] = [] + for panel_id, panel_nodes in grouped.items(): + panel = by_panel.get(panel_id) or panels[0] + node_ids = {item["id"] for item in panel_nodes} + local_relations = [ + item for item in relations + if item["source"] in node_ids and item["target"] in node_ids + ] + local_layout = _layout_nodes(panel_nodes, _rank_nodes(panel_nodes, local_relations)) + box = panel["bbox_percent"] + inner_x = box["x"] + box["w"] * 0.035 + inner_y = box["y"] + box["h"] * 0.24 + inner_w = box["w"] * 0.93 + inner_h = box["h"] * 0.70 + for item in local_layout: + local_box = item["bbox_percent"] + positioned.append({ + **item, + "panel_id": panel["id"], + "bbox_percent": { + "x": round(inner_x + local_box["x"] * inner_w, 6), + "y": round(inner_y + local_box["y"] * inner_h, 6), + "w": round(local_box["w"] * inner_w, 6), + "h": round(local_box["h"] * inner_h, 6), + }, + }) + return positioned + + def _visible_relations(specification: dict[str, Any], node_ids: set[str]) -> list[dict[str, str]]: relations: list[dict[str, str]] = [] seen: set[tuple[str, str]] = set() @@ -262,8 +370,12 @@ def compile_semantic_blueprint( if len(connected_nodes) >= 2: nodes = connected_nodes - ranks = _rank_nodes(nodes, relations) - positioned = _layout_nodes(nodes, ranks) + dense_panels = _layout_dense_panels(_dense_overview_panels(specification)) if topology == "dense_multiframe" else [] + if len(dense_panels) >= 3: + positioned = _layout_nodes_in_dense_panels(nodes, relations, dense_panels) + else: + ranks = _rank_nodes(nodes, relations) + positioned = _layout_nodes(nodes, ranks) by_id = {item["id"]: item for item in positioned} incoming_groups: dict[str, list[dict[str, str]]] = defaultdict(list) outgoing_groups: dict[str, list[dict[str, str]]] = defaultdict(list) @@ -298,6 +410,7 @@ def compile_semantic_blueprint( "node_count": len(positioned), "connector_count": len(connectors), "isolated_node_ids": isolated_node_ids, + "panels": dense_panels, "nodes": positioned, "connectors": connectors, "source": "figure_specification", diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index 92d5326..a1953d1 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -268,6 +268,23 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", selected = dict(matches[0]) selected["selection"] = {"mode": "explicit", "score": 1.0, "reasons": ["explicit CLI selection"]} return selected + topology_template = { + "dense_multiframe": "dense-multiframe", + "branch": "branch", + "feedback": "feedback", + "multimodal": "multimodal", + }.get(str(contract_topology or "").casefold()) + if topology_template: + match = next((profile for profile in profiles if profile.get("template_id") == topology_template), None) + if match: + selected = dict(match) + selected["selection"] = { + "mode": "contract_topology", + "score": 1.0, + "reasons": [f"normalized scientific contract topology={contract_topology}"], + "paper_features": _paper_features(review), + } + return selected features = _paper_features(review) requested_ratio = None if target_ratio != "auto": @@ -283,9 +300,6 @@ def select_template(profiles: list[dict], review: dict, requested: str = "auto", module_fit = 1.0 if low <= features["module_count"] <= high else max(0.0, 1.0 - min(abs(features["module_count"] - low), abs(features["module_count"] - high)) / 10) topology = 0.0 reasons = [f"module_fit={module_fit:.2f}"] - topology_template = {"dense_multiframe": "dense-multiframe"}.get(str(contract_topology or ""), str(contract_topology or "")) - if topology_template and template_id == topology_template: - topology += 1.5; reasons.append(f"normalized contract topology={contract_topology}") if template_id == "feedback" and features["has_loop"] and not features["has_tree"]: topology += 1.0; reasons.append("explicit feedback-loop topology") if template_id == "branch" and features["has_branch"] and not features["has_loop"] and not features["has_tree"] and not features["has_multimodal"] and not features["has_retrieval"]: @@ -597,6 +611,26 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) centered_text(region(model_box, 0.03, 0.03, 0.97, 0.15), exact("Segment Anything Model"), title_size) centered_text(region(data_box, 0.03, 0.03, 0.97, 0.15), exact("Data Engine"), title_size, fill=(177, 92, 33)) elif semantic_generic: + for panel in generic_semantic_plan.get("panels", []): + bbox = panel.get("bbox_percent", {}) + panel_box = ( + round(float(bbox.get("x", 0)) * width), + round(float(bbox.get("y", 0)) * height), + round(float(bbox.get("x", 0) + bbox.get("w", 0)) * width), + round(float(bbox.get("y", 0) + bbox.get("h", 0)) * height), + ) + draw.rounded_rectangle( + panel_box, + radius=max(10, round(height * 0.018)), + fill=(247, 249, 249), + outline=(104, 132, 151), + width=3, + ) + title_box = region(panel_box, 0.025, 0.025, 0.975, 0.20) + panel_marker = str(panel.get("panel_label") or "").strip() + panel_title = str(panel.get("label") or "").strip() + visible_title = f"{panel_marker} {panel_title}" if panel_marker else panel_title + centered_wrapped_text(title_box, visible_title, max(13, round(height * 0.020)), fill=(42, 72, 91)) node_boxes: dict[str, tuple[int, int, int, int]] = {} node_styles: dict[str, tuple[tuple[int, int, int], tuple[int, int, int]]] = {} for node in generic_semantic_plan.get("nodes", []): @@ -638,7 +672,7 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) label = str(connector.get("label") or "").strip() if label and len(points) >= 2: middle = points[len(points) // 2] - label_box = (middle[0] - 60, middle[1] - 18, middle[0] + 60, middle[1] + 18) + label_box = (middle[0] - 58, middle[1] - 48, middle[0] + 58, middle[1] - 18) draw.rounded_rectangle(label_box, radius=7, fill=(250, 250, 249), outline=color, width=1) centered_wrapped_text(label_box, label, max(11, round(height * 0.014)), fill=color, bold=False) @@ -662,7 +696,8 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) "Assisted-manual", "Semi-automatic", "Fully Automatic", "SA-1B", ] elif semantic_generic: - semantic_label_order = [str(item.get("label") or "") for item in generic_semantic_plan.get("nodes", [])] + semantic_label_order = [str(item.get("label") or "") for item in generic_semantic_plan.get("panels", [])] + semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("nodes", [])) semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("connectors", []) if str(item.get("label") or "").strip()) return { "summary": "Paper-semantic layout blueprint rendered from the normalized contract." if semantic_blueprint else "Content-free layout blueprint rendered from selected template.", diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 74db08e..ec910bb 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -362,9 +362,31 @@ def critic(path, template, prompt): self.assertFalse(review["production_pass"]) self.assertEqual(review["ocr"]["unexpected_labels"], ["Contrastive Loss"]) + self.assertFalse(review["scientific"]["passed"]) + self.assertIn("Contrastive Loss", review["scientific"]["invented_items"]) self.assertIn("Remove unexpected visible label: Contrastive Loss", review["repair"]) self.assertIn("Contrastive Loss", review["remove"]) + def test_candidate_review_allows_evidence_backed_optional_panel_and_symbol_labels(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["optional_visible_labels"] = ["a", "b", "c", "[CLS]"] + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].extend(["a", "b", "c", "[CLS]"]) + return result + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["unexpected_labels"], []) + def test_candidate_review_rejects_dataset_drawn_as_peer_modality(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -571,6 +593,53 @@ def test_generic_dense_multiframe_prompt_does_not_inject_sam_terms(self): self.assertNotIn("Image must connect only to Image Encoder", prompt) self.assertIn("Do not introduce SAM-specific encoders", prompt) + def test_dense_overview_prompt_preserves_panels_without_exposing_supporting_labels(self): + plan = _plan() + plan["figure_specification"].update({ + "topology": "dense_multiframe", + "overview_panels": [ + {"id": "panel_a", "panel_label": "a", "label": "Model architecture", "description": "Model architecture and inference flow", "entity_ids": ["parser"]}, + {"id": "panel_b", "panel_label": "b", "label": "Tile-level pretraining", "description": "Tile-level pretraining using a teacher-student objective", "entity_ids": []}, + {"id": "panel_c", "panel_label": "c", "label": "Slide-level pretraining", "description": "Slide-level masked pretraining", "entity_ids": []}, + ], + "supporting_details": [ + {"name": "Contrastive Loss", "visual_cue": "paired augmented views converging on an unlabeled comparison objective"}, + ], + }) + + prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) + + self.assertIn("Preserve exactly 3 macro panels", prompt) + self.assertIn("Tile-level pretraining", prompt) + self.assertIn('\"supporting_visual_cues\"', prompt) + self.assertNotIn('\"label\":\"Contrastive Loss\"', prompt) + self.assertIn("Never render the cue wording itself", prompt) + + def test_generic_dense_semantic_blueprint_preserves_three_panel_geometry(self): + specification = { + "topology": "dense_multiframe", + "required_labels": ["Input", "Encoder", "Output", "Architecture", "Pretraining A", "Pretraining B"], + "overview_panels": [ + {"id": "panel_a", "panel_label": "a", "label": "Architecture", "description": "Main architecture", "entity_ids": ["input", "encoder", "output"]}, + {"id": "panel_b", "panel_label": "b", "label": "Pretraining A", "description": "First pretraining flow", "entity_ids": []}, + {"id": "panel_c", "panel_label": "c", "label": "Pretraining B", "description": "Second pretraining flow", "entity_ids": []}, + ], + "inputs": [{"id": "input", "name": "Input"}], + "modules": [{"id": "encoder", "name": "Encoder"}], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "input", "target": "encoder", "type": "data_flow"}, + {"source": "encoder", "target": "output", "type": "data_flow"}, + ], + } + + semantic = compile_semantic_blueprint(specification) + + self.assertTrue(semantic["applied"]) + self.assertEqual([item["id"] for item in semantic["panels"]], ["panel_a", "panel_b", "panel_c"]) + self.assertTrue(all(item.get("panel_id") == "panel_a" for item in semantic["nodes"])) + self.assertTrue(all(item["bbox_percent"]["y"] < 0.47 for item in semantic["nodes"])) + def test_generic_linear_blueprint_renders_contract_nodes_and_connectors(self): with tempfile.TemporaryDirectory() as temp: profiles = build_template_profiles([], Path(temp) / "profiles", mode="heuristic") diff --git a/tests/integration/test_rebuild_editable.py b/tests/integration/test_rebuild_editable.py index c601f14..a2c65e4 100644 --- a/tests/integration/test_rebuild_editable.py +++ b/tests/integration/test_rebuild_editable.py @@ -37,6 +37,37 @@ def _fixture(path: Path) -> Path: class RebuildEditableTests(unittest.TestCase): + def test_skip_analysis_reuses_existing_text_layer_without_rerunning_ocr(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = _fixture(root / "pipeline.png") + out = root / "rebuild" + rebuild_editable( + reference, + out, + asset_mode="placeholder", + text_mode="manual", + layout_mode="heuristic", + control_mode="heuristic", + design_plan_mode="heuristic", + ) + + with patch("rfs.editable_rebuild.build_text_layer") as text_builder: + result = rebuild_editable( + reference, + out, + asset_mode="placeholder", + text_mode="ocr", + layout_mode="hybrid", + control_mode="hybrid", + design_plan_mode="vlm", + skip_analysis=True, + ) + + text_builder.assert_not_called() + self.assertTrue(result["ok"]) + self.assertTrue(result["text_layer_reused"]) + def test_global_design_plan_artifacts_are_written(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index 27446d2..d045895 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -2,11 +2,77 @@ import unittest from rfs.paper_to_image.planner import validate_plan_grounding -from rfs.paper_to_image.preparation import _demote_unconnected_completion_entities, _detect_dense_overview_panels, _fast_cache_path, _merge_isolated_architecture_entities, _repair_cross_script_entity_labels, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract +from rfs.paper_to_image.preparation import _demote_unconnected_completion_entities, _detect_dense_overview_panels, _extract_overview_panels, _fast_cache_path, _merge_isolated_architecture_entities, _repair_cross_script_entity_labels, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract from rfs.paper_to_image.contract_completion import _overview_candidates class PaperContractTests(unittest.TestCase): + def test_extracts_cross_column_figure_one_panel_descriptions(self): + parsed = { + "document_index": {"figures": [{ + "page": 3, + "block_id": "P003_B010", + "caption": "Fig. 1 | Overview of Example. a, Flow chart showing the model architecture of Example.", + }]}, + "pages": [{ + "page": 3, + "height": 790, + "blocks": [ + {"id": "P003_B010", "reading_order": 10, "bbox": [40, 365, 290, 374], "font_size": 7, "text": "Fig. 1 | Overview of Example. a, Flow chart showing the model architecture of Example."}, + {"id": "P003_B011", "reading_order": 11, "bbox": [40, 376, 290, 385], "font_size": 7, "text": "The model maps the input through two encoders."}, + {"id": "P003_B067", "reading_order": 67, "bbox": [306, 366, 560, 375], "font_size": 7, "text": "b, Image tile-level pretraining using DINOv2."}, + {"id": "P003_B068", "reading_order": 68, "bbox": [306, 376, 560, 385], "font_size": 7, "text": "c, Slide-level pretraining with LongNet using masked autoencoder."}, + {"id": "P003_B069", "reading_order": 69, "bbox": [306, 435, 560, 445], "font_size": 8.25, "text": "Body text must not enter the caption."}, + ], + }], + "evidence": [ + {"id": "E_A", "block_id": "P003_B010", "text": "Fig. 1 | Overview of Example. a, Flow chart showing the model architecture of Example."}, + {"id": "E_A2", "block_id": "P003_B011", "text": "The model maps the input through two encoders."}, + {"id": "E_B", "block_id": "P003_B067", "text": "b, Image tile-level pretraining using DINOv2."}, + {"id": "E_C", "block_id": "P003_B068", "text": "c, Slide-level pretraining with LongNet using masked autoencoder."}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E_A2"]}], + "modules": [], "outputs": [], "innovations": [], + } + + panels = _extract_overview_panels(parsed, spec) + + self.assertEqual([item["panel_label"] for item in panels], ["a", "b", "c"]) + self.assertEqual(panels[0]["label"], "Model architecture of Example") + self.assertEqual(panels[1]["label"], "Image tile-level pretraining using DINOv2") + self.assertEqual(panels[2]["label"], "Slide-level pretraining with LongNet using masked autoencoder") + self.assertEqual(panels[0]["entity_ids"], ["input"]) + + def test_supporting_details_are_exact_deduplicated_and_weak_verbs_are_dropped(self): + spec = { + "inputs": [], + "modules": [ + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_CLASS"]}, + {"id": "generate", "name": "Generate", "evidence_ids": ["E_GENERATE"]}, + ], + "outputs": [], "innovations": [], "relations": [], + "supporting_details": [ + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_CLASS"]}, + {"id": "class_token", "name": "Class Token", "evidence_ids": ["E_CLASS"]}, + ], + } + completion = {"added_entities": ["class_token", "generate"]} + parsed = {"evidence": [ + {"id": "E_CLASS", "page": 3, "text": "classification token."}, + {"id": "E_SYMBOL", "page": 3, "text": "[CLS] is the classification token."}, + {"id": "E_GENERATE", "page": 3, "text": "to generate contextualized embeddings, which can serve as the basis for downstream applications"}, + ]} + + _demote_unconnected_completion_entities(spec, completion, parsed) + _demote_unconnected_completion_entities(spec, completion, parsed) + + self.assertEqual(len(spec["supporting_details"]), 1) + self.assertEqual(spec["supporting_details"][0]["name"], "classification token") + self.assertEqual(spec["supporting_details"][0]["visual_cue"], "one distinct highlighted token block among repeated token blocks") + self.assertEqual(spec["supporting_details"][0]["optional_visible_label"], "[CLS]") + def test_dense_overview_panels_and_local_details_are_not_flattened_into_peer_nodes(self): parsed = { "document_index": { diff --git a/tests/unit/test_semantic_contract.py b/tests/unit/test_semantic_contract.py index 6e1610d..ff1ed16 100644 --- a/tests/unit/test_semantic_contract.py +++ b/tests/unit/test_semantic_contract.py @@ -6,6 +6,56 @@ class SemanticContractTests(unittest.TestCase): + def test_card_ids_and_ocr_geometry_drive_semantic_mapping_and_hidden_innovation_is_skipped(self): + program = { + "slots": [ + {"id": "slot_micro_left", "bbox_percent": {"x": 0.08, "y": 0.3, "w": 0.08, "h": 0.1}}, + {"id": "slot_micro_middle", "bbox_percent": {"x": 0.43, "y": 0.3, "w": 0.08, "h": 0.1}}, + {"id": "slot_micro_right", "bbox_percent": {"x": 0.78, "y": 0.3, "w": 0.08, "h": 0.1}}, + ], + "cards": [ + {"id": "card_exact_input", "bbox_percent": {"x": 0.05, "y": 0.2, "w": 0.2, "h": 0.35}}, + {"id": "card_paper_method", "bbox_percent": {"x": 0.4, "y": 0.2, "w": 0.2, "h": 0.35}}, + {"id": "card_exact_output", "bbox_percent": {"x": 0.75, "y": 0.2, "w": 0.2, "h": 0.35}}, + ], + "panels": [], + "arrows": [ + {"id": "cross_card_guess", "source_id": "slot_micro_left", "target_id": "slot_micro_middle", "path_percent": [[0.16, 0.35], [0.43, 0.35]]}, + {"id": "local_method_detail", "source_id": "slot_micro_middle", "target_id": "slot_micro_middle", "path_percent": [[0.44, 0.32], [0.5, 0.38]]}, + ], + "text_program": {"items": [ + {"id": "ocr_input", "text": "Exact Input", "bbox_percent": {"x": 0.08, "y": 0.22, "w": 0.1, "h": 0.04}}, + {"id": "ocr_method", "text": "Paper Method", "bbox_percent": {"x": 0.44, "y": 0.22, "w": 0.1, "h": 0.04}}, + {"id": "ocr_output", "text": "Exact Output", "bbox_percent": {"x": 0.79, "y": 0.22, "w": 0.1, "h": 0.04}}, + ]}, + } + contract = {"figure_specification": { + "required_labels": ["Exact Input", "Paper Method", "Exact Output"], + "inputs": [{"id": "input", "name": "Exact Input", "evidence_ids": ["E1"]}], + "modules": [{"id": "method", "name": "Paper Method", "evidence_ids": ["E2"]}], + "outputs": [{"id": "output", "name": "Exact Output", "evidence_ids": ["E3"]}], + "innovations": [{"id": "hidden", "name": "Background Innovation", "evidence_ids": ["E4"]}], + "relations": [], + }} + with tempfile.TemporaryDirectory() as tmp: + updated, report = apply_paper_semantic_contract(program, contract, Path(tmp)) + updated, second_report = apply_paper_semantic_contract(updated, contract, Path(tmp)) + + mapped = {item["entity_id"]: item["object_id"] for item in report["mappings"]} + self.assertEqual(mapped, { + "input": "card_exact_input", + "method": "card_paper_method", + "output": "card_exact_output", + }) + self.assertEqual(report["status"], "pass") + self.assertEqual(report["low_confidence_mappings"], []) + self.assertIn("local_method_detail", {item["id"] for item in updated["arrows"]}) + self.assertNotIn("cross_card_guess", {item["id"] for item in updated["arrows"]}) + self.assertEqual(second_report["status"], "pass") + self.assertEqual({item["entity_id"]: item["object_id"] for item in second_report["mappings"]}, mapped) + self.assertEqual(len([item for item in updated["text_program"]["items"] if item.get("semantic_entity_id")]), 3) + self.assertNotIn("Background Innovation", {item["text"] for item in updated["text_program"]["items"]}) + def test_exact_paper_labels_and_relations_override_image_interpretation(self): program = { "slots": [ From 72d2d6f375202bb201b811d831bbde0001d1d849 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 05:21:17 +0800 Subject: [PATCH 67/74] Stabilize panel-local paper figure contracts --- rfs/control_localizer.py | 18 +- rfs/paper_to_image/analyzer.py | 375 +++++++++++++++- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/planner.py | 278 +++++++++++- rfs/paper_to_image/preparation.py | 544 ++++++++++++++++++++++- rfs/paper_to_image/semantic_blueprint.py | 201 ++++++++- rfs/paper_to_image/templates.py | 10 + tests/unit/test_control_localizer.py | 19 + tests/unit/test_paper_analyzer.py | 93 +++- tests/unit/test_paper_contract.py | 139 +++++- tests/unit/test_semantic_blueprint.py | 72 +++ 11 files changed, 1685 insertions(+), 66 deletions(-) create mode 100644 tests/unit/test_control_localizer.py diff --git a/rfs/control_localizer.py b/rfs/control_localizer.py index 27761e3..b3c9f19 100644 --- a/rfs/control_localizer.py +++ b/rfs/control_localizer.py @@ -51,6 +51,19 @@ def _sequence_controls(slots: list[dict], palette: list[str]) -> list[dict]: return arrows +def _hough_line_coordinates(line) -> tuple[int, int, int, int] | None: + """Normalize OpenCV HoughLinesP rows across array-shape variants.""" + try: + values = line.reshape(-1).tolist() if hasattr(line, "reshape") else line + while isinstance(values, (list, tuple)) and len(values) == 1 and isinstance(values[0], (list, tuple)): + values = values[0] + if not isinstance(values, (list, tuple)) or len(values) < 4: + return None + return tuple(int(value) for value in values[:4]) + except (TypeError, ValueError): + return None + + def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str]) -> list[dict]: try: import cv2 @@ -67,7 +80,10 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str return [] candidates = [] for line in lines[:80]: - x1, y1, x2, y2 = [int(v) for v in line[0]] + coordinates = _hough_line_coordinates(line) + if coordinates is None: + continue + x1, y1, x2, y2 = coordinates length = math.hypot(x2 - x1, y2 - y1) if length < min(width, height) * 0.06: continue diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index f337c0b..19c98e8 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -40,8 +40,8 @@ } OCR_RENDER_DPI = 84 LATIN_CAPTION_PREFIX = re.compile( - r"^(?Pfigure|fig\.|table|figura|abbildung|tabla|tableau|tabelle)\s*" - r"(?P[a-z]?\d+(?:[.\-]\d+)*|[ivxlcdm]+|[a-z])\b", + r"^(?Pfigure|fig\.?|table|figura|abbildung|tabla|tableau|tabelle)" + r"(?=\s|\d)\s*(?P[a-z]?\d+(?:[.\-]\d+)*|[ivxlcdm]+)\b", re.IGNORECASE, ) @@ -1310,13 +1310,41 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) record = {"page": page["page"], "block_id": block["id"], "bbox": block.get("bbox"), "caption": block.get("text", "")} if block.get("kind") == "caption": parent = block.get("parent_block") - continuation = [] + continuation_blocks = [] for following in page_blocks[block_index + 1:]: if parent is None or following.get("parent_block") != parent or _caption_prefix_kind(str(following.get("text") or "")): break - continuation.append(str(following.get("text") or "")) - if continuation: - record["caption"] = _clean(record["caption"] + " " + " ".join(continuation)) + continuation_blocks.append(following) + + # A caption may continue in the next column at the same vertical + # position while PyMuPDF assigns a different parent block. Recover + # those lines by geometry and font, then keep column-major order. + start_bbox = block.get("bbox") if isinstance(block.get("bbox"), list) else [0, 0, 0, 0] + start_y = float(start_bbox[1] or 0) + start_font = float(block.get("font_size") or 0) + page_height = max(1.0, float(page.get("height") or 1)) + max_y = start_y + max(42.0, page_height * 0.09) + included_ids = {str(block.get("id") or ""), *(str(item.get("id") or "") for item in continuation_blocks)} + for candidate in page_blocks: + candidate_id = str(candidate.get("id") or "") + if not candidate_id or candidate_id in included_ids: + continue + if int(candidate.get("column") if candidate.get("column") is not None else -99) == int(block.get("column") if block.get("column") is not None else -98): + continue + bbox = candidate.get("bbox") if isinstance(candidate.get("bbox"), list) else [0, 0, 0, 0] + candidate_y = float(bbox[1] or 0) + candidate_font = float(candidate.get("font_size") or 0) + if not (start_y - 2.0 <= candidate_y <= max_y): + continue + if start_font and candidate_font and abs(candidate_font - start_font) > 0.8: + continue + if _caption_prefix_kind(str(candidate.get("text") or "")): + continue + continuation_blocks.append(candidate) + included_ids.add(candidate_id) + if continuation_blocks: + continuation_blocks.sort(key=lambda item: (int(item.get("column") or 0), float((item.get("bbox") or [0, 0, 0, 0])[1]), float((item.get("bbox") or [0, 0, 0, 0])[0]))) + record["caption"] = _clean(record["caption"] + " " + " ".join(str(item.get("text") or "") for item in continuation_blocks)) figures.append({"id": f"figure_{len(figures) + 1:03d}", **record}) elif block.get("kind") == "table": tables.append({"id": f"table_{len(tables) + 1:03d}", **record}) @@ -1410,10 +1438,27 @@ def _document_index(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) def _section_coverage(pages: list[dict[str, Any]], sections: list[dict[str, Any]]) -> dict[str, bool]: candidates = ("\n".join(item["title"] for item in sections) + "\n" + "\n".join(page.get("text", "") for page in pages)).casefold() compact = _normalized_compare_text(candidates) - return { + coverage = { name: any(re.search(rf"\b{re.escape(alias)}\b", candidates) or (_normalized_compare_text(alias) and _normalized_compare_text(alias) in compact) for alias in aliases) for name, aliases in SECTION_ALIASES.items() } + # Nature-style articles and several journals print a lead abstract without + # an explicit "Abstract" heading. Treat a substantial first-page research + # summary as abstract coverage when it contains an authorial contribution + # statement, instead of emitting a false extraction warning. + first_page = str(pages[0].get("text") or "") if pages else "" + first_page_compact = _normalized_compare_text(first_page) + implicit_abstract = bool( + len(first_page_compact) >= 700 + and re.search( + r"\b(?:here|in this (?:work|paper|study))\s+(?:we\s+)?(?:present|introduce|propose|develop|show|demonstrate)\b" + r"|\bwe\s+(?:present|introduce|propose|develop|show|demonstrate)\b", + first_page, + re.IGNORECASE, + ) + ) + coverage["abstract"] = bool(coverage.get("abstract") or implicit_abstract) + return coverage def _ocr_latin_lexical_quality(pages: list[dict[str, Any]]) -> dict[str, Any]: @@ -1774,18 +1819,316 @@ def paper_markdown(parsed: dict[str, Any]) -> str: return "\n\n".join(parts).strip() + "\n" +def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 2) -> dict[str, tuple[int, int]]: + """Prioritize labels spatially inside likely overview figures. + + Figure-local labels are often short text blocks with no section heading, so + a section-first evidence sort can push them past the fast planner's character + budget. Recover the visual region around the strongest overview captions + and rank those blocks immediately after their caption. + """ + positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|method|model|system|workflow)\b", re.IGNORECASE) + negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention map)\b", re.IGNORECASE) + ranked: list[tuple[int, int, dict[str, Any]]] = [] + for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): + if not isinstance(figure, dict): + continue + caption = str(figure.get("caption") or "").strip() + if not caption or not positive.search(caption): + continue + score = 8 + if re.match(r"^(?:figure|fig\.)\s*[12]\b", caption, re.IGNORECASE): + score += 5 + if 160 <= len(caption) <= 1600: + score += 4 + if negative.search(caption): + score -= 7 + ranked.append((score, -index, figure)) + candidates = [item for _, _, item in sorted(ranked, reverse=True)[: max(1, int(limit))]] + if not candidates: + return {} + + pages = { + int(page.get("page") or 0): page + for page in parsed.get("pages", []) + if isinstance(page, dict) and page.get("page") + } + evidence = [item for item in parsed.get("evidence", []) if isinstance(item, dict)] + priority: dict[str, tuple[int, int]] = {} + for candidate_index, figure in enumerate(candidates): + page_number = int(figure.get("page") or 0) + page = pages.get(page_number, {}) + page_height = float(page.get("height") or 0.0) + caption = str(figure.get("caption") or "") + caption_record = next( + ( + item + for item in evidence + if int(item.get("page") or 0) == page_number + and str(item.get("kind") or "").casefold() == "caption" + and str(item.get("text") or "").startswith(caption[:60]) + ), + None, + ) + if not caption_record: + continue + caption_id = str(caption_record.get("id") or "") + if caption_id: + priority[caption_id] = (candidate_index * 2, -1) + caption_bbox = caption_record.get("bbox") + if not (isinstance(caption_bbox, list) and len(caption_bbox) == 4 and page_height > 0): + continue + caption_top = float(caption_bbox[1]) + caption_bottom = float(caption_bbox[3]) + caption_center_ratio = ((caption_top + caption_bottom) / 2.0) / page_height + for item in evidence: + if int(item.get("page") or 0) != page_number: + continue + item_id = str(item.get("id") or "") + item_bbox = item.get("bbox") + if not item_id or not (isinstance(item_bbox, list) and len(item_bbox) == 4): + continue + item_top = float(item_bbox[1]) + item_bottom = float(item_bbox[3]) + if caption_center_ratio >= 0.38: + in_visual_region = item_bottom <= caption_bottom + 6.0 and item_top >= max(0.0, caption_top - 0.62 * page_height) + elif caption_center_ratio <= 0.25: + in_visual_region = item_top >= caption_top - 6.0 and item_bottom <= min(page_height, caption_bottom + 0.62 * page_height) + else: + in_visual_region = item_bottom >= caption_top - 0.42 * page_height and item_top <= caption_bottom + 0.42 * page_height + if not in_visual_region: + continue + block_id = str(item.get("block_id") or "") + reading_order = next( + ( + int(block.get("reading_order") or 0) + for block in page.get("blocks", []) + if isinstance(block, dict) and str(block.get("id") or "") == block_id + ), + 0, + ) + value = (candidate_index * 2 + 1, reading_order) + if item_id not in priority or value < priority[item_id]: + priority[item_id] = value + return priority + + +def overview_figure_spatial_excerpt(parsed: dict[str, Any], max_items: int = 180) -> str: + """Return compact bbox-aware evidence for likely overview figure regions.""" + priority = _overview_figure_evidence_priority(parsed) + if not priority: + return "[]" + pages = { + int(page.get("page") or 0): page + for page in parsed.get("pages", []) + if isinstance(page, dict) and page.get("page") + } + records = [] + for item in parsed.get("evidence", []): + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "") + if item_id not in priority: + continue + page_number = int(item.get("page") or 0) + page = pages.get(page_number, {}) + width = max(1.0, float(page.get("width") or 1.0)) + height = max(1.0, float(page.get("height") or 1.0)) + bbox = item.get("bbox") if isinstance(item.get("bbox"), list) and len(item.get("bbox")) == 4 else None + bbox_percent = None + if bbox: + bbox_percent = [ + round(float(bbox[0]) / width, 4), + round(float(bbox[1]) / height, 4), + round(float(bbox[2]) / width, 4), + round(float(bbox[3]) / height, 4), + ] + records.append({ + "id": item_id, + "page": page_number, + "block_id": str(item.get("block_id") or ""), + "kind": str(item.get("kind") or "paragraph"), + "reading_order": priority[item_id][1], + "bbox_percent": bbox_percent, + "text": str(item.get("text") or ""), + "_priority": priority[item_id], + }) + records.sort(key=lambda item: (item["_priority"], item["page"], item["block_id"])) + for item in records: + item.pop("_priority", None) + return json.dumps(records[: max(1, int(max_items))], ensure_ascii=False, separators=(",", ":")) + + +def overview_figure_panel_excerpt(parsed: dict[str, Any], max_items_per_panel: int = 48) -> str: + """Group concise target-figure labels by explicit panel markers.""" + def role_hint(text_value: str) -> str | None: + low = text_value.casefold().strip(" .:") + words = text_value.split() + if len(text_value) > 72 or len(words) > 8 or low.startswith(("we ", "with ", "the vector ", "each node ")): + return None + if re.search(r"\b(loss|objective|optimization)\b", low): + return "objective" + if re.search(r"\b(predictions?|predicted .*distribution|outputs?|results?)\b", low) or re.fullmatch(r"y\s*test", low): + return "output" + # Figure authors often use a determiner-led dataset phrase as a heading + # for a whole visual region (for example, "A synthetic dataset"), not + # as a separate tensor/table node in the data-flow graph. Keep explicit + # operational labels such as "Input dataset" and X_train/Y_test as + # inputs, while treating descriptive noun phrases as group containers. + if re.fullmatch(r"(?:a|an|the)\s+(?:[\w-]+\s+){1,6}datasets?", low): + return "group" + if re.search(r"\b(dataset|input data|input dataset|source data)\b", low) or re.fullmatch(r"[xy]\s*(?:train|test)", low): + return "input" + if re.search(r"\b(layer|block|stack)\b", low) and re.search(r"\b\d+\s*[×x]\b|\(\d+", low): + return "group" + if len(words) <= 5 and re.search(r"\b(model|network|encoder|decoder|attention|mlp|transformer|retriever|generator|classifier|backbone|module|tabpfn)\b", low): + return "module" + if re.fullmatch(r"[A-Z][A-Z0-9-]{2,}", text_value.strip()): + return "module" + return None + + priority = _overview_figure_evidence_priority(parsed, limit=1) + if not priority: + return "{}" + candidate_ids = {item_id for item_id, value in priority.items() if value[0] <= 1} + evidence = [ + item for item in parsed.get("evidence", []) + if isinstance(item, dict) and str(item.get("id") or "") in candidate_ids + ] + caption_record = next((item for item in evidence if str(item.get("kind") or "").casefold() == "caption"), None) + if not caption_record: + return "{}" + caption = str(caption_record.get("text") or "") + panel_matches = list(re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", caption, re.IGNORECASE)) + panel_labels = [match.group(1).casefold() for match in panel_matches] + if len(panel_labels) < 2: + return "{}" + descriptions: dict[str, str] = {} + for index, match in enumerate(panel_matches): + end = panel_matches[index + 1].start() if index + 1 < len(panel_matches) else len(caption) + descriptions[match.group(1).casefold()] = re.sub(r"\s+", " ", caption[match.end():end]).strip(" .") + + page_number = int(caption_record.get("page") or 0) + page = next((item for item in parsed.get("pages", []) if isinstance(item, dict) and int(item.get("page") or 0) == page_number), {}) + width = max(1.0, float(page.get("width") or 1.0)) + height = max(1.0, float(page.get("height") or 1.0)) + caption_bbox = caption_record.get("bbox") if isinstance(caption_record.get("bbox"), list) and len(caption_record.get("bbox")) == 4 else [0.0, height, width, height] + visual_records = [ + item for item in evidence + if item is not caption_record + and isinstance(item.get("bbox"), list) + and len(item.get("bbox")) == 4 + and float(item["bbox"][3]) <= float(caption_bbox[3]) + 6.0 + ] + markers: list[tuple[str, dict[str, Any]]] = [] + for panel_label in panel_labels: + candidates = [item for item in visual_records if str(item.get("text") or "").strip().casefold() == panel_label] + if candidates: + markers.append((panel_label, min(candidates, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))))) + if len(markers) < 2: + return "{}" + markers.sort(key=lambda item: (float(item[1]["bbox"][1]), float(item[1]["bbox"][0]))) + x_centers = [(float(item[1]["bbox"][0]) + float(item[1]["bbox"][2])) / 2.0 for item in markers] + y_centers = [(float(item[1]["bbox"][1]) + float(item[1]["bbox"][3])) / 2.0 for item in markers] + orientation = "vertical" if (max(y_centers) - min(y_centers)) >= (max(x_centers) - min(x_centers)) else "horizontal" + visual_top = min(float(item["bbox"][1]) for item in visual_records) + visual_left = min(float(item["bbox"][0]) for item in visual_records) + visual_right = max(float(item["bbox"][2]) for item in visual_records) + visual_bottom = min(float(caption_bbox[1]), max(float(item["bbox"][3]) for item in visual_records)) + marker_positions = y_centers if orientation == "vertical" else x_centers + panels = [] + for index, (panel_label, marker) in enumerate(markers): + if orientation == "vertical": + start = max(visual_top, float(marker["bbox"][1]) - 0.01 * height) + end = (float(markers[index + 1][1]["bbox"][1]) - 0.01 * height) if index + 1 < len(markers) else visual_bottom + panel_bbox = [visual_left, start, visual_right, end] + else: + start = max(visual_left, float(marker["bbox"][0]) - 0.01 * width) + end = (float(markers[index + 1][1]["bbox"][0]) - 0.01 * width) if index + 1 < len(markers) else visual_right + panel_bbox = [start, visual_top, end, visual_bottom] + labels = [] + for item in visual_records: + text_value = re.sub(r"\s+", " ", str(item.get("text") or "")).strip() + if not text_value or text_value.casefold() in panel_labels or len(text_value) > 140: + continue + if sum(char.isalpha() for char in text_value) < 2: + continue + bbox = item["bbox"] + center_x = (float(bbox[0]) + float(bbox[2])) / 2.0 + center_y = (float(bbox[1]) + float(bbox[3])) / 2.0 + if not (panel_bbox[0] <= center_x <= panel_bbox[2] and panel_bbox[1] <= center_y <= panel_bbox[3]): + continue + hint = role_hint(text_value) + labels.append({ + "evidence_id": str(item.get("id") or ""), + "text": text_value, + "role_hint": hint, + "required_candidate": bool(hint), + "bbox_percent": [ + round(float(bbox[0]) / width, 4), + round(float(bbox[1]) / height, 4), + round(float(bbox[2]) / width, 4), + round(float(bbox[3]) / height, 4), + ], + }) + generic_module_labels = {"model", "network", "neuralnetwork", "encoder", "decoder"} + for item in labels: + if item.get("role_hint") != "module" or _normalized_compare_text(str(item.get("text") or "")) not in generic_module_labels: + continue + bbox = item.get("bbox_percent") or [0.0, 0.0, 0.0, 0.0] + for other in labels: + if other is item or other.get("role_hint") != "module": + continue + other_key = _normalized_compare_text(str(other.get("text") or "")) + if not other_key or other_key in generic_module_labels: + continue + other_bbox = other.get("bbox_percent") or [0.0, 0.0, 0.0, 0.0] + overlap = max(0.0, min(float(bbox[2]), float(other_bbox[2])) - max(float(bbox[0]), float(other_bbox[0]))) + min_width = max(1e-6, min(float(bbox[2]) - float(bbox[0]), float(other_bbox[2]) - float(other_bbox[0]))) + vertical_gap = float(bbox[1]) - float(other_bbox[3]) + if overlap / min_width >= 0.6 and -0.004 <= vertical_gap <= 0.018: + item["role_hint"] = "supporting" + item["required_candidate"] = False + break + panels.append({ + "panel_label": panel_label, + "description": descriptions.get(panel_label, ""), + "bbox_percent": [ + round(panel_bbox[0] / width, 4), + round(panel_bbox[1] / height, 4), + round(panel_bbox[2] / width, 4), + round(panel_bbox[3] / height, 4), + ], + "label_inventory": labels[: max(1, int(max_items_per_panel))], + }) + return json.dumps({ + "summary": "Concise panel-local label inventory recovered from explicit figure panel markers.", + "page": page_number, + "orientation": orientation, + "panels": panels, + }, ensure_ascii=False, separators=(",", ":")) + + def evidence_excerpt(parsed: dict, max_chars: int = 58000) -> str: + overview_priority = _overview_figure_evidence_priority(parsed) + + def priority_key(item: dict[str, Any]) -> tuple[int, int, int, str]: + item_id = str(item.get("id") or "") + if item_id in overview_priority: + local_tier, reading_order = overview_priority[item_id] + return local_tier, reading_order, int(item.get("page") or 0), str(item.get("block_id") or "") + tier = ( + 10 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else + 11 if item.get("kind") == "caption" and re.search(r"\b(overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else + 12 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) else + 13 if item.get("kind") == "caption" else + 14 if any(term in str(item.get("section_hint") or "").casefold() for term in ("abstract", "introduction", "method", "approach", "architecture", "framework", "conclusion")) else 15 + ) + return tier, 0, int(item.get("page") or 0), str(item.get("block_id") or "") + prioritized = sorted( parsed.get("evidence", []), - key=lambda item: ( - 0 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) and re.search(r"\b(components?|overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else - 1 if item.get("kind") == "caption" and re.search(r"\b(overview|framework|architecture|pipeline)\b", str(item.get("text") or ""), re.IGNORECASE) else - 2 if item.get("kind") == "caption" and re.match(r"^(figure|fig\.)\s*1\b", str(item.get("text") or ""), re.IGNORECASE) else - 3 if item.get("kind") == "caption" else - 4 if any(term in str(item.get("section_hint") or "").casefold() for term in ("abstract", "introduction", "method", "approach", "architecture", "framework", "conclusion")) else 5, - int(item.get("page") or 0), - str(item.get("block_id") or ""), - ), + key=priority_key, ) lines: list[str] = [] used = 0 diff --git a/rfs/paper_to_image/document_cache.py b/rfs/paper_to_image/document_cache.py index 723536c..f9dc990 100644 --- a/rfs/paper_to_image/document_cache.py +++ b/rfs/paper_to_image/document_cache.py @@ -9,7 +9,7 @@ from ..utils import read_json, write_json -DOCUMENT_CACHE_VERSION = 31 +DOCUMENT_CACHE_VERSION = 32 def document_model_cacheable(parsed: dict[str, Any]) -> bool: diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index af30556..e2b9da0 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -6,7 +6,7 @@ from typing import Any from ..vlm_client import call_vlm_json, resolve_vlm_model, vlm_credentials_available -from .analyzer import evidence_excerpt +from .analyzer import evidence_excerpt, overview_figure_panel_excerpt, overview_figure_spatial_excerpt DEFAULT_PREFERENCES = { @@ -86,6 +86,14 @@ def _planner_prompt(parsed: dict, preferences: dict, paper_review: dict | None = "outputs": [{{"id": "stable_output_id", "name": "exact paper term", "evidence_ids": ["E0001"]}}], "innovations": [], "feedback_loops": [], + "panel_graphs": [{{ + "id": "panel_a", + "panel_label": "a", + "label": "exact panel meaning", + "evidence_ids": [], + "nodes": [{{"instance_id": "panel_a_unique_instance", "name": "exact visible term", "role": "input|module|objective|output|group", "evidence_ids": []}}], + "relations": [{{"source": "panel_a_unique_instance", "target": "panel_a_other_instance", "type": "data_flow|conditioning|prediction|training_objective|feedback", "label": "", "evidence_ids": []}}] + }}], "training_flow": [], "inference_flow": [], "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", @@ -172,7 +180,7 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in You are compiling a scientific paper into a directed semantic graph for ONE framework figure. Return JSON only. Focus exclusively on scientific meaning; layout, illustration style, and image-generation wording will be compiled later by code. Primary task: -1. Select the most information-rich method/architecture/framework figure caption from the candidates below. Figure 1 is not automatically best; prefer a caption that explicitly names components and directed operations. +1. Select the most information-rich method/architecture/framework figure caption from the candidates below. Figure 1 is not automatically best; prefer a caption that explicitly names components and directed operations. However, when an overview caption explicitly declares panels such as "a, high-level pre-training and usage" and "b, architecture", treat all declared panels as one indivisible target figure. Use detailed architecture or method evidence to enrich those panels, never to replace or omit one of them. 2. Extract every separately named input, component, intermediate representation, conditioning signal, training-only objective, inference-only step, and output needed to reproduce that overview. 3. Use exact short paper terms as visible names. Do not replace "image encoder" with a prose description such as "network that extracts image features". A visible name must be copied verbatim from one of its cited evidence blocks. Preserve the paper's original language and writing system; never translate a Chinese, Japanese, Korean, or other non-English term because the user preference says English. If no concise source phrase exists, use the shortest verbatim source-language phrase. Relation labels must follow the same rule or be left blank. @@ -181,6 +189,13 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in 6. Separate training and inference. Matching, losses, supervision, optimization, and feedback belong in training_flow or training-only entities; test-time classification, decoding, retrieval, rendering, or deployment belong in inference_flow. 7. Every factual item must cite evidence_ids copied exactly from the evidence. Unsupported details must be omitted and recorded under uncertainties. 8. Ignore baseline/comparison methods mentioned only to contrast with the proposed method. Never import terms from references, acknowledgements, bibliography, or result-only figures. +9. For a multi-panel overview, set topology to dense_multiframe and include evidence-backed high-level training/usage stages as well as internal architecture modules. Do not collapse a training-versus-usage overview into only the internal neural-network layers. +10. Treat the bbox-aware figure-local blocks below as direct visual evidence. Short labels inside the target figure outrank generic architecture-family prose. A phrase such as "adaptation of the standard transformer encoder" is a description, not a separate visible module, unless "Transformer Encoder" itself appears as a short figure label or is required as an explicit operational stage. +11. Reconstruct each declared panel as a separate local graph. Do not connect a high-level workflow panel directly into an internal architecture panel merely because both describe the same model. Add cross-panel relations only when the evidence explicitly states that edge. +12. Preserve exact operational order. When a panel contains exact component labels and the paper describes them as followed-by or transformed-by stages, include every intermediate relation instead of jumping from a container title directly to the output. +13. Audit every panel against the concise panel-local inventory. A high-level workflow panel is incomplete if it contains only tensor variables while the inventory also shows a named model/process box, prediction, objective/loss, dataset, or deployment stage. Architecture-family prose must not substitute for those exact panel-local objects. +14. For every declared multi-panel overview, populate panel_graphs. Each panel-local visual occurrence gets a unique instance_id, so repeated labels such as the same model used in pre-training and inference remain separate instances. panel_graph relations must stay within their panel; use the global graph only as a compatibility summary. +15. In the concise panel-local inventory, every entry marked required_candidate=true must map to a panel_graph node using its role_hint and evidence_id. Do not merge two repeated occurrences with different evidence IDs into one instance. Group/container labels may be isolated groups, but operational inputs, modules, objectives, and outputs must have their directed panel-local relations. Return exactly this compact schema: {{ @@ -213,6 +228,14 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in "relations": [{{"source": "...", "target": "...", "type": "data_flow|encoding|conditioning|branch|alignment|prediction|training_objective|rendering_input|feedback", "label": "", "evidence_ids": []}}], "innovations": [], "feedback_loops": [], + "panel_graphs": [{{ + "id": "panel_a", + "panel_label": "a", + "label": "exact panel meaning", + "evidence_ids": [], + "nodes": [{{"instance_id": "panel_a_unique_instance", "name": "exact visible term", "role": "input|module|objective|output|group", "evidence_ids": []}}], + "relations": [{{"source": "panel_a_unique_instance", "target": "panel_a_other_instance", "type": "data_flow|conditioning|prediction|training_objective|feedback", "label": "", "evidence_ids": []}}] + }}], "training_flow": [], "inference_flow": [], "topology": "linear|branch|feedback|multimodal|dense_multiframe|unknown", @@ -234,6 +257,12 @@ def _fast_planner_prompt(parsed: dict, preferences: dict, evidence_max_chars: in Prioritized paper evidence: {evidence_excerpt(parsed, max_chars=evidence_max_chars)} + +BBox-aware target overview figure blocks (normalized [x0,y0,x1,y1], page-local; use for panel membership and visible-label selection, not as permission to invent arrows): +{overview_figure_spatial_excerpt(parsed)} + +Concise panel-local label inventory (use this as a completeness checklist for each declared panel): +{overview_figure_panel_excerpt(parsed)} """.strip() @@ -507,9 +536,15 @@ def plan_paper_image(parsed: dict, preferences: dict, mode: str = "vlm", model: return normalize_plan(_heuristic_plan(parsed, preferences, paper_review=paper_review), preferences), metadata -def collect_visible_labels(spec: dict, limit: int = 24) -> list[str]: +def _item_label_for_prompt(item: dict[str, Any]) -> str: + return str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("label") or item.get("id") or "").strip() + + +def collect_visible_labels(spec: dict, limit: int = 40) -> list[str]: labels: list[str] = [] normalized_labels: set[str] = set() + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + panel_graph_active = len(panel_graphs) >= 2 def add(value: object) -> None: if isinstance(value, dict): @@ -529,19 +564,57 @@ def add(value: object) -> None: elif isinstance(terminology, list): for item in terminology: add(item) - for field in ("inputs", "modules", "outputs"): - for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: - add(item) - for item in spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []: - if isinstance(item, dict) and (item.get("visible_label_required") or item.get("must_appear_in_figure")): - add(item) - for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: - if isinstance(relation, dict): - add(relation.get("label")) + if not panel_graph_active: + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + add(item) + for item in spec.get("innovations", []) if isinstance(spec.get("innovations"), list) else []: + if isinstance(item, dict) and (item.get("visible_label_required") or item.get("must_appear_in_figure")): + add(item) + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if isinstance(relation, dict): + add(relation.get("label")) + for panel in panel_graphs: + if not isinstance(panel, dict): + continue + add(panel.get("label")) + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + add(node) + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if isinstance(relation, dict): + add(relation.get("label")) return labels[: max(1, int(limit))] def collect_visual_relations(spec: dict) -> list[dict[str, str]]: + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(panel_graphs) >= 2: + result: list[dict[str, str]] = [] + for panel in panel_graphs: + panel_id = str(panel.get("id") or "") + panel_label = str(panel.get("panel_label") or panel_id) + nodes = { + str(item.get("instance_id") or item.get("id") or ""): item + for item in panel.get("nodes", []) if isinstance(item, dict) + } + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + if source not in nodes or target not in nodes or source == target: + continue + result.append({ + "source": source, + "source_label": _item_label_for_prompt(nodes[source]), + "target": target, + "target_label": _item_label_for_prompt(nodes[target]), + "type": str(relation.get("type") or "data_flow"), + "label": str(relation.get("label") or "").strip(), + "panel_id": panel_id, + "panel_label": panel_label, + }) + return result entities = { str(item.get("id")): str(item.get("visible_label") or item.get("name") or item.get("text") or item.get("statement") or item.get("id")) for field in ("inputs", "modules", "outputs", "innovations") @@ -584,6 +657,21 @@ def collect_visual_relations(spec: dict) -> list[dict[str, str]]: def collect_visual_roles(spec: dict) -> list[dict[str, str]]: + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(panel_graphs) >= 2: + return [ + { + "id": str(item.get("instance_id") or item.get("id") or ""), + "label": _item_label_for_prompt(item), + "field": "panel_instance", + "role": str(item.get("role") or "module"), + "visual_constraint": "Keep this occurrence inside its declared panel and connect it only through panel-local relations.", + "visual_cue": "A compact paper-grounded visual object matching this panel-local role.", + "panel_id": str(panel.get("id") or ""), + } + for panel in panel_graphs + for item in panel.get("nodes", []) if isinstance(item, dict) + ] result: list[dict[str, str]] = [] label_to_index: dict[str, int] = {} for field in ("inputs", "modules", "outputs", "innovations"): @@ -713,17 +801,79 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = f"{index}. {label}" + (" (may repeat to show evidence-supported component reuse)" if re.sub(r"[^\w\u4e00-\u9fff]+", "", label.casefold()) in repeatable_normalized else "") for index, label in enumerate(labels, start=1) ) - visual_relations = collect_visual_relations(spec) + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + panel_graph_active = len(panel_graphs) >= 2 + visual_relations = [] if panel_graph_active else collect_visual_relations(spec) visual_roles = collect_visual_roles(spec) - connector_checklist = "\n".join( + panel_instance_lines: list[str] = [] + panel_connector_lines: list[str] = [] + for panel in panel_graphs: + panel_id = str(panel.get("id") or "panel") + panel_label = str(panel.get("panel_label") or panel_id) + node_by_id = { + str(item.get("instance_id") or item.get("id") or ""): item + for item in panel.get("nodes", []) if isinstance(item, dict) + } + for item in node_by_id.values(): + instance_id = str(item.get("instance_id") or item.get("id") or "") + panel_instance_lines.append(f"- Panel {panel_label}: {instance_id} = {_item_label_for_prompt(item)} [{str(item.get('role') or 'module')}]" ) + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if not isinstance(relation, dict): + continue + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + if source not in node_by_id or target not in node_by_id: + continue + source_label = _item_label_for_prompt(node_by_id[source]) + target_label = _item_label_for_prompt(node_by_id[target]) + relation_label = str(relation.get("label") or "").strip() + panel_connector_lines.append( + f"{len(panel_connector_lines) + 1}. Panel {panel_label}: {source_label} ({source}) -> {target_label} ({target}) [{str(relation.get('type') or 'data_flow')}]" + + (f" label: {relation_label}" if relation_label else "") + ) + connector_checklist = "\n".join(panel_connector_lines) if panel_graph_active else "\n".join( f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" + (f" label: {item['label']}" if item["label"] else "") for index, item in enumerate(visual_relations, start=1) ) + input_roles = [item for item in visual_roles if item.get("role") == "input"] + output_roles = [item for item in visual_roles if item.get("role") == "output"] + relations_by_target: dict[str, list[dict]] = {} + for relation in visual_relations: + relations_by_target.setdefault(str(relation.get("target") or ""), []).append(relation) + shared_input_rules = [] + for target_id, incoming_relations in relations_by_target.items(): + peer_inputs = [item for item in incoming_relations if any(role.get("id") == item.get("source") for role in input_roles)] + if len(peer_inputs) < 2: + continue + target_label = str(peer_inputs[0].get("target_label") or target_id) + source_labels = [str(item.get("source_label") or item.get("source")) for item in peer_inputs] + shared_input_rules.append( + f"- Treat {', '.join(source_labels)} as peer input sources. Never draw arrows between these peer cards. Draw one separate directed arrow from each peer input to {target_label}." + ) + output_labels = [str(item.get("label") or item.get("id")) for item in output_roles] + boundary_role_rules = ("- Panel-local instance roles and panel-local connector checklists are authoritative. Never connect instances across panels unless a separate explicit cross-panel relation is listed." if panel_graph_active else "\n".join([ + *shared_input_rules, + *( [f"- Treat {', '.join(output_labels)} as output-only nodes. Never draw an arrow from an output back into an input or method module unless an explicit feedback relation appears in the connector checklist."] if output_labels else [] ), + ]) or "- Keep all declared inputs as source roles and all declared outputs as sink roles unless an explicit feedback relation says otherwise.") topology_specific_rules = _topology_specific_prompt_rules(spec) - compact_roles = [ - {"id": item["id"], "label": item["label"], "role": item["role"], "visual_cue": item["visual_cue"]} - for item in visual_roles - ] + compact_roles = ( + [ + { + "id": str(item.get("instance_id") or item.get("id") or ""), + "label": _item_label_for_prompt(item), + "role": str(item.get("role") or "module"), + "panel_id": str(panel.get("id") or ""), + "visual_cue": "A compact paper-grounded visual object matching this panel-local role.", + } + for panel in panel_graphs + for item in panel.get("nodes", []) if isinstance(item, dict) + ] + if panel_graph_active else + [ + {"id": item["id"], "label": item["label"], "role": item["role"], "visual_cue": item["visual_cue"]} + for item in visual_roles + ] + ) compact_contract = { "figure_goal": spec.get("figure_goal"), "central_claim": spec.get("central_claim", {}).get("text") if isinstance(spec.get("central_claim"), dict) else spec.get("central_claim"), @@ -732,7 +882,16 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = "inference_flow": spec.get("inference_flow", []), "forbidden_inventions": spec.get("forbidden_inventions", []), "uncertainties": spec.get("uncertainties", []), - "overview_panels": [ + "overview_panels": ([ + { + "id": item.get("id"), + "panel_label": item.get("panel_label"), + "label": item.get("label"), + "description": item.get("description"), + "entity_ids": [str(node.get("instance_id") or node.get("id") or "") for node in item.get("nodes", []) if isinstance(node, dict)], + } + for item in panel_graphs + ] if panel_graph_active else [ { "id": item.get("id"), "panel_label": item.get("panel_label"), @@ -741,6 +900,31 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = "entity_ids": item.get("entity_ids", []), } for item in spec.get("overview_panels", []) if isinstance(item, dict) + ]), + "panel_graphs": [ + { + "id": panel.get("id"), + "panel_label": panel.get("panel_label"), + "label": panel.get("label"), + "nodes": [ + { + "instance_id": item.get("instance_id") or item.get("id"), + "name": _item_label_for_prompt(item), + "role": item.get("role"), + } + for item in panel.get("nodes", []) if isinstance(item, dict) + ], + "relations": [ + { + "source": item.get("source"), + "target": item.get("target"), + "type": item.get("type"), + "label": item.get("label"), + } + for item in panel.get("relations", []) if isinstance(item, dict) + ], + } + for panel in panel_graphs ], "supporting_visual_cues": [ { @@ -776,6 +960,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = "panel_label": item.get("panel_label"), "label": item.get("label"), "bbox_percent": item.get("bbox_percent"), + "groups": item.get("groups", []), } for item in semantic_plan.get("panels", []) if isinstance(item, dict) ], @@ -838,6 +1023,14 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows. +Boundary-role anti-hallucination rules: +{boundary_role_rules} + +Mandatory panel-local instance checklist: +{chr(10).join(panel_instance_lines) if panel_instance_lines else '- No separate panel-local instance graph; use the global role checklist.'} + +When the panel-local instance checklist is present, it is the authoritative visible occurrence model. Repeated labels are separate visible instances with different instance IDs. The global entity graph is only a compatibility summary and must not create extra cross-panel arrows or replace panel-local nodes. + Mandatory entity role and containment checklist: {json.dumps(compact_roles, ensure_ascii=False, separators=(',', ':'))} @@ -1015,6 +1208,52 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * errors.append(f"relation {source} -> {target} references unknown evidence ids: {invalid}") validate_visible_label(f"relation {source} -> {target}", str(relation.get("label") or ""), evidence_ids) + panel_graph_relation_count = 0 + panel_instance_ids: set[str] = set() + for panel_index, panel in enumerate(spec.get("panel_graphs") if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + errors.append(f"panel_graphs[{panel_index}] is not an object") + continue + panel_evidence_ids = panel.get("evidence_ids") if isinstance(panel.get("evidence_ids"), list) else [] + invalid_panel_evidence = [value for value in panel_evidence_ids if value not in valid_evidence] + if invalid_panel_evidence: + errors.append(f"panel_graphs[{panel_index}] references unknown evidence ids: {invalid_panel_evidence}") + local_ids: set[str] = set() + for node_index, node in enumerate(panel.get("nodes") if isinstance(panel.get("nodes"), list) else []): + if not isinstance(node, dict): + errors.append(f"panel_graphs[{panel_index}].nodes[{node_index}] is not an object") + continue + instance_id = str(node.get("instance_id") or node.get("id") or "").strip() + if not instance_id: + errors.append(f"panel_graphs[{panel_index}].nodes[{node_index}] has no instance_id") + elif instance_id in panel_instance_ids: + errors.append(f"duplicate panel instance id: {instance_id}") + else: + panel_instance_ids.add(instance_id) + local_ids.add(instance_id) + node_evidence_ids = node.get("evidence_ids") if isinstance(node.get("evidence_ids"), list) else [] + if not node_evidence_ids: + errors.append(f"panel instance {instance_id or node_index} has no evidence_ids") + invalid_node_evidence = [value for value in node_evidence_ids if value not in valid_evidence] + if invalid_node_evidence: + errors.append(f"panel instance {instance_id or node_index} references unknown evidence ids: {invalid_node_evidence}") + validate_visible_label(f"panel instance {instance_id or node_index}", _item_label_for_prompt(node), node_evidence_ids) + for relation_index, panel_relation in enumerate(panel.get("relations") if isinstance(panel.get("relations"), list) else []): + if not isinstance(panel_relation, dict): + errors.append(f"panel_graphs[{panel_index}].relations[{relation_index}] is not an object") + continue + source = str(panel_relation.get("source") or "").strip() + target = str(panel_relation.get("target") or "").strip() + if source not in local_ids or target not in local_ids: + errors.append(f"panel_graphs[{panel_index}] relation has non-local endpoint: {source} -> {target}") + relation_evidence_ids = panel_relation.get("evidence_ids") if isinstance(panel_relation.get("evidence_ids"), list) else [] + if not relation_evidence_ids: + errors.append(f"panel relation {source} -> {target} has no evidence_ids") + invalid_relation_evidence = [value for value in relation_evidence_ids if value not in valid_evidence] + if invalid_relation_evidence: + errors.append(f"panel relation {source} -> {target} references unknown evidence ids: {invalid_relation_evidence}") + panel_graph_relation_count += 1 + terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} for source_term, visible_label in terminology.items(): validate_visible_label(f"terminology[{source_term}]", str(visible_label or ""), [], fallback_text=all_evidence_text) @@ -1048,6 +1287,7 @@ def validate_visible_label(location: str, label: str, evidence_ids: list[str], * "ok": not errors, "module_count": len(modules), "relation_count": len(relations), + "panel_graph_relation_count": panel_graph_relation_count, "evidence_count": len(valid_evidence), "errors": errors, "warnings": warnings, diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index a778186..d99f490 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -10,7 +10,7 @@ from typing import Any, Callable from ..utils import ensure_dir, read_json, write_json, write_text -from .analyzer import paper_markdown, parse_paper +from .analyzer import overview_figure_panel_excerpt, paper_markdown, parse_paper from .contract_completion import augment_contract_from_evidence from .document_cache import DOCUMENT_CACHE_VERSION, read_document_cache, write_document_cache from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_fast_paper_contract, plan_paper_image, validate_plan_grounding @@ -157,8 +157,16 @@ def _repair_cross_script_entity_labels(spec: dict[str, Any], parsed: dict[str, A if isinstance(item, dict) and item.get("id") } repaired: list[str] = [] - for field in ("inputs", "modules", "outputs", "innovations"): - for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + collections: list[tuple[str, list[Any]]] = [ + (field, spec.get(field, []) if isinstance(spec.get(field), list) else []) + for field in ("inputs", "modules", "outputs", "innovations") + ] + collections.extend( + (f"panel_graphs.{panel.get('id')}.nodes", panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []) + for panel in spec.get("panel_graphs", []) if isinstance(panel, dict) + ) + for field, items in collections: + for item in items: if not isinstance(item, dict): continue label = _item_label(item) @@ -168,6 +176,23 @@ def _repair_cross_script_entity_labels(spec: dict[str, Any], parsed: dict[str, A source_cjk, source_latin = _script_counts(source) if not (label_cjk >= 1 and label_latin >= 2 and source_cjk == 0 and source_latin >= 4): continue + label_without_cjk = re.sub(r"[\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]+", "", label) + label_key = _normalized_label(label_without_cjk) + exact_record = next( + ( + str(record).strip() + for record in sorted(records, key=len) + if str(record).strip() + and len(str(record).strip()) <= max(120, len(label) + 24) + and _normalized_label(str(record)) == label_key + ), + None, + ) + if exact_record: + item["name"] = exact_record + item.pop("visible_label", None) + repaired.append(f"{item.get('id') or item.get('instance_id')}:{label}->{exact_record}") + continue tokens = re.findall(r"[A-Za-z]+|\d+(?:\.\d+)?", label) if len(tokens) < 2: continue @@ -180,7 +205,7 @@ def _repair_cross_script_entity_labels(spec: dict[str, Any], parsed: dict[str, A continue item["name"] = exact item.pop("visible_label", None) - repaired.append(f"{item.get('id')}:{label}->{exact}") + repaired.append(f"{item.get('id') or item.get('instance_id')}:{label}->{exact}") return repaired @@ -218,6 +243,54 @@ def _ground_exact_visible_entities(spec: dict[str, Any], parsed: dict[str, Any]) return grounded +def _repair_relation_evidence_after_grounding(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: + """Attach conservative endpoint evidence after late exact-label grounding.""" + endpoints = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + evidence_by_id = { + str(item.get("id")): item + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + repaired: list[str] = [] + for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: + if not isinstance(relation, dict) or relation.get("evidence_ids"): + continue + source = endpoints.get(str(relation.get("source") or "")) + target = endpoints.get(str(relation.get("target") or "")) + if not source or not target: + continue + source_ids = [str(value) for value in source.get("evidence_ids", []) if str(value) in evidence_by_id] + target_ids = [str(value) for value in target.get("evidence_ids", []) if str(value) in evidence_by_id] + candidate_ids = list(dict.fromkeys([*source_ids, *target_ids])) + if not candidate_ids: + continue + source_pages = {int(evidence_by_id[value].get("page") or 0) for value in source_ids} + target_pages = {int(evidence_by_id[value].get("page") or 0) for value in target_ids} + nearby = bool(source_pages and target_pages and min(abs(left - right) for left in source_pages for right in target_pages) <= 1) + source_label = _normalized_label(_item_label(source)) + target_label = _normalized_label(_item_label(target)) + bridge_ids = [ + str(item["id"]) + for item in parsed.get("evidence", []) + if isinstance(item, dict) + and item.get("id") + and source_label + and target_label + and source_label in _normalized_label(str(item.get("text") or "")) + and target_label in _normalized_label(str(item.get("text") or "")) + ][:2] + if not nearby and not bridge_ids: + continue + relation["evidence_ids"] = list(dict.fromkeys([*candidate_ids, *bridge_ids])) + repaired.append(f"{relation.get('source')}->{relation.get('target')}") + return repaired + + def _prune_out_of_scope_entities(spec: dict[str, Any], parsed: dict[str, Any], completion_report: dict[str, Any]) -> list[str]: selected_ids = {str(value) for value in completion_report.get("selected_overview_evidence_ids", []) if str(value)} if not selected_ids: @@ -405,7 +478,7 @@ def _detect_dense_overview_panels(parsed: dict[str, Any]) -> dict[str, Any]: labels.update(match.group(1).casefold() for match in re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s+", caption, re.IGNORECASE)) ordered = sorted(labels) return { - "detected": len(ordered) >= 3, + "detected": len(ordered) >= 2, "page": page_number, "panel_labels": ordered, "panel_count": len(ordered), @@ -515,11 +588,11 @@ def _extract_overview_panels(parsed: dict[str, Any], spec: dict[str, Any]) -> li evidence_id = evidence_by_block.get(str(block.get("id") or "")) if evidence_id: evidence_ids.setdefault(current, []).append(evidence_id) - if len(descriptions) < 3: + if len(descriptions) < 2: return [] entities = [ - item + (field, item) for field in ("inputs", "modules", "outputs", "innovations") for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) if isinstance(item, dict) and item.get("id") @@ -534,7 +607,7 @@ def _extract_overview_panels(parsed: dict[str, Any], spec: dict[str, Any]) -> li panel_evidence_set = set(panel_evidence) entity_ids = [ str(item.get("id")) - for item in entities + for _, item in entities if panel_evidence_set.intersection(str(value) for value in item.get("evidence_ids", []) if value) ] panels.append({ @@ -546,7 +619,58 @@ def _extract_overview_panels(parsed: dict[str, Any], spec: dict[str, Any]) -> li "evidence_ids": panel_evidence, "entity_ids": list(dict.fromkeys(entity_ids)), }) - return panels if len(panels) >= 3 else [] + if len(panels) < 2: + return [] + + # Caption evidence and exact entity evidence often use different blocks. + # Assign otherwise-unbound entities by semantic panel role so a two-panel + # overview does not place the complete architecture into the high-level + # training/usage panel by default. + assigned_ids = {entity_id for panel in panels for entity_id in panel.get("entity_ids", [])} + architecture_terms = ("architecture", "internal", "layer", "encoder", "decoder", "network", "model structure") + high_level_terms = ("overview", "workflow", "pipeline", "pre-training", "pretraining", "training", "usage", "inference", "application") + training_terms = ("train", "pretrain", "loss", "objective", "synthetic", "supervision") + inference_terms = ("usage", "inference", "prediction", "application", "forward pass", "test") + for field, item in entities: + item_id = str(item.get("id") or "") + if not item_id or item_id in assigned_ids: + continue + label = f"{_item_label(item)} {item.get('role', '')}".casefold() + item_evidence = {str(value) for value in item.get("evidence_ids", []) if value} + scored: list[tuple[int, int]] = [] + for index, panel in enumerate(panels): + panel_text = f"{panel.get('label', '')} {panel.get('description', '')}".casefold() + score = 0 + if item_evidence.intersection(str(value) for value in panel.get("evidence_ids", []) if value): + score += 100 + normalized_label = _normalized_label(_item_label(item)) + if normalized_label and normalized_label in _normalized_label(panel_text): + score += 12 + if field == "modules": + if any(term in panel_text for term in architecture_terms): + score += 5 + if any(term in label for term in training_terms) and any(term in panel_text for term in training_terms): + score += 7 + if any(term in label for term in inference_terms) and any(term in panel_text for term in inference_terms): + score += 7 + elif field == "inputs": + if any(term in panel_text for term in high_level_terms): + score += 4 + if any(term in panel_text for term in ("input", "data", "train", "pretrain")): + score += 3 + elif field == "outputs": + if any(term in panel_text for term in high_level_terms): + score += 4 + if any(term in panel_text for term in inference_terms): + score += 3 + scored.append((score, index)) + best_score, best_index = max(scored, default=(0, 0)) + if best_score > 0: + panels[best_index]["entity_ids"].append(item_id) + assigned_ids.add(item_id) + for panel in panels: + panel["entity_ids"] = list(dict.fromkeys(panel.get("entity_ids", []))) + return panels def _merge_isolated_architecture_entities(spec: dict[str, Any], parsed: dict[str, Any]) -> list[str]: @@ -866,6 +990,248 @@ def _normalize_contract_entities(raw_items: Any, field: str) -> list[dict[str, A return normalized +def _normalize_panel_graphs(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + evidence_ids = { + str(item.get("id")) + for item in parsed.get("evidence", []) + if isinstance(item, dict) and item.get("id") + } + normalized_panels: list[dict[str, Any]] = [] + removed_nodes: list[str] = [] + removed_relations: list[str] = [] + for panel_index, raw_panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(raw_panel, dict): + continue + panel_id = str(raw_panel.get("id") or f"panel_{panel_index + 1}").strip() + panel_label = str(raw_panel.get("panel_label") or chr(ord("a") + panel_index)).strip() + label = str(raw_panel.get("label") or raw_panel.get("description") or panel_label).strip() + if not panel_id or not label: + continue + nodes: list[dict[str, Any]] = [] + seen_instance_ids: set[str] = set() + for node_index, raw_node in enumerate(raw_panel.get("nodes", []) if isinstance(raw_panel.get("nodes"), list) else []): + if not isinstance(raw_node, dict): + continue + node = dict(raw_node) + node_label = _item_label(node) + instance_id = str(node.get("instance_id") or node.get("id") or _stable_id(f"{panel_id}_instance", node_label, node_index)).strip() + if not node_label or not instance_id or instance_id in seen_instance_ids: + removed_nodes.append(f"{panel_id}:{instance_id or node_index}") + continue + node["instance_id"] = instance_id + node["name"] = node_label + node["role"] = str(node.get("role") or "module").strip() + node["evidence_ids"] = [ + str(value) for value in node.get("evidence_ids", []) + if str(value) in evidence_ids + ] + nodes.append(node) + seen_instance_ids.add(instance_id) + node_by_id = {str(item.get("instance_id")): item for item in nodes} + relations: list[dict[str, Any]] = [] + seen_relations: set[tuple[str, str, str]] = set() + for raw_relation in raw_panel.get("relations", []) if isinstance(raw_panel.get("relations"), list) else []: + if not isinstance(raw_relation, dict): + continue + source = str(raw_relation.get("source") or raw_relation.get("source_id") or "").strip() + target = str(raw_relation.get("target") or raw_relation.get("target_id") or "").strip() + relation_type = str(raw_relation.get("type") or "data_flow").strip() + key = (source, target, relation_type) + if not source or not target or source == target or source not in node_by_id or target not in node_by_id or key in seen_relations: + removed_relations.append(f"{panel_id}:{source}->{target}:{relation_type}") + continue + relation = dict(raw_relation) + relation["source"] = source + relation["target"] = target + relation["type"] = relation_type + relation["label"] = str(relation.get("label") or "").strip() + relation["evidence_ids"] = [ + str(value) for value in relation.get("evidence_ids", []) + if str(value) in evidence_ids + ] + if not relation["evidence_ids"]: + relation["evidence_ids"] = list(dict.fromkeys([ + *node_by_id[source].get("evidence_ids", []), + *node_by_id[target].get("evidence_ids", []), + ])) + relations.append(relation) + seen_relations.add(key) + normalized_panels.append({ + "id": panel_id, + "panel_label": panel_label, + "label": label, + "description": str(raw_panel.get("description") or label).strip(), + "evidence_ids": [str(value) for value in raw_panel.get("evidence_ids", []) if str(value) in evidence_ids], + "nodes": nodes, + "relations": relations, + }) + spec["panel_graphs"] = normalized_panels + return { + "panel_count": len(normalized_panels), + "node_count": sum(len(item["nodes"]) for item in normalized_panels), + "relation_count": sum(len(item["relations"]) for item in normalized_panels), + "removed_nodes": removed_nodes, + "removed_relations": removed_relations, + } + + +def _complete_panel_graphs_from_inventory(spec: dict[str, Any], parsed: dict[str, Any]) -> dict[str, Any]: + try: + inventory = json.loads(overview_figure_panel_excerpt(parsed)) + except (TypeError, ValueError, json.JSONDecodeError): + inventory = {} + graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict)] if isinstance(spec.get("panel_graphs"), list) else [] + panels = [item for item in inventory.get("panels", []) if isinstance(item, dict)] if isinstance(inventory, dict) else [] + if len(graphs) < 2 or not panels: + return {"applied": False, "added_nodes": [], "rebuilt_panels": [], "required_candidate_coverage": None} + + added_nodes: list[str] = [] + removed_nodes: list[str] = [] + rebuilt_panels: list[str] = [] + required_total = 0 + covered_total = 0 + + def center(node: dict[str, Any]) -> tuple[float, float]: + bbox = node.get("bbox_percent") if isinstance(node.get("bbox_percent"), list) and len(node.get("bbox_percent")) == 4 else [0.0, 0.0, 0.0, 0.0] + return (float(bbox[0]) + float(bbox[2])) / 2.0, (float(bbox[1]) + float(bbox[3])) / 2.0 + + def relation(source: dict[str, Any], target: dict[str, Any], relation_type: str) -> dict[str, Any]: + return { + "source": str(source.get("instance_id")), + "target": str(target.get("instance_id")), + "type": relation_type, + "label": "", + "evidence_ids": list(dict.fromkeys([*source.get("evidence_ids", []), *target.get("evidence_ids", [])])), + } + + inventory_by_label = {str(item.get("panel_label") or "").casefold(): item for item in panels} + for graph in graphs: + panel_label = str(graph.get("panel_label") or "").casefold() + panel_inventory = inventory_by_label.get(panel_label) + if not panel_inventory: + continue + required_items = [item for item in panel_inventory.get("label_inventory", []) if isinstance(item, dict) and item.get("required_candidate")] + required_evidence_ids = {str(item.get("evidence_id") or "") for item in required_items if item.get("evidence_id")} + nodes = [] + for item in graph.get("nodes", []) if isinstance(graph.get("nodes"), list) else []: + if not isinstance(item, dict): + continue + source_evidence_id = str(item.get("source_evidence_id") or "") + if source_evidence_id and source_evidence_id not in required_evidence_ids: + removed_nodes.append(f"{graph.get('id')}:{item.get('instance_id')}") + continue + nodes.append(item) + by_evidence = { + str(evidence_id): node + for node in nodes + for evidence_id in node.get("evidence_ids", []) + if evidence_id + } + required_total += len(required_items) + occurrence_counts: dict[str, int] = {} + for item in required_items: + evidence_id = str(item.get("evidence_id") or "") + text_value = str(item.get("text") or "").strip() + role = str(item.get("role_hint") or "module").strip() + existing = by_evidence.get(evidence_id) + if existing: + existing["name"] = text_value + existing["role"] = role + existing["bbox_percent"] = item.get("bbox_percent") + existing["source_evidence_id"] = evidence_id + covered_total += 1 + continue + key = _normalized_label(text_value) or role + occurrence_counts[key] = occurrence_counts.get(key, 0) + 1 + instance_id = f"{graph.get('id')}_{key[:32]}_{occurrence_counts[key]}" + while any(str(node.get("instance_id")) == instance_id for node in nodes): + occurrence_counts[key] += 1 + instance_id = f"{graph.get('id')}_{key[:32]}_{occurrence_counts[key]}" + node = { + "instance_id": instance_id, + "name": text_value, + "role": role, + "evidence_ids": [evidence_id] if evidence_id else [], + "bbox_percent": item.get("bbox_percent"), + "source_evidence_id": evidence_id, + } + nodes.append(node) + if evidence_id: + by_evidence[evidence_id] = node + added_nodes.append(f"{graph.get('id')}:{instance_id}") + covered_total += 1 + + objectives = [item for item in nodes if str(item.get("role") or "").casefold() == "objective"] + if objectives: + objective_x = min(center(item)[0] for item in objectives) + for node in nodes: + if _normalized_label(_item_label(node)) == "ytest" and center(node)[0] < objective_x: + node["role"] = "target" + + groups = [item for item in nodes if str(item.get("role") or "").casefold() == "group"] + modules = [item for item in nodes if str(item.get("role") or "").casefold() == "module"] + inputs = [item for item in nodes if str(item.get("role") or "").casefold() == "input"] + targets = [item for item in nodes if str(item.get("role") or "").casefold() in {"target", "conditioning"}] + outputs = [item for item in nodes if str(item.get("role") or "").casefold() == "output"] + inferred: list[dict[str, Any]] = [] + for input_node in inputs: + input_x, input_y = center(input_node) + candidates = [item for item in modules if center(item)[0] >= input_x - 0.02] + candidates = candidates or modules + if candidates: + target_node = min(candidates, key=lambda item: abs(center(item)[0] - input_x) + 0.45 * abs(center(item)[1] - input_y)) + inferred.append(relation(input_node, target_node, "data_flow")) + ordered_modules = sorted(modules, key=lambda item: (center(item)[0], center(item)[1])) + for source, target in zip(ordered_modules, ordered_modules[1:]): + source_label = _normalized_label(_item_label(source)) + target_label = _normalized_label(_item_label(target)) + source_x, source_y = center(source) + target_x, target_y = center(target) + if source_label == target_label or target_x <= source_x or abs(target_y - source_y) > 0.22: + continue + inferred.append(relation(source, target, "data_flow")) + for output_node in outputs: + output_x, output_y = center(output_node) + candidates = [item for item in modules if center(item)[0] <= output_x + 0.02] + candidates = candidates or modules + if candidates: + source_node = min(candidates, key=lambda item: abs(output_x - center(item)[0]) + 0.45 * abs(output_y - center(item)[1])) + inferred.append(relation(source_node, output_node, "prediction")) + for objective in objectives: + objective_x, objective_y = center(objective) + sources = [ + item for item in [*outputs, *targets] + if abs(center(item)[0] - objective_x) + abs(center(item)[1] - objective_y) <= 0.38 + ] + if not sources and modules: + sources = [min(modules, key=lambda item: abs(center(item)[0] - objective_x) + abs(center(item)[1] - objective_y))] + for source in sources: + inferred.append(relation(source, objective, "training_objective")) + seen: set[tuple[str, str, str]] = set() + deduplicated = [] + for item in inferred: + key = (str(item.get("source")), str(item.get("target")), str(item.get("type"))) + if key in seen or key[0] == key[1]: + continue + seen.add(key) + deduplicated.append(item) + if deduplicated: + graph["relations"] = deduplicated + rebuilt_panels.append(str(graph.get("id"))) + graph["nodes"] = nodes + graph["group_instance_ids"] = [str(item.get("instance_id")) for item in groups] + + return { + "applied": bool(rebuilt_panels or added_nodes), + "added_nodes": added_nodes, + "removed_nodes": removed_nodes, + "rebuilt_panels": rebuilt_panels, + "required_candidate_count": required_total, + "covered_required_candidate_count": covered_total, + "required_candidate_coverage": round(covered_total / max(1, required_total), 4) if required_total else None, + } + + def _deduplicate_contract_entities(spec: dict[str, Any]) -> dict[str, list[str]]: replacements: dict[str, str] = {} removed: list[str] = [] @@ -1908,7 +2274,10 @@ def normalize_figure_contract(plan: dict[str, Any], parsed: dict[str, Any]) -> d spec[field] = _normalize_contract_entities(spec.get(field), field) entity_deduplication = _deduplicate_contract_entities(spec) spec["relations"] = _normalize_contract_relations(spec) + panel_graph_report = _normalize_panel_graphs(spec, parsed) completion_report = augment_contract_from_evidence(spec, parsed) + completion_report["panel_graphs"] = panel_graph_report + completion_report["panel_graph_completion"] = _complete_panel_graphs_from_inventory(spec, parsed) completion_report["removed_duplicate_entities"] = entity_deduplication["removed"] completion_report["duplicate_entity_replacements"] = entity_deduplication["replacements"] plan["contract_completion_report"] = completion_report @@ -2064,6 +2433,40 @@ def choose_input_target(label: str) -> dict[str, Any] | None: existing_pairs.add(pair) outgoing.add(source) incoming.add(pair[1]) + + # When several peer inputs already converge on the same first module, + # connect any remaining declared input to that same target. This covers + # generic tensor/table inputs such as X_test without inventing a new stage. + input_ids = { + str(item.get("id")) + for item in spec.get("inputs", []) if isinstance(item, dict) and item.get("id") + } + input_target_counts: dict[str, int] = {} + for relation in relations: + if not isinstance(relation, dict) or str(relation.get("source") or "") not in input_ids: + continue + target_id = str(relation.get("target") or "") + if target_id: + input_target_counts[target_id] = input_target_counts.get(target_id, 0) + 1 + dominant_input_target = next( + (target_id for target_id, count in sorted(input_target_counts.items(), key=lambda item: (-item[1], item[0])) if count >= 2), + None, + ) + dominant_target_item = next((item for item in modules if str(item.get("id") or "") == dominant_input_target), None) + if dominant_target_item: + for item in spec.get("inputs", []) if isinstance(spec.get("inputs"), list) else []: + if not isinstance(item, dict): + continue + source = str(item.get("id") or "") + if not source or source in outgoing: + continue + pair = (source, str(dominant_target_item.get("id"))) + evidence_ids = list(dict.fromkeys([*item.get("evidence_ids", []), *dominant_target_item.get("evidence_ids", [])])) + if pair not in existing_pairs and evidence_ids: + relations.append({"source": pair[0], "target": pair[1], "type": "encoding", "label": "", "evidence_ids": evidence_ids}) + existing_pairs.add(pair) + outgoing.add(source) + incoming.add(pair[1]) module_sources = {str(item.get("source")) for item in relations if isinstance(item, dict)} completion_added_ids = {str(value) for value in completion_report.get("added_entities", []) if str(value)} eligible_output_modules = [item for item in modules if str(item.get("id") or "") not in completion_added_ids] @@ -2116,6 +2519,7 @@ def choose_input_target(label: str) -> dict[str, Any] | None: if str(spec.get("topology") or "") not in allowed_topologies or inferred_topology in {"feedback", "multimodal", "dense_multiframe"}: spec["topology"] = inferred_topology completion_report["exact_visible_entity_groundings"] = _ground_exact_visible_entities(spec, parsed) + completion_report["late_repaired_relation_evidence"] = _repair_relation_evidence_after_grounding(spec, parsed) completion_report["removed_out_of_scope_entities"] = _prune_out_of_scope_entities(spec, parsed, completion_report) completion_report["removed_ungrounded_innovations"] = _remove_ungrounded_innovations(spec) completion_report["removed_input_shortcuts"] = _remove_input_shortcuts_through_intermediates(spec) @@ -2154,17 +2558,58 @@ def add_required_label(value: object) -> None: labels.append(label) normalized_labels.add(normalized) terminology = spec.get("terminology") if isinstance(spec.get("terminology"), dict) else {} - for field in ("inputs", "modules", "outputs"): - for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: - if isinstance(item, dict) and _item_label(item): - add_required_label(_item_label(item)) - for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: - if isinstance(panel, dict): + active_panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(active_panel_graphs) >= 2: + for panel in active_panel_graphs: add_required_label(panel.get("label")) + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + if isinstance(node, dict): + add_required_label(_item_label(node)) + else: + for field in ("inputs", "modules", "outputs"): + for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: + if isinstance(item, dict) and _item_label(item): + add_required_label(_item_label(item)) + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if isinstance(panel, dict): + add_required_label(panel.get("label")) for value in terminology.values(): add_required_label(value) spec["required_labels"] = labels repeatable_labels = [str(value).strip() for value in spec.get("repeatable_labels", []) if str(value).strip()] if isinstance(spec.get("repeatable_labels"), list) else [] + entity_by_id = { + str(item.get("id")): item + for field in ("inputs", "modules", "outputs", "innovations") + for item in (spec.get(field, []) if isinstance(spec.get(field), list) else []) + if isinstance(item, dict) and item.get("id") + } + panel_membership_counts: dict[str, int] = {} + for panel in spec.get("overview_panels", []) if isinstance(spec.get("overview_panels"), list) else []: + if not isinstance(panel, dict): + continue + for item_id in {str(value) for value in panel.get("entity_ids", []) if value}: + panel_membership_counts[item_id] = panel_membership_counts.get(item_id, 0) + 1 + for item_id, count in panel_membership_counts.items(): + item = entity_by_id.get(item_id) + label = _item_label(item) if item else "" + if count >= 2 and label and label not in repeatable_labels: + repeatable_labels.append(label) + panel_instance_label_counts: dict[str, tuple[str, int]] = {} + for panel in spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []: + if not isinstance(panel, dict): + continue + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + if not isinstance(node, dict): + continue + label = _item_label(node) + key = _normalized_label(label) + if not key: + continue + original, count = panel_instance_label_counts.get(key, (label, 0)) + panel_instance_label_counts[key] = (original, count + 1) + for label, count in panel_instance_label_counts.values(): + if count >= 2 and label not in repeatable_labels: + repeatable_labels.append(label) central_claim_text = str(spec.get("central_claim", {}).get("text") or "") if isinstance(spec.get("central_claim"), dict) else str(spec.get("central_claim") or "") shared_component_signal = bool(re.search(r"\b(?:same|shared|reused?|single underlying)\b", central_claim_text, re.IGNORECASE)) if shared_component_signal: @@ -2175,6 +2620,9 @@ def add_required_label(value: object) -> None: role = str(item.get("role") or "").casefold() if label and ("model" in label.casefold() or "shared" in role or "reused" in role) and label not in repeatable_labels: repeatable_labels.append(label) + if len(active_panel_graphs) >= 2: + required_normalized = {_normalized_label(value) for value in labels if _normalized_label(value)} + repeatable_labels = [value for value in repeatable_labels if _normalized_label(value) in required_normalized] spec["repeatable_labels"] = repeatable_labels raw_uncertainties = list(summary.get("unknowns", []) if isinstance(summary.get("unknowns"), list) else []) raw_uncertainties.extend( @@ -2204,6 +2652,58 @@ def add_required_label(value: object) -> None: def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: spec = plan.get("figure_specification", {}) + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] + if len(panel_graphs) >= 2: + labels = [] + connectors = [] + groups = [] + reading_order = [] + for panel in panel_graphs: + panel_id = str(panel.get("id") or "") + groups.append({ + "id": panel_id, + "panel_label": str(panel.get("panel_label") or ""), + "label": str(panel.get("label") or ""), + "member_ids": [str(item.get("instance_id") or item.get("id") or "") for item in panel.get("nodes", []) if isinstance(item, dict)], + }) + node_ids = set() + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + if not isinstance(node, dict): + continue + instance_id = str(node.get("instance_id") or node.get("id") or "") + text_value = _item_label(node) + if not instance_id or not text_value: + continue + node_ids.add(instance_id) + reading_order.append(instance_id) + labels.append({ + "target_id": instance_id, + "text": text_value, + "role": str(node.get("role") or "module"), + "panel_id": panel_id, + "bbox_percent": node.get("bbox_percent"), + }) + for item in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if not isinstance(item, dict): + continue + source = str(item.get("source") or "") + target = str(item.get("target") or "") + if source not in node_ids or target not in node_ids or source == target: + continue + connectors.append({ + "source": source, + "target": target, + "type": str(item.get("type") or "data_flow"), + "label": str(item.get("label") or ""), + "panel_id": panel_id, + }) + return { + "summary": "Exact editable panel-instance labels and local directed connectors derived from the paper contract.", + "labels": labels, + "connectors": connectors, + "groups": groups, + "reading_order": reading_order, + } labels = [] visible_targets: dict[str, str] = {} target_aliases: dict[str, str] = {} @@ -2243,6 +2743,7 @@ def build_overlay_spec(plan: dict[str, Any]) -> dict[str, Any]: def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: spec = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] entities = [ item for field in ("inputs", "modules", "outputs", "innovations") @@ -2252,7 +2753,12 @@ def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: required_labels = [str(value).strip() for value in spec.get("required_labels", []) if str(value).strip()] allowed = {_normalized_label(value) for value in [*required_labels, *spec.get("repeatable_labels", [])] if _normalized_label(value)} visible_entities = [item for item in entities if _normalized_label(_item_label(item)) in allowed] if allowed else entities - entity_ids = [str(item.get("id") or "") for item in visible_entities if str(item.get("id") or "")] + panel_nodes = [node for panel in panel_graphs for node in panel.get("nodes", []) if isinstance(node, dict)] + entity_ids = ( + [str(item.get("instance_id") or item.get("id") or "") for item in panel_nodes if str(item.get("instance_id") or item.get("id") or "")] + if len(panel_graphs) >= 2 else + [str(item.get("id") or "") for item in visible_entities if str(item.get("id") or "")] + ) labels = required_labels or [_item_label(item) for item in visible_entities if _item_label(item)] spec["storyline"] = labels spec["must_show"] = labels @@ -2279,8 +2785,8 @@ def synchronize_plan_to_contract(plan: dict[str, Any]) -> None: plan["visual_metaphors"] = { "summary": "Paper-grounded visual metaphors synchronized to the normalized contract.", "items": [ - {"module_id": str(item.get("id") or ""), "metaphor": _item_label(item), "must_show": [], "avoid_showing": list(spec.get("forbidden_inventions", []))} - for item in visible_entities + {"module_id": str(item.get("instance_id") or item.get("id") or ""), "metaphor": _item_label(item), "must_show": [], "avoid_showing": list(spec.get("forbidden_inventions", []))} + for item in (panel_nodes if len(panel_graphs) >= 2 else visible_entities) ], } @@ -2388,7 +2894,7 @@ def fact(item: Any, item_id: str) -> dict[str, Any]: def _fast_cache_path(parsed: dict[str, Any], model: str, domain_profile: str) -> Path: - signature = json.dumps({"version": 32, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "domain_profile": domain_profile}, sort_keys=True).encode("utf-8") + signature = json.dumps({"version": 41, "document_cache_version": DOCUMENT_CACHE_VERSION, "model": model, "domain_profile": domain_profile}, sort_keys=True).encode("utf-8") variant = hashlib.sha256(signature).hexdigest()[:16] root = Path(os.getenv("RFS_CACHE_DIR", "").strip() or (Path.home() / ".cache" / "research-figure-studio")) return root / "paper_contracts" / str(parsed.get("source_sha256")) / variant / "fast_plan.json" diff --git a/rfs/paper_to_image/semantic_blueprint.py b/rfs/paper_to_image/semantic_blueprint.py index 2f480de..da952e8 100644 --- a/rfs/paper_to_image/semantic_blueprint.py +++ b/rfs/paper_to_image/semantic_blueprint.py @@ -109,6 +109,7 @@ def _layout_nodes_in_dense_panels( nodes: list[dict[str, Any]], relations: list[dict[str, str]], panels: list[dict[str, Any]], + global_ranks: dict[str, int], ) -> list[dict[str, Any]]: if not panels: return _layout_nodes(nodes, _rank_nodes(nodes, relations)) @@ -149,6 +150,7 @@ def _layout_nodes_in_dense_panels( positioned.append({ **item, "panel_id": panel["id"], + "global_rank": int(global_ranks.get(item["id"], item.get("rank", 0))), "bbox_percent": { "x": round(inner_x + local_box["x"] * inner_w, 6), "y": round(inner_y + local_box["y"] * inner_h, 6), @@ -299,7 +301,9 @@ def _connector_path( target_box = target["bbox_percent"] source_center = (source_box["x"] + source_box["w"] / 2, source_box["y"] + source_box["h"] / 2) target_center = (target_box["x"] + target_box["w"] / 2, target_box["y"] + target_box["h"] / 2) - is_return = relation_type in _FEEDBACK_TYPES or target["rank"] <= source["rank"] + source_rank = int(source.get("global_rank", source.get("rank", 0))) + target_rank = int(target.get("global_rank", target.get("rank", 0))) + is_return = relation_type in _FEEDBACK_TYPES or target_rank <= source_rank if is_return: route_y = 0.95 if source_center[1] >= 0.42 or target_center[1] >= 0.42 else 0.05 source_anchor_y = source_box["y"] + source_box["h"] if route_y > 0.5 else source_box["y"] @@ -318,9 +322,13 @@ def _connector_path( start = [source_center[0], source_box["y"]] end = [target_center[0], target_box["y"] + target_box["h"]] return ([[round(value, 6) for value in start], [round(value, 6) for value in end]], "vertical") - start = [source_box["x"] + source_box["w"], source_box["y"] + source_port_fraction * source_box["h"]] - end = [target_box["x"], target_box["y"] + target_port_fraction * target_box["h"]] - if target["rank"] - source["rank"] > 1: + if target_center[0] >= source_center[0]: + start = [source_box["x"] + source_box["w"], source_box["y"] + source_port_fraction * source_box["h"]] + end = [target_box["x"], target_box["y"] + target_port_fraction * target_box["h"]] + else: + start = [source_box["x"], source_box["y"] + source_port_fraction * source_box["h"]] + end = [target_box["x"] + target_box["w"], target_box["y"] + target_port_fraction * target_box["h"]] + if target_rank - source_rank > 1: lane_y = max(0.035, min(0.965, min(source_box["y"], target_box["y"]) - 0.045)) if lane_y < 0.055: lane_y = min(0.965, max(source_box["y"] + source_box["h"], target_box["y"] + target_box["h"]) + 0.045) @@ -344,6 +352,179 @@ def _connector_path( ], "orthogonal") +def _compile_panel_graph_blueprint( + specification: dict[str, Any], + *, + max_nodes: int, +) -> dict[str, Any] | None: + raw_panels = specification.get("panel_graphs", []) + if not isinstance(raw_panels, list) or len(raw_panels) < 2: + return None + panels: list[dict[str, Any]] = [] + panel_nodes: dict[str, list[dict[str, Any]]] = {} + panel_relations: dict[str, list[dict[str, str]]] = {} + panel_groups: dict[str, list[dict[str, Any]]] = {} + total_nodes = 0 + total_relations = 0 + for panel_index, raw_panel in enumerate(raw_panels): + if not isinstance(raw_panel, dict): + continue + panel_id = str(raw_panel.get("id") or f"panel_{panel_index + 1}").strip() + label = str(raw_panel.get("label") or raw_panel.get("description") or "").strip() + if not panel_id or not label: + continue + nodes: list[dict[str, Any]] = [] + groups: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for node_index, raw_node in enumerate(raw_panel.get("nodes", []) if isinstance(raw_panel.get("nodes"), list) else []): + if not isinstance(raw_node, dict): + continue + instance_id = str(raw_node.get("instance_id") or raw_node.get("id") or "").strip() + node_label = _label(raw_node) + if not instance_id or not node_label or instance_id in seen_ids: + continue + role = str(raw_node.get("role") or "module").strip() + role_key = role.casefold() + if role_key == "group": + groups.append({ + "id": instance_id, + "label": node_label, + "role": role, + "evidence_ids": [str(value) for value in raw_node.get("evidence_ids", []) if value], + }) + seen_ids.add(instance_id) + continue + field = "inputs" if "input" in role_key or "dataset" in role_key else "outputs" if any(term in role_key for term in ("output", "prediction", "objective", "loss")) else "modules" + nodes.append({ + "id": instance_id, + "semantic_id": str(raw_node.get("semantic_id") or "").strip() or None, + "label": node_label, + "field": field, + "role": role, + "evidence_ids": [str(value) for value in raw_node.get("evidence_ids", []) if value], + "order": node_index, + "panel_id": panel_id, + }) + seen_ids.add(instance_id) + operational_ids = {item["id"] for item in nodes} + relations: list[dict[str, str]] = [] + for raw_relation in raw_panel.get("relations", []) if isinstance(raw_panel.get("relations"), list) else []: + if not isinstance(raw_relation, dict): + continue + source = str(raw_relation.get("source") or "").strip() + target = str(raw_relation.get("target") or "").strip() + if not source or not target or source == target or source not in operational_ids or target not in operational_ids: + continue + relations.append({ + "source": source, + "target": target, + "type": str(raw_relation.get("type") or "data_flow").strip(), + "label": str(raw_relation.get("label") or "").strip(), + }) + panels.append({ + "id": panel_id, + "panel_label": str(raw_panel.get("panel_label") or chr(ord("a") + panel_index)).strip(), + "label": label, + "description": str(raw_panel.get("description") or label).strip(), + "evidence_ids": [str(value) for value in raw_panel.get("evidence_ids", []) if value], + "entity_ids": [item["id"] for item in nodes], + "groups": groups, + "order": panel_index, + }) + panel_nodes[panel_id] = nodes + panel_relations[panel_id] = relations + panel_groups[panel_id] = groups + total_nodes += len(nodes) + total_relations += len(relations) + if len(panels) < 2 or total_nodes < 2 or total_relations < 1: + return None + panel_node_limit = max(24, int(max_nodes)) + if total_nodes > panel_node_limit: + return { + "summary": "Panel-local semantic blueprint skipped because the instance graph is too dense for one raster guide.", + "applied": False, + "topology": "dense_multiframe", + "nodes": [], + "connectors": [], + "reason": "too_many_nodes", + "node_count": total_nodes, + "source": "panel_graphs", + } + + positioned_panels = _layout_dense_panels(panels) + by_panel = {item["id"]: item for item in positioned_panels} + positioned_nodes: list[dict[str, Any]] = [] + connectors: list[dict[str, Any]] = [] + isolated_node_ids: list[str] = [] + connector_index = 1 + for panel in positioned_panels: + panel_id = panel["id"] + nodes = panel_nodes.get(panel_id, []) + relations = panel_relations.get(panel_id, []) + ranks = _rank_nodes(nodes, relations) + local_layout = _layout_nodes(nodes, ranks) + box = by_panel[panel_id]["bbox_percent"] + inner_x = box["x"] + box["w"] * 0.035 + inner_y = box["y"] + box["h"] * 0.22 + inner_w = box["w"] * 0.93 + inner_h = box["h"] * 0.72 + local_positioned: list[dict[str, Any]] = [] + for item in local_layout: + local_box = item["bbox_percent"] + local_positioned.append({ + **item, + "panel_id": panel_id, + "global_rank": int(ranks.get(item["id"], item.get("rank", 0))), + "bbox_percent": { + "x": round(inner_x + local_box["x"] * inner_w, 6), + "y": round(inner_y + local_box["y"] * inner_h, 6), + "w": round(local_box["w"] * inner_w, 6), + "h": round(local_box["h"] * inner_h, 6), + }, + }) + positioned_nodes.extend(local_positioned) + local_by_id = {item["id"]: item for item in local_positioned} + connected_ids = {value for relation in relations for value in (relation["source"], relation["target"])} + isolated_node_ids.extend(item["id"] for item in nodes if item["id"] not in connected_ids) + incoming_groups: dict[str, list[dict[str, str]]] = defaultdict(list) + outgoing_groups: dict[str, list[dict[str, str]]] = defaultdict(list) + for relation in relations: + incoming_groups[relation["target"]].append(relation) + outgoing_groups[relation["source"]].append(relation) + for relation in relations: + incoming = incoming_groups[relation["target"]] + outgoing = outgoing_groups[relation["source"]] + target_fraction = (incoming.index(relation) + 1) / (len(incoming) + 1) + source_fraction = (outgoing.index(relation) + 1) / (len(outgoing) + 1) + path, route_style = _connector_path( + local_by_id[relation["source"]], + local_by_id[relation["target"]], + relation["type"], + source_port_fraction=source_fraction, + target_port_fraction=target_fraction, + ) + connectors.append({ + "id": f"semantic_connector_{connector_index:02d}", + "panel_id": panel_id, + **relation, + "path_percent": path, + "route_style": route_style, + }) + connector_index += 1 + return { + "summary": "Panel-local paper semantic graphs compiled into normalized node and connector geometry.", + "applied": True, + "topology": "dense_multiframe", + "node_count": len(positioned_nodes), + "connector_count": len(connectors), + "isolated_node_ids": isolated_node_ids, + "panels": positioned_panels, + "nodes": positioned_nodes, + "connectors": connectors, + "source": "panel_graphs", + } + + def compile_semantic_blueprint( specification: dict[str, Any] | None, *, @@ -351,6 +532,10 @@ def compile_semantic_blueprint( ) -> dict[str, Any]: specification = specification if isinstance(specification, dict) else {} topology = str(specification.get("topology") or "unknown") + if topology == "dense_multiframe": + panel_graph_blueprint = _compile_panel_graph_blueprint(specification, max_nodes=max_nodes) + if panel_graph_blueprint is not None: + return panel_graph_blueprint nodes = _visible_entities(specification) if topology not in _SUPPORTED_TOPOLOGIES: return {"summary": "Generic semantic blueprint not applicable to this topology.", "applied": False, "topology": topology, "nodes": [], "connectors": [], "reason": "unsupported_topology"} @@ -370,12 +555,12 @@ def compile_semantic_blueprint( if len(connected_nodes) >= 2: nodes = connected_nodes + global_ranks = _rank_nodes(nodes, relations) dense_panels = _layout_dense_panels(_dense_overview_panels(specification)) if topology == "dense_multiframe" else [] - if len(dense_panels) >= 3: - positioned = _layout_nodes_in_dense_panels(nodes, relations, dense_panels) + if len(dense_panels) >= 2: + positioned = _layout_nodes_in_dense_panels(nodes, relations, dense_panels, global_ranks) else: - ranks = _rank_nodes(nodes, relations) - positioned = _layout_nodes(nodes, ranks) + positioned = _layout_nodes(nodes, global_ranks) by_id = {item["id"]: item for item in positioned} incoming_groups: dict[str, list[dict[str, str]]] = defaultdict(list) outgoing_groups: dict[str, list[dict[str, str]]] = defaultdict(list) diff --git a/rfs/paper_to_image/templates.py b/rfs/paper_to_image/templates.py index a1953d1..933e6a4 100644 --- a/rfs/paper_to_image/templates.py +++ b/rfs/paper_to_image/templates.py @@ -631,6 +631,11 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) panel_title = str(panel.get("label") or "").strip() visible_title = f"{panel_marker} {panel_title}" if panel_marker else panel_title centered_wrapped_text(title_box, visible_title, max(13, round(height * 0.020)), fill=(42, 72, 91)) + groups = [item for item in panel.get("groups", []) if isinstance(item, dict) and str(item.get("label") or "").strip()] + for group_index, group in enumerate(groups[:2]): + group_box = region(panel_box, 0.18, 0.165 + group_index * 0.055, 0.82, 0.215 + group_index * 0.055) + draw.rounded_rectangle(group_box, radius=max(5, round(height * 0.008)), fill=(244, 242, 251), outline=(98, 84, 147), width=2) + centered_wrapped_text(group_box, str(group.get("label") or ""), max(11, round(height * 0.014)), fill=(98, 84, 147), bold=False) node_boxes: dict[str, tuple[int, int, int, int]] = {} node_styles: dict[str, tuple[tuple[int, int, int], tuple[int, int, int]]] = {} for node in generic_semantic_plan.get("nodes", []): @@ -697,6 +702,11 @@ def bottom(box): return ((box[0] + box[2]) // 2, box[3]) ] elif semantic_generic: semantic_label_order = [str(item.get("label") or "") for item in generic_semantic_plan.get("panels", [])] + semantic_label_order.extend( + str(group.get("label") or "") + for panel in generic_semantic_plan.get("panels", []) if isinstance(panel, dict) + for group in panel.get("groups", []) if isinstance(group, dict) + ) semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("nodes", [])) semantic_label_order.extend(str(item.get("label") or "") for item in generic_semantic_plan.get("connectors", []) if str(item.get("label") or "").strip()) return { diff --git a/tests/unit/test_control_localizer.py b/tests/unit/test_control_localizer.py new file mode 100644 index 0000000..ed2d1d0 --- /dev/null +++ b/tests/unit/test_control_localizer.py @@ -0,0 +1,19 @@ +import unittest + +import numpy as np + +from rfs.control_localizer import _hough_line_coordinates + + +class ControlLocalizerTests(unittest.TestCase): + def test_hough_line_coordinates_accepts_opencv_shape_variants(self): + self.assertEqual(_hough_line_coordinates(np.array([[1, 2, 3, 4]], dtype=np.int32)), (1, 2, 3, 4)) + self.assertEqual(_hough_line_coordinates(np.array([1, 2, 3, 4], dtype=np.int32)), (1, 2, 3, 4)) + self.assertEqual(_hough_line_coordinates([[1, 2, 3, 4]]), (1, 2, 3, 4)) + + def test_hough_line_coordinates_rejects_incomplete_rows(self): + self.assertIsNone(_hough_line_coordinates(np.array([1, 2, 3], dtype=np.int32))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_paper_analyzer.py b/tests/unit/test_paper_analyzer.py index 1597375..bc78d98 100644 --- a/tests/unit/test_paper_analyzer.py +++ b/tests/unit/test_paper_analyzer.py @@ -7,7 +7,7 @@ import fitz -from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _deadline_ocr_wave, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, parse_paper +from rfs.paper_to_image.analyzer import _assign_ocr_parent_blocks, _block_kind, _column_groups, _deadline_ocr_wave, _filter_ocr_margin_noise, _normalized_compare_text, _prioritize_ocr_candidates, _rapidocr_worker_count, _repair_ocr_spacing, _section_coverage, _token_overlap_agreement, evidence_excerpt, overview_figure_panel_excerpt, overview_figure_spatial_excerpt, parse_paper from rfs.paper_to_image.inspection import inspect_paper from rfs.paper_to_image.document_cache import DOCUMENT_CACHE_VERSION, write_document_cache from rfs.paper_to_image.preparation import _semantic_cache_safe @@ -87,6 +87,85 @@ def draw(page): self.assertIn("data engine", caption) self.assertTrue(any(item["kind"] == "caption" and "data engine" in item["text"] for item in parsed["evidence"])) + def test_cross_column_caption_continuation_is_recovered(self): + def draw(page): + page.insert_textbox((40, 330, 290, 345), "Fig. 1 | Overview of the method. a, High-level pre-training and", fontsize=7) + page.insert_textbox((40, 345, 290, 360), "usage. b, The model architecture is adapted from the", fontsize=7) + page.insert_textbox((310, 330, 560, 345), "standard transformer encoder for two-dimensional tables.", fontsize=7) + page.insert_textbox((40, 390, 290, 470), "Body text in the left column must not enter the caption.", fontsize=9) + page.insert_textbox((310, 390, 560, 470), "Body text in the right column must not enter the caption.", fontsize=9) + + parsed = parse_paper(self._pdf(draw)) + captions = parsed["document_index"]["figures"] + + self.assertEqual(len(captions), 1) + self.assertIn("standard transformer encoder", captions[0]["caption"]) + self.assertNotIn("Body text", captions[0]["caption"]) + + def test_evidence_excerpt_prioritizes_labels_inside_overview_figure_region(self): + caption = "Figure 1: Overview of the proposed method and model architecture." + parsed = { + "pages": [{ + "page": 2, + "width": 600, + "height": 800, + "blocks": [ + {"id": "P002_B001", "reading_order": 1}, + {"id": "P002_B002", "reading_order": 2}, + {"id": "P002_B003", "reading_order": 3}, + {"id": "P002_B004", "reading_order": 4}, + ], + }], + "document_index": {"figures": [{"page": 2, "block_id": "P002_B004", "caption": caption}]}, + "evidence": [ + {"id": "E_CAP", "page": 2, "block_id": "P002_B004", "bbox": [40, 340, 560, 360], "kind": "caption", "text": caption}, + {"id": "E_FEATURE", "page": 2, "block_id": "P002_B001", "bbox": [310, 180, 410, 195], "kind": "paragraph", "text": "1D feature attention"}, + {"id": "E_SAMPLE", "page": 2, "block_id": "P002_B002", "bbox": [420, 180, 520, 195], "kind": "paragraph", "text": "1D sample attention"}, + {"id": "E_OUTPUT", "page": 2, "block_id": "P002_B003", "bbox": [500, 260, 550, 275], "kind": "paragraph", "text": "Predictions"}, + *[ + {"id": f"E_OTHER_{index}", "page": 3 + index, "block_id": f"P{3 + index:03d}_B001", "bbox": [40, 50, 560, 120], "kind": "caption", "text": "Figure 9: Architecture visualization. " + ("irrelevant analysis " * 20)} + for index in range(8) + ], + ], + } + + excerpt = evidence_excerpt(parsed, max_chars=700) + + self.assertIn("1D feature attention", excerpt) + self.assertIn("1D sample attention", excerpt) + self.assertIn("Predictions", excerpt) + spatial = json.loads(overview_figure_spatial_excerpt(parsed)) + by_text = {item["text"]: item for item in spatial} + self.assertEqual(by_text["1D feature attention"]["bbox_percent"], [0.5167, 0.225, 0.6833, 0.2437]) + + def test_panel_excerpt_treats_descriptive_dataset_headings_as_groups(self): + caption = "Figure 1: Overview. a, Pre-training and use. b, Model architecture." + parsed = { + "pages": [{"page": 1, "width": 600, "height": 800, "blocks": []}], + "document_index": {"figures": [{"page": 1, "block_id": "P001_B009", "caption": caption}]}, + "evidence": [ + {"id": "E_A", "page": 1, "block_id": "P001_B001", "bbox": [40, 80, 48, 92], "kind": "paragraph", "text": "a"}, + {"id": "E_A_GROUP", "page": 1, "block_id": "P001_B002", "bbox": [55, 105, 180, 120], "kind": "paragraph", "text": "A synthetic dataset"}, + {"id": "E_A_INPUT", "page": 1, "block_id": "P001_B003", "bbox": [55, 135, 120, 150], "kind": "paragraph", "text": "X train"}, + {"id": "E_B", "page": 1, "block_id": "P001_B004", "bbox": [310, 80, 318, 92], "kind": "paragraph", "text": "b"}, + {"id": "E_B_GROUP", "page": 1, "block_id": "P001_B005", "bbox": [330, 105, 500, 120], "kind": "paragraph", "text": "An arbitrary real-world dataset"}, + {"id": "E_B_INPUT", "page": 1, "block_id": "P001_B006", "bbox": [330, 135, 420, 150], "kind": "paragraph", "text": "Input dataset"}, + {"id": "E_CAP", "page": 1, "block_id": "P001_B009", "bbox": [40, 380, 560, 410], "kind": "caption", "text": caption}, + ], + } + + excerpt = json.loads(overview_figure_panel_excerpt(parsed)) + labels = { + item["text"]: item + for panel in excerpt["panels"] + for item in panel["label_inventory"] + } + + self.assertEqual(labels["A synthetic dataset"]["role_hint"], "group") + self.assertEqual(labels["An arbitrary real-world dataset"]["role_hint"], "group") + self.assertEqual(labels["X train"]["role_hint"], "input") + self.assertEqual(labels["Input dataset"]["role_hint"], "input") + def test_two_column_blocks_are_sorted_left_then_right(self): def draw(page): page.insert_textbox((330, 100, 560, 180), "RIGHT COLUMN SECOND with enough words to form a complete paragraph.", fontsize=10) @@ -137,11 +216,23 @@ def test_cjk_section_coverage_and_caption_kinds(self): self.assertEqual(_block_kind("表 2 实验结果"), "table") self.assertEqual(_block_kind("2 方法"), "heading") + def test_implicit_first_page_abstract_is_recognized(self): + lead = ( + "Tabular data are used across scientific fields. " + "Here we present a foundation model that learns an efficient prediction algorithm. " + + "This lead summary reports the problem, method, evidence and implications. " * 14 + ) + coverage = _section_coverage([{"page": 1, "text": lead}], [{"title": "Methods"}]) + + self.assertTrue(coverage["abstract"]) + self.assertTrue(coverage["method"]) + def test_localized_caption_prefix_requires_a_real_identifier(self): self.assertEqual(_block_kind("Figura 1: Arquitectura del sistema"), "caption") self.assertEqual(_block_kind("Tabelle A2: Ablation results"), "table") self.assertEqual(_block_kind("figura editable"), "paragraph") self.assertEqual(_block_kind("tableau comparatif"), "paragraph") + self.assertEqual(_block_kind("Figures 1 and 2 outline our approach"), "paragraph") def test_three_column_blocks_are_sorted_column_major(self): def draw(page): diff --git a/tests/unit/test_paper_contract.py b/tests/unit/test_paper_contract.py index d045895..b5811ff 100644 --- a/tests/unit/test_paper_contract.py +++ b/tests/unit/test_paper_contract.py @@ -2,11 +2,69 @@ import unittest from rfs.paper_to_image.planner import validate_plan_grounding -from rfs.paper_to_image.preparation import _demote_unconnected_completion_entities, _detect_dense_overview_panels, _extract_overview_panels, _fast_cache_path, _merge_isolated_architecture_entities, _repair_cross_script_entity_labels, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract +from rfs.paper_to_image.preparation import _demote_unconnected_completion_entities, _detect_dense_overview_panels, _extract_overview_panels, _fast_cache_path, _merge_isolated_architecture_entities, _normalize_panel_graphs, _repair_cross_script_entity_labels, _repair_relation_evidence_after_grounding, build_overlay_spec, expand_plan_evidence, normalize_figure_contract, synchronize_plan_to_contract from rfs.paper_to_image.contract_completion import _overview_candidates class PaperContractTests(unittest.TestCase): + def test_panel_graph_normalization_keeps_local_instances_and_repairs_relation_evidence(self): + parsed = {"evidence": [ + {"id": "E_DATA", "page": 2, "text": "Dataset"}, + {"id": "E_MODEL", "page": 2, "text": "Model"}, + ]} + spec = {"panel_graphs": [{ + "id": "panel_a", + "panel_label": "a", + "label": "Workflow", + "nodes": [ + {"instance_id": "data", "name": "Dataset", "role": "input", "evidence_ids": ["E_DATA"]}, + {"instance_id": "model", "name": "Model", "role": "module", "evidence_ids": ["E_MODEL"]}, + ], + "relations": [{"source": "data", "target": "model", "type": "data_flow", "evidence_ids": []}], + }]} + + report = _normalize_panel_graphs(spec, parsed) + + self.assertEqual(report["node_count"], 2) + self.assertEqual(report["relation_count"], 1) + self.assertEqual(spec["panel_graphs"][0]["relations"][0]["evidence_ids"], ["E_DATA", "E_MODEL"]) + + def test_two_panel_overview_is_preserved_as_dense_multiframe(self): + parsed = { + "document_index": {"figures": [{ + "page": 2, + "block_id": "P002_B010", + "caption": "Fig. 1 | Overview of the method. a, High-level pre-training and usage. b, The model architecture.", + }]}, + "pages": [{ + "page": 2, + "height": 790, + "blocks": [ + {"id": "P002_B010", "reading_order": 10, "bbox": [40, 335, 290, 344], "font_size": 7, "text": "Fig. 1 | Overview of the method. a, High-level pre-training and usage."}, + {"id": "P002_B011", "reading_order": 11, "bbox": [306, 335, 560, 344], "font_size": 7, "text": "b, The model architecture."}, + ], + }], + "evidence": [ + {"id": "E_A", "block_id": "P002_B010", "page": 2, "text": "Fig. 1 | Overview of the method. a, High-level pre-training and usage."}, + {"id": "E_B", "block_id": "P002_B011", "page": 2, "text": "b, The model architecture."}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input dataset", "evidence_ids": []}], + "modules": [{"id": "encoder", "name": "attention over features", "evidence_ids": []}], + "outputs": [{"id": "prediction", "name": "Prediction", "evidence_ids": []}], + "innovations": [], + } + + dense = _detect_dense_overview_panels(parsed) + panels = _extract_overview_panels(parsed, spec) + + self.assertTrue(dense["detected"]) + self.assertEqual(dense["panel_labels"], ["a", "b"]) + self.assertEqual([item["panel_label"] for item in panels], ["a", "b"]) + self.assertEqual(panels[0]["entity_ids"], ["input", "prediction"]) + self.assertEqual(panels[1]["entity_ids"], ["encoder"]) + def test_extracts_cross_column_figure_one_panel_descriptions(self): parsed = { "document_index": {"figures": [{ @@ -121,6 +179,69 @@ def test_dense_overview_panels_and_local_details_are_not_flattened_into_peer_nod self.assertEqual(demoted, ["class_token"]) self.assertEqual(spec["supporting_details"][0]["visibility"], "unlabeled_visual_detail_not_global_node") + def test_late_exact_grounding_repairs_boundary_relation_evidence(self): + parsed = { + "document_index": {"figures": [{"page": 2, "caption": "Figure 1: Model overview."}]}, + "pages": [], + "evidence": [ + {"id": "E_INPUT", "page": 2, "text": "Input dataset"}, + {"id": "E_MODEL", "page": 2, "text": "Input dataset enters the model architecture"}, + {"id": "E_OUTPUT", "page": 2, "text": "model architecture produces Predictions"}, + ], + } + spec = { + "inputs": [{"id": "input", "name": "Input dataset", "evidence_ids": ["E_INPUT", "E_MODEL"]}], + "modules": [{"id": "model", "name": "model architecture", "evidence_ids": ["E_MODEL", "E_OUTPUT"]}], + "outputs": [{"id": "output", "name": "Predictions", "evidence_ids": ["E_OUTPUT"]}], + "innovations": [], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow", "evidence_ids": []}, + {"source": "model", "target": "output", "type": "prediction", "evidence_ids": []}, + ], + } + + repaired = _repair_relation_evidence_after_grounding(spec, parsed) + + self.assertTrue(all(item["evidence_ids"] for item in spec["relations"])) + self.assertEqual(repaired, ["input->model", "model->output"]) + + def test_peer_generic_inputs_converge_on_existing_shared_target(self): + evidence = [ + {"id": "E_XTRAIN", "page": 2, "text": "X train"}, + {"id": "E_YTRAIN", "page": 2, "text": "y train"}, + {"id": "E_XTEST", "page": 2, "text": "X test"}, + {"id": "E_LINEAR", "page": 2, "text": "linear encoders"}, + {"id": "E_OUTPUT", "page": 2, "text": "y test"}, + ] + parsed = { + "document_index": {"figures": [{"page": 2, "caption": "Figure 1: Model overview."}]}, + "pages": [], + "evidence": evidence, + } + plan = { + "paper_summary": {}, + "figure_specification": { + "inputs": [ + {"id": "xtrain", "name": "X train", "evidence_ids": ["E_XTRAIN"]}, + {"id": "ytrain", "name": "y train", "evidence_ids": ["E_YTRAIN"]}, + {"id": "xtest", "name": "X test", "evidence_ids": ["E_XTEST"]}, + ], + "modules": [{"id": "linear", "name": "linear encoders", "evidence_ids": ["E_LINEAR"]}], + "outputs": [{"id": "output", "name": "y test", "evidence_ids": ["E_OUTPUT"]}], + "innovations": [], + "relations": [ + {"source": "xtrain", "target": "linear", "type": "encoding", "evidence_ids": ["E_XTRAIN", "E_LINEAR"]}, + {"source": "ytrain", "target": "linear", "type": "encoding", "evidence_ids": ["E_YTRAIN", "E_LINEAR"]}, + {"source": "linear", "target": "output", "type": "prediction", "evidence_ids": ["E_LINEAR", "E_OUTPUT"]}, + ], + }, + } + + normalize_figure_contract(plan, parsed) + relations = plan["figure_specification"]["relations"] + + self.assertTrue(any(item["source"] == "xtest" and item["target"] == "linear" for item in relations)) + def test_cross_script_symbol_hallucination_is_repaired_from_exact_evidence_span(self): spec = { "inputs": [{"id": "tiles", "name": "256 脳 256 image tiles", "evidence_ids": ["E1"]}], @@ -135,6 +256,22 @@ def test_cross_script_symbol_hallucination_is_repaired_from_exact_evidence_span( self.assertEqual(spec["inputs"][0]["name"], "256 × 256 image tiles") self.assertEqual(repaired, ["tiles:256 脳 256 image tiles->256 × 256 image tiles"]) + def test_cross_script_repair_restores_exact_multiplication_symbol_label(self): + parsed = { + "evidence": [{"id": "E_LAYER", "page": 2, "text": "2D TabPFN layer (12×)"}], + } + spec = { + "inputs": [], + "modules": [{"id": "layer", "name": "2D TabPFN layer (12脳)", "evidence_ids": ["E_LAYER"]}], + "outputs": [], + "innovations": [], + } + + repaired = _repair_cross_script_entity_labels(spec, parsed) + + self.assertEqual(spec["modules"][0]["name"], "2D TabPFN layer (12×)") + self.assertTrue(repaired) + def test_overview_selection_rejects_late_scaling_figure_when_figure_one_is_a_true_model_overview(self): parsed = { "document_index": { diff --git a/tests/unit/test_semantic_blueprint.py b/tests/unit/test_semantic_blueprint.py index d2a56da..c72f4ff 100644 --- a/tests/unit/test_semantic_blueprint.py +++ b/tests/unit/test_semantic_blueprint.py @@ -183,6 +183,78 @@ def test_contract_larger_than_blueprint_limit_is_not_forced_into_one_canvas(self self.assertFalse(report["applied"]) self.assertEqual(report["reason"], "too_many_nodes") + def test_two_panel_dense_overview_uses_panel_local_layout(self): + specification = { + "topology": "dense_multiframe", + "required_labels": ["Input", "Model", "Prediction"], + "inputs": [{"id": "input", "name": "Input", "evidence_ids": ["E_A"]}], + "modules": [{"id": "model", "name": "Model", "evidence_ids": ["E_B"]}], + "outputs": [{"id": "output", "name": "Prediction", "evidence_ids": ["E_A"]}], + "relations": [ + {"source": "input", "target": "model", "type": "data_flow"}, + {"source": "model", "target": "output", "type": "prediction"}, + ], + "overview_panels": [ + {"id": "panel_a", "panel_label": "a", "label": "Training and usage", "evidence_ids": ["E_A"], "entity_ids": ["input", "output"]}, + {"id": "panel_b", "panel_label": "b", "label": "Architecture", "evidence_ids": ["E_B"], "entity_ids": ["model"]}, + ], + } + + report = compile_semantic_blueprint(specification) + nodes = {item["id"]: item for item in report["nodes"]} + + self.assertEqual(len(report["panels"]), 2) + self.assertEqual(nodes["input"]["panel_id"], "panel_a") + self.assertEqual(nodes["model"]["panel_id"], "panel_b") + forward = next(item for item in report["connectors"] if item["source"] == "input") + self.assertNotEqual(forward["route_style"], "outer_feedback") + + def test_panel_instance_graph_preserves_repeated_labels_and_local_connectors(self): + specification = { + "topology": "dense_multiframe", + "panel_graphs": [ + { + "id": "panel_a", + "panel_label": "a", + "label": "Training and usage", + "nodes": [ + {"instance_id": "a_data", "name": "Dataset", "role": "input", "evidence_ids": ["E1"]}, + {"instance_id": "a_model_train", "name": "Model", "role": "module", "evidence_ids": ["E2"]}, + {"instance_id": "a_prediction", "name": "Prediction", "role": "output", "evidence_ids": ["E3"]}, + {"instance_id": "a_model_use", "name": "Model", "role": "module", "evidence_ids": ["E4"]}, + ], + "relations": [ + {"source": "a_data", "target": "a_model_train", "type": "data_flow"}, + {"source": "a_model_train", "target": "a_prediction", "type": "prediction"}, + ], + }, + { + "id": "panel_b", + "panel_label": "b", + "label": "Architecture", + "nodes": [ + {"instance_id": "b_model", "name": "Model", "role": "group", "evidence_ids": ["E5"]}, + {"instance_id": "b_attention", "name": "Attention", "role": "module", "evidence_ids": ["E6"]}, + {"instance_id": "b_output", "name": "Output", "role": "output", "evidence_ids": ["E7"]}, + ], + "relations": [ + {"source": "b_attention", "target": "b_output", "type": "prediction"}, + ], + }, + ], + } + + report = compile_semantic_blueprint(specification) + + self.assertTrue(report["applied"]) + self.assertEqual(report["source"], "panel_graphs") + self.assertEqual(report["node_count"], 6) + self.assertEqual(sum(item["label"] == "Model" for item in report["nodes"]), 2) + self.assertEqual(report["panels"][1]["groups"][0]["label"], "Model") + self.assertTrue(all(item["panel_id"] == "panel_a" for item in report["connectors"][:2])) + self.assertEqual(report["connectors"][2]["panel_id"], "panel_b") + self.assertIn("a_model_use", report["isolated_node_ids"]) + if __name__ == "__main__": unittest.main() From 47bd591643a0d8ae61984014d28e211d59957914 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 05:49:20 +0800 Subject: [PATCH 68/74] Fix editable framework compatibility --- rfs/control_localizer.py | 17 +++-------------- rfs/cv_compat.py | 16 ++++++++++++++++ rfs/program_builder.py | 5 ++++- rfs/reference_analyzer.py | 13 +++++++++++-- rfs/text_layer_ownership.py | 4 ++-- tests/unit/test_control_localizer.py | 10 +++++----- tests/unit/test_program_builder.py | 23 +++++++++++++++++++++++ tests/unit/test_text_layer_ocr.py | 4 +++- 8 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 rfs/cv_compat.py create mode 100644 tests/unit/test_program_builder.py diff --git a/rfs/control_localizer.py b/rfs/control_localizer.py index b3c9f19..63b8f20 100644 --- a/rfs/control_localizer.py +++ b/rfs/control_localizer.py @@ -6,6 +6,8 @@ from PIL import Image, ImageDraw +from .cv_compat import hough_line_coordinates + def _center(box: dict) -> tuple[float, float]: return float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2 @@ -51,19 +53,6 @@ def _sequence_controls(slots: list[dict], palette: list[str]) -> list[dict]: return arrows -def _hough_line_coordinates(line) -> tuple[int, int, int, int] | None: - """Normalize OpenCV HoughLinesP rows across array-shape variants.""" - try: - values = line.reshape(-1).tolist() if hasattr(line, "reshape") else line - while isinstance(values, (list, tuple)) and len(values) == 1 and isinstance(values[0], (list, tuple)): - values = values[0] - if not isinstance(values, (list, tuple)) or len(values) < 4: - return None - return tuple(int(value) for value in values[:4]) - except (TypeError, ValueError): - return None - - def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str]) -> list[dict]: try: import cv2 @@ -80,7 +69,7 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str return [] candidates = [] for line in lines[:80]: - coordinates = _hough_line_coordinates(line) + coordinates = hough_line_coordinates(line) if coordinates is None: continue x1, y1, x2, y2 = coordinates diff --git a/rfs/cv_compat.py b/rfs/cv_compat.py new file mode 100644 index 0000000..53abbee --- /dev/null +++ b/rfs/cv_compat.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from typing import Any + + +def hough_line_coordinates(line: Any) -> tuple[int, int, int, int] | None: + """Normalize OpenCV HoughLinesP rows across array-shape variants.""" + try: + values = line.reshape(-1).tolist() if hasattr(line, "reshape") else line + while isinstance(values, (list, tuple)) and len(values) == 1 and isinstance(values[0], (list, tuple)): + values = values[0] + if not isinstance(values, (list, tuple)) or len(values) < 4: + return None + return tuple(int(value) for value in values[:4]) + except (TypeError, ValueError): + return None diff --git a/rfs/program_builder.py b/rfs/program_builder.py index d85e156..9267dc9 100644 --- a/rfs/program_builder.py +++ b/rfs/program_builder.py @@ -124,7 +124,10 @@ def build_figure_program(paper_brief: dict, inventory: dict, style: dict, out_di canvas_ratio = float(inventory.get("canvas_aspect_ratio") or 16 / 9) height_in = 7.5 - width_in = max(13.333, min(16.0, height_in * canvas_ratio)) + width_in = height_in * canvas_ratio + if width_in > 16.0: + width_in = 16.0 + height_in = width_in / canvas_ratio program = { "summary": "Structured figure program used as the only layout source for PPT composition.", "canvas": {"width_in": round(width_in, 3), "height_in": height_in, "ratio": round(canvas_ratio, 4), "background": "#FFFFFF"}, diff --git a/rfs/reference_analyzer.py b/rfs/reference_analyzer.py index 811a7c3..f6e5c6b 100644 --- a/rfs/reference_analyzer.py +++ b/rfs/reference_analyzer.py @@ -10,6 +10,7 @@ import requests from PIL import Image, ImageDraw +from .cv_compat import hough_line_coordinates from .utils import ratio_string, write_json @@ -374,8 +375,11 @@ def _detect_cv_control_candidates(reference_image: Image.Image, image_width: int raw: list[tuple[float, float, list[list[float]]]] = [] diag = (image_width * image_width + image_height * image_height) ** 0.5 - for line in lines[:, 0, :]: - x1, y1, x2, y2 = [float(v) for v in line] + for line in lines[:80]: + coordinates = hough_line_coordinates(line) + if coordinates is None: + continue + x1, y1, x2, y2 = [float(v) for v in coordinates] length = ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5 if length / max(diag, 1.0) < 0.025: continue @@ -1551,6 +1555,11 @@ def analyze_reference( reference_palette.append(color) reference_palette = _diverse_palette(reference_palette) for slot in slots: + if not slot.get("reference_dominant_colors"): + panel_id = str(slot.get("panel_id") or "") + panel_colors = panel_styles.get(panel_id, {}).get("dominant_colors", []) + fallback_colors = list(panel_colors[:3]) or list(reference_palette[:3]) or ["#4B86C5"] + slot["reference_dominant_colors"] = fallback_colors token_ids = [] for color_index, color in enumerate(slot.get("reference_dominant_colors", [])[:3], start=1): token_ids.append( diff --git a/rfs/text_layer_ownership.py b/rfs/text_layer_ownership.py index c213ac0..007a566 100644 --- a/rfs/text_layer_ownership.py +++ b/rfs/text_layer_ownership.py @@ -53,10 +53,10 @@ def _decide_heuristic(region: dict, program: dict) -> tuple[str, str]: role = str(region.get("role") or "") if not text: return "ignore", "empty text" - if _font_size(region) < 4.0 or len(text) <= 1: - return "decorative_asset_text", "tiny or one-character OCR text is treated as decorative asset text" if role in CRITICAL_EDITABLE_ROLES: return "editable_text_layer", f"{role} is a critical editable text role" + if _font_size(region) < 4.0 or len(text) <= 1: + return "decorative_asset_text", "tiny or one-character OCR text is treated as decorative asset text" containing_slots = _slot_ids_containing_text(region, program) target_id = str(region.get("target_id") or "") if target_id in containing_slots or (containing_slots and role in {"free_text", "body_label", "annotation"}): diff --git a/tests/unit/test_control_localizer.py b/tests/unit/test_control_localizer.py index ed2d1d0..8e23b1e 100644 --- a/tests/unit/test_control_localizer.py +++ b/tests/unit/test_control_localizer.py @@ -2,17 +2,17 @@ import numpy as np -from rfs.control_localizer import _hough_line_coordinates +from rfs.cv_compat import hough_line_coordinates class ControlLocalizerTests(unittest.TestCase): def test_hough_line_coordinates_accepts_opencv_shape_variants(self): - self.assertEqual(_hough_line_coordinates(np.array([[1, 2, 3, 4]], dtype=np.int32)), (1, 2, 3, 4)) - self.assertEqual(_hough_line_coordinates(np.array([1, 2, 3, 4], dtype=np.int32)), (1, 2, 3, 4)) - self.assertEqual(_hough_line_coordinates([[1, 2, 3, 4]]), (1, 2, 3, 4)) + self.assertEqual(hough_line_coordinates(np.array([[1, 2, 3, 4]], dtype=np.int32)), (1, 2, 3, 4)) + self.assertEqual(hough_line_coordinates(np.array([1, 2, 3, 4], dtype=np.int32)), (1, 2, 3, 4)) + self.assertEqual(hough_line_coordinates([[1, 2, 3, 4]]), (1, 2, 3, 4)) def test_hough_line_coordinates_rejects_incomplete_rows(self): - self.assertIsNone(_hough_line_coordinates(np.array([1, 2, 3], dtype=np.int32))) + self.assertIsNone(hough_line_coordinates(np.array([1, 2, 3], dtype=np.int32))) if __name__ == "__main__": diff --git a/tests/unit/test_program_builder.py b/tests/unit/test_program_builder.py new file mode 100644 index 0000000..18b0b0d --- /dev/null +++ b/tests/unit/test_program_builder.py @@ -0,0 +1,23 @@ +import tempfile +import unittest +from pathlib import Path + +from rfs.program_builder import build_figure_program + + +class ProgramBuilderTests(unittest.TestCase): + def test_canvas_preserves_reference_aspect_ratio(self): + inventory = {"canvas_aspect_ratio": 1.5, "reference_path": "reference.png"} + layout_plan = {"locator_mode": "heuristic", "panels": [], "slots": [], "arrows": []} + style = {"palette": [], "color_tokens": []} + + with tempfile.TemporaryDirectory() as tmp: + program = build_figure_program({}, inventory, style, Path(tmp), layout_plan) + + canvas = program["canvas"] + self.assertAlmostEqual(canvas["width_in"] / canvas["height_in"], 1.5, places=3) + self.assertEqual(canvas["ratio"], 1.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_text_layer_ocr.py b/tests/unit/test_text_layer_ocr.py index 7ab2a01..b15007e 100644 --- a/tests/unit/test_text_layer_ocr.py +++ b/tests/unit/test_text_layer_ocr.py @@ -47,6 +47,7 @@ def test_text_ownership_keeps_critical_labels_editable(self): } regions = [ {"id": "title", "text": "Method", "role": "panel_title", "target_id": "slot_a", "font_size_pt": 12, "bbox_percent": {"x": 0.2, "y": 0.2, "w": 0.2, "h": 0.1}}, + {"id": "inferred_title", "text": "Architecture", "role": "panel_title", "target_id": "slot_a", "bbox_percent": {"x": 0.2, "y": 0.1, "w": 0.2, "h": 0.05}}, {"id": "decorative", "text": "x", "role": "free_text", "target_id": "slot_a", "font_size_pt": 3, "bbox_percent": {"x": 0.3, "y": 0.3, "w": 0.05, "h": 0.05}}, ] @@ -54,8 +55,9 @@ def test_text_ownership_keeps_critical_labels_editable(self): by_id = {item["id"]: item for item in planned} self.assertEqual(by_id["title"]["layer_ownership"], "editable_text_layer") + self.assertEqual(by_id["inferred_title"]["layer_ownership"], "editable_text_layer") self.assertEqual(by_id["decorative"]["layer_ownership"], "decorative_asset_text") - self.assertEqual(report["editable_text_count"], 1) + self.assertEqual(report["editable_text_count"], 2) def test_fake_ocr_creates_reference_text_geometry_without_duplicate_panel_title(self): with tempfile.TemporaryDirectory() as tmp: From 622b6ea0e1708ef3acbc12feb8479cd7f7dc6a03 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 06:18:49 +0800 Subject: [PATCH 69/74] Bridge fast paper contracts into editable frameworks --- README.md | 32 ++ docs/workflow.md | 11 + pyproject.toml | 1 + rfs/arrow_router.py | 14 + rfs/asset_generator.py | 64 +--- rfs/cli.py | 15 + rfs/contracts/__init__.py | 4 +- rfs/contracts/semantic.py | 328 ++++++++++++++++++- rfs/layout_locator.py | 2 + rfs/reference_analyzer.py | 32 +- rfs/workflow.py | 77 ++++- rfs/workflows/paper_to_editable.py | 56 ++++ tests/unit/test_contract_framework_bridge.py | 99 ++++++ tests/unit/test_paper_to_editable_route.py | 51 +++ 14 files changed, 712 insertions(+), 74 deletions(-) create mode 100644 tests/unit/test_contract_framework_bridge.py create mode 100644 tests/unit/test_paper_to_editable_route.py diff --git a/README.md b/README.md index aee3976..8dfa28b 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,38 @@ rfs validate --out "output\demo_placeholder" --json `output/` is intentionally ignored by Git. +## Reuse the Fast Paper Contract + +When `fast-framework-prompt` or `paper-to-image` has already produced a validated +`figure_specification.json`, pass that run directory directly to +`make-framework`. This bypasses the legacy paper planner: panel-local node +instances, exact paper labels, evidence IDs, and panel-local relations become +the scientific authority for slots, editable text, and PPT connectors. + +```powershell +rfs make-framework ` + --paper "C:\path\paper.pdf" ` + --reference "output\paper_to_image\run\selected_image.png" ` + --paper-contract-dir "output\paper_to_image\run" ` + --out "output\editable\contract_driven" ` + --slot-count 25 ` + --slot-source reference-primary ` + --asset-mode placeholder ` + --candidates-per-slot 1 ` + --asset-workers 3 ` + --locator-mode heuristic ` + --control-localizer-mode off ` + --prompt-plan-mode heuristic ` + --text-extractor-mode heuristic ` + --ocr-engine off ` + --no-export ` + --json +``` + +For the complete paper-to-image-to-editable route, use +`paper-to-editable --editable-route framework`. The default `rebuild` route is +kept for compatibility with older image-reconstruction runs. + ## Real VLM + Image Generation Set API credentials only through environment variables. Do not write keys into source files. diff --git a/docs/workflow.md b/docs/workflow.md index 515ca6a..94d846e 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -33,6 +33,17 @@ rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.pn rfs validate --out .\output\offline_check --json ``` +Contract-driven engineering check (recommended after a fast/paper-to-image run): + +```powershell +rfs make-framework --paper "C:\path\paper.pdf" --reference ".\output\paper_to_image\run\selected_image.png" --paper-contract-dir ".\output\paper_to_image\run" --out .\output\contract_check --asset-mode placeholder --asset-workers 3 --locator-mode heuristic --control-localizer-mode off --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --text-extractor-mode heuristic --ocr-engine off --no-export --json +``` + +This route does not call the legacy paper planner. `panel_graphs` instance IDs +remain distinct across panels, local relations are authoritative, exact labels +are editable PPT text, and any extra slots needed to reach 25 are reference +visual splits rather than summary-sentence fillers. + Small real image check: ```powershell diff --git a/pyproject.toml b/pyproject.toml index 4b8768a..cc3c97a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ version = "0.1.0" description = "PPTX-first research figure generation pipeline guided by papers and reference images." requires-python = ">=3.10" dependencies = [ + "numpy", "pillow", "python-pptx", "PyMuPDF", diff --git a/rfs/arrow_router.py b/rfs/arrow_router.py index 99aef2b..91cd8c4 100644 --- a/rfs/arrow_router.py +++ b/rfs/arrow_router.py @@ -476,6 +476,18 @@ def style_and_route_arrows( panels = program.get("panels", []) if isinstance(program.get("panels"), list) else [] slots = program.get("slots", []) if isinstance(program.get("slots"), list) else [] + program_style = program.get("style") if isinstance(program.get("style"), dict) else {} + color_tokens = program_style.get("color_tokens", []) if isinstance(program_style.get("color_tokens"), list) else [] + default_style_token_id = next( + ( + str(item.get("token_id") or "") + for item in color_tokens + if isinstance(item, dict) and any(term in str(item.get("usage") or "").casefold() for term in ("arrow", "connector", "line")) + ), + "", + ) + if not default_style_token_id: + default_style_token_id = next((str(item.get("token_id") or "") for item in color_tokens if isinstance(item, dict) and item.get("token_id")), "") objects = {str(item.get("id")): item for item in panels + slots if isinstance(item, dict) and item.get("id")} slot_objects = {str(item.get("id")): item for item in slots if isinstance(item, dict) and item.get("id")} panel_ids = {str(panel.get("id")) for panel in panels if isinstance(panel, dict)} @@ -504,6 +516,8 @@ def style_and_route_arrows( arrow["target_id"] = target arrow["source"] = source arrow["target"] = target + if not str(arrow.get("style_token_id") or "").strip(): + arrow["style_token_id"] = default_style_token_id points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] locked = _reference_locked(arrow) route_meta: dict[str, Any] = {"routing_algorithm": "preserve_reference_path", "route_generation_status": "reference_locked"} diff --git a/rfs/asset_generator.py b/rfs/asset_generator.py index 51abb66..bca97af 100644 --- a/rfs/asset_generator.py +++ b/rfs/asset_generator.py @@ -12,6 +12,7 @@ from typing import Iterable import requests +import numpy as np from PIL import Image, ImageDraw, ImageFont from .utils import ensure_dir, write_json, write_text @@ -196,7 +197,7 @@ def build_slot_prompt(slot: dict, style: dict, candidate_index: int = 1) -> str: def _draw_placeholder(slot: dict, path: Path, style: dict, candidate_index: int = 1) -> None: - width, height = _target_size(slot["target_canvas_ratio"]) + width, height = _target_size(slot["target_canvas_ratio"], max_side=512) palette = style.get("palette", ["#EAF5FF", "#DFF7EF", "#8EC9E8", "#66C6A4", "#163B4D"]) bg = palette[(candidate_index - 1) % 2] line = palette[4] if len(palette) > 4 else "#163B4D" @@ -404,22 +405,17 @@ def _content_bbox(rgba: Image.Image) -> tuple[list[int], float, float]: width, height = rgba.size pixels = rgba.load() background = _median_corner_color(pixels, width, height) - xs: list[int] = [] - ys: list[int] = [] - color_threshold = 30.0 - alpha_threshold = 12 - for y in range(height): - for x in range(width): - r, g, b, a = pixels[x, y] - if a <= alpha_threshold: - continue - if a < 245 or _color_distance((r, g, b), background) > color_threshold: - xs.append(x) - ys.append(y) - if not xs or not ys: + rgba_array = np.asarray(rgba, dtype=np.int16) + rgb = rgba_array[:, :, :3] + alpha = rgba_array[:, :, 3] + background_array = np.asarray(background, dtype=np.float32) + color_distance_sq = np.sum((rgb.astype(np.float32) - background_array) ** 2, axis=2) + mask = (alpha > 12) & ((alpha < 245) | (color_distance_sq > 30.0 ** 2)) + ys, xs = np.nonzero(mask) + if xs.size == 0 or ys.size == 0: return [0, 0, width, height], 0.0, 100.0 - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) + left, right = int(xs.min()), int(xs.max()) + top, bottom = int(ys.min()), int(ys.max()) bbox_w = max(1, right - left + 1) bbox_h = max(1, bottom - top + 1) content_fill = bbox_w * bbox_h / float(width * height) * 100.0 @@ -524,40 +520,7 @@ def _estimate_quality(path: Path, slot: dict, candidate_index: int, source: str) with Image.open(path) as image: rgba = image.convert("RGBA") width, height = rgba.size - pixels = rgba.load() - background = _median_corner_color(pixels, width, height) - - xs: list[int] = [] - ys: list[int] = [] - color_threshold = 30.0 - alpha_threshold = 12 - for y in range(height): - for x in range(width): - r, g, b, a = pixels[x, y] - if a <= alpha_threshold: - continue - if a < 245 or _color_distance((r, g, b), background) > color_threshold: - xs.append(x) - ys.append(y) - - if not xs or not ys: - content_fill = 0.0 - empty_margin = 100.0 - bbox = [0, 0, width, height] - else: - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) - bbox_w = max(1, right - left + 1) - bbox_h = max(1, bottom - top + 1) - bbox = [left, top, right + 1, bottom + 1] - content_fill = bbox_w * bbox_h / float(width * height) * 100.0 - margins = ( - left / width * 100.0, - (width - right - 1) / width * 100.0, - top / height * 100.0, - (height - bottom - 1) / height * 100.0, - ) - empty_margin = max(margins) + bbox, content_fill, empty_margin = _content_bbox(rgba) image_ratio = width / max(height, 1) target_ratio = _ratio_value(slot["target_canvas_ratio"]) @@ -760,7 +723,6 @@ def _generate_one_candidate(slot: dict, style: dict, path: Path, asset_mode: str support_color = str(style["reference_palette"][0]) if asset_mode == "placeholder": _draw_placeholder(slot, path, style, candidate_index=candidate_index) - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) return "explicit_placeholder_candidate" if asset_mode == "gemini": diff --git a/rfs/cli.py b/rfs/cli.py index 4e8dd20..ba5379d 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -171,6 +171,9 @@ def build_parser() -> argparse.ArgumentParser: make.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") make.add_argument("--reference", required=True, help="User-provided visual reference image path.") make.add_argument("--out", required=True, help="Output directory.") + contract_source = make.add_mutually_exclusive_group() + contract_source.add_argument("--paper-contract-dir", help="Fast paper-to-image/fast-framework output directory containing figure_specification.json. Bypasses the legacy paper planner.") + contract_source.add_argument("--semantic-contract", help="Semantic contract JSON or figure_specification.json. Bypasses the legacy paper planner.") make.add_argument("--profile", default="ai-ml-paper", help="Figure profile. Default: ai-ml-paper.") make.add_argument("--slot-count", type=int, default=36, help="Target slot count, clamped to 25-50. Default: 36.") make.add_argument("--slot-source", choices=["paper", "reference-primary"], default="reference-primary", help="Slot content source. Default reference-primary makes the reference figure drive visual objects, layout, color, and flow logic.") @@ -243,6 +246,11 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") paper_editable.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") paper_editable.add_argument("--repair-rounds", type=int, default=1) + paper_editable.add_argument("--editable-route", choices=["rebuild", "framework"], default="rebuild", help="Editable compiler. Use framework for the authoritative 25-50 slot paper-contract route; rebuild preserves the legacy image reconstruction route.") + paper_editable.add_argument("--framework-slot-count", type=int, default=25, help="Non-arrow slot count for --editable-route framework. Default: 25.") + paper_editable.add_argument("--framework-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset backend for --editable-route framework.") + paper_editable.add_argument("--framework-candidates-per-slot", type=int, default=1, help="Asset candidates per slot for --editable-route framework. Default: 1.") + paper_editable.add_argument("--framework-asset-workers", type=int, default=3, help="Parallel slot generation workers for --editable-route framework. Default: 3.") paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") paper_editable.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") @@ -449,6 +457,8 @@ def main(argv: list[str] | None = None) -> int: presentations_workspace=args.presentations_workspace, presentations_scale=args.presentations_scale, export=not args.no_export, + semantic_contract=args.semantic_contract, + paper_contract_dir=args.paper_contract_dir, ) elif args.command == "paper-to-image": result = run_paper_to_image( @@ -505,6 +515,11 @@ def main(argv: list[str] | None = None) -> int: allow_engineering_preview=args.allow_engineering_preview, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, + editable_route=args.editable_route, + framework_slot_count=args.framework_slot_count, + framework_asset_mode=args.framework_asset_mode, + framework_candidates_per_slot=args.framework_candidates_per_slot, + framework_asset_workers=args.framework_asset_workers, ) elif args.command == "rebuild-editable": rebuild_adapters = build_rebuild_vlm_adapters(args.out) diff --git a/rfs/contracts/__init__.py b/rfs/contracts/__init__.py index 7c8325d..c866d03 100644 --- a/rfs/contracts/__init__.py +++ b/rfs/contracts/__init__.py @@ -1,5 +1,5 @@ """Stable data-contract APIs shared across ResearchFigureStudio workflows.""" -from .semantic import apply_paper_semantic_contract, load_paper_semantic_contract +from .semantic import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract, load_paper_semantic_contract -__all__ = ["apply_paper_semantic_contract", "load_paper_semantic_contract"] +__all__ = ["apply_paper_semantic_contract", "compile_paper_brief_from_semantic_contract", "load_paper_semantic_contract"] diff --git a/rfs/contracts/semantic.py b/rfs/contracts/semantic.py index cbd6894..9cea2ca 100644 --- a/rfs/contracts/semantic.py +++ b/rfs/contracts/semantic.py @@ -4,20 +4,37 @@ from pathlib import Path from typing import Any -from ..utils import read_json, write_json +from ..utils import read_json, write_json, write_text def load_paper_semantic_contract(run_dir: str | Path) -> dict[str, Any]: root = Path(run_dir) + if root.is_file(): + payload = read_json(root) + if root.name == "figure_specification.json" or "panel_graphs" in payload or "relations" in payload: + return { + "summary": "Paper-grounded semantic contract loaded from a figure specification.", + "source_run_dir": str(root.parent.resolve()), + "figure_specification": payload, + } + return payload + + def optional(name: str) -> dict[str, Any]: + path = root / name + return read_json(path) if path.exists() else {} + + specification_path = root / "figure_specification.json" + if not specification_path.exists(): + raise FileNotFoundError(f"semantic contract directory has no figure_specification.json: {root}") return { "summary": "Paper-grounded semantic contract for editable figure reconstruction.", "source_run_dir": str(root.resolve()), - "paper_review": read_json(root / "paper_review.json"), + "paper_review": optional("paper_review.json"), "figure_specification": read_json(root / "figure_specification.json"), - "paper_summary": read_json(root / "paper_summary.json"), - "design_plan": read_json(root / "design_plan.json"), - "layout_intent": read_json(root / "layout_intent.json"), - "style_plan": read_json(root / "style_plan.json"), + "paper_summary": optional("paper_summary.json"), + "design_plan": optional("design_plan.json"), + "layout_intent": optional("layout_intent.json"), + "style_plan": optional("style_plan.json"), } @@ -29,8 +46,48 @@ def _text(item: dict, fallback: str = "") -> str: return fallback +def _specification(contract: dict) -> dict: + return contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + + +def _panel_graph_entities(spec: dict) -> list[dict]: + entities: list[dict] = [] + seen_ids: set[str] = set() + for panel_index, panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + continue + panel_id = str(panel.get("id") or f"panel_{panel_index + 1}").strip() + panel_label = str(panel.get("label") or panel.get("description") or panel_id).strip() + for node_index, raw in enumerate(panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []): + if not isinstance(raw, dict): + continue + role = str(raw.get("role") or "module").strip() + if role.casefold() in {"group", "container", "panel", "stage"}: + continue + label = _text(raw) + entity_id = str(raw.get("instance_id") or raw.get("id") or f"{panel_id}_node_{node_index + 1:02d}").strip() + if not label or not entity_id or entity_id in seen_ids: + continue + seen_ids.add(entity_id) + entities.append({ + "id": entity_id, + "semantic_id": str(raw.get("semantic_id") or "").strip() or None, + "label": label, + "role": role, + "panel_id": panel_id, + "panel_label": panel_label, + "bbox_percent": raw.get("bbox_percent"), + "evidence_ids": [str(value) for value in raw.get("evidence_ids", []) if value], + "source": raw, + }) + return entities + + def _entities(contract: dict) -> list[dict]: - spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract + spec = _specification(contract) + panel_entities = _panel_graph_entities(spec) + if panel_entities: + return panel_entities required_labels = { re.sub(r"[^a-z0-9]+", "", str(value).casefold()) for value in spec.get("required_labels", []) @@ -61,6 +118,153 @@ def _entities(contract: dict) -> list[dict]: return entities +def _relations(contract: dict) -> tuple[list[dict], str]: + spec = _specification(contract) + panel_entities = _panel_graph_entities(spec) + if panel_entities: + entity_ids = {item["id"] for item in panel_entities} + relations: list[dict] = [] + for panel_index, panel in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(panel, dict): + continue + panel_id = str(panel.get("id") or f"panel_{panel_index + 1}").strip() + for relation_index, raw in enumerate(panel.get("relations", []) if isinstance(panel.get("relations"), list) else []): + if not isinstance(raw, dict): + continue + source = str(raw.get("source") or raw.get("source_id") or "").strip() + target = str(raw.get("target") or raw.get("target_id") or "").strip() + if not source or not target or source == target or source not in entity_ids or target not in entity_ids: + continue + relations.append({ + **raw, + "id": str(raw.get("id") or f"{panel_id}_relation_{relation_index + 1:02d}"), + "source": source, + "target": target, + "panel_id": panel_id, + }) + return relations, "panel_graphs" + return [item for item in spec.get("relations", []) if isinstance(item, dict)] if isinstance(spec.get("relations"), list) else [], "global_graph" + + +def _composition_type(role: str, label: str) -> str: + low = f"{role} {label}".casefold() + if any(token in low for token in ("input", "dataset", "sample", "image", "text", "audio", "video")): + return "scene_thumbnail" + if any(token in low for token in ("output", "prediction", "loss", "objective", "score", "distribution")): + return "full_frame_icon" + return "full_bleed_card" + + +def compile_paper_brief_from_semantic_contract( + contract: dict, + out_dir: str | Path | None = None, + *, + source_path: str | Path | None = None, +) -> dict[str, Any]: + """Compile the fast paper contract into the legacy brief shape without rereading the paper.""" + + spec = _specification(contract) + entities = _entities(contract) + relations, relation_source = _relations(contract) + panels = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict)] if isinstance(spec.get("panel_graphs"), list) else [] + modules = [str(item.get("label") or item.get("description") or item.get("id") or "").strip() for item in panels] + modules = [item for item in modules if item] + if not modules: + modules = list(dict.fromkeys(item["panel_label"] for item in entities if item.get("panel_label"))) + if not modules: + modules = list(dict.fromkeys(item["label"] for item in entities if item["role"].casefold() in {"module", "stage"})) + if not modules: + modules = ["Paper-grounded method"] + + outgoing: dict[str, list[str]] = {} + incoming: dict[str, list[str]] = {} + labels_by_id = {item["id"]: item["label"] for item in entities} + for relation in relations: + source = str(relation.get("source") or "") + target = str(relation.get("target") or "") + outgoing.setdefault(source, []).append(labels_by_id.get(target, target)) + incoming.setdefault(target, []).append(labels_by_id.get(source, source)) + + suggestions = [] + for entity in entities: + panel = entity.get("panel_label") or modules[0] + neighbor_cues = [*incoming.get(entity["id"], []), *outgoing.get(entity["id"], [])] + must_show = [f"paper-grounded visual object for {entity['label']}", "local reference shape and visual density"] + if neighbor_cues: + must_show.append("declared local context: " + ", ".join(list(dict.fromkeys(neighbor_cues))[:3])) + suggestions.append({ + "id": entity["id"], + "macro_panel": panel, + "paper_concept": entity["label"], + "composition_type": _composition_type(entity["role"], entity["label"]), + "visual_metaphor": f"reference-grounded scientific visual for the paper term {entity['label']}", + "must_show": must_show, + "avoid_showing": ["unmentioned architecture modules", "fake formulas", "fake numeric chart text", "critical text baked into the image"], + "semantic_instance_id": entity["id"], + "semantic_id": entity.get("semantic_id"), + "semantic_panel_id": entity.get("panel_id"), + "semantic_role": entity["role"], + "evidence_ids": entity["evidence_ids"], + "contract_source": relation_source, + }) + + review = contract.get("paper_review") if isinstance(contract.get("paper_review"), dict) else {} + summary = contract.get("paper_summary") if isinstance(contract.get("paper_summary"), dict) else {} + title = str(review.get("title") or review.get("paper_title") or summary.get("title") or "Paper-grounded framework").strip() + goal = str(spec.get("figure_goal") or "").strip() + if not goal: + central_claim = spec.get("central_claim") + goal = _text(central_claim) if isinstance(central_claim, dict) else str(central_claim or "").strip() + if not goal: + goal = "Visualize only the modules and local relations declared in the paper semantic contract." + + brief = { + "summary": "Paper brief compiled directly from the fast evidence-grounded semantic contract.", + "planner": "semantic_contract", + "semantic_authority": relation_source, + "title_guess": title, + "figure_kind": str(spec.get("topology") or "paper_driven_generic"), + "figure_goal": goal, + "modules": modules[:8], + "concepts": [item["label"] for item in entities], + "variables": [str(item) for item in spec.get("required_labels", []) if str(item).strip()], + "slot_suggestions": suggestions, + "semantic_relations": relations, + "semantic_entity_count": len(entities), + "semantic_relation_count": len(relations), + "source_path": str(source_path or contract.get("source_run_dir") or ""), + "source_loader": "semantic_contract", + "char_count": 0, + "warnings": [] if relation_source == "panel_graphs" else ["panel_graphs unavailable; using the compatibility global graph"], + } + if out_dir is not None: + out = Path(out_dir) + write_json(out / "paper_brief.json", brief) + lines = [ + "# Summary", + "Paper brief compiled directly from the fast evidence-grounded semantic contract.", + "", + "## Source", + f"- Contract authority: {relation_source}", + f"- Source: `{brief['source_path']}`", + f"- Title: {title}", + "", + "## Figure Goal", + goal, + "", + "## Panels", + *[f"- {item}" for item in modules], + "", + "## Contract Entities", + *[f"- {item['id']}: {item['label']} ({item['role']})" for item in entities], + "", + "## Contract Relations", + *[f"- {item.get('source')} -> {item.get('target')} ({item.get('type') or 'data_flow'})" for item in relations], + ] + write_text(out / "paper_brief.md", "\n".join(lines) + "\n") + return brief + + def _tokens(value: object) -> set[str]: return {token for token in re.findall(r"[a-z0-9]+", str(value or "").lower()) if len(token) > 1} @@ -114,6 +318,12 @@ def object_priority(obj: dict) -> int: searchable = object_search_text(obj) obj_tokens = _tokens(searchable) score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) + semantic_instance_id = str(obj.get("semantic_instance_id") or "").strip() + if semantic_instance_id and semantic_instance_id == entity["id"]: + score += 10.0 + semantic_panel_id = str(obj.get("semantic_panel_id") or "").strip() + if semantic_panel_id and semantic_panel_id == str(entity.get("panel_id") or ""): + score += 0.5 entity_compact = re.sub(r"[^a-z0-9]+", "", str(entity.get("label") or "").casefold()) object_compact = re.sub(r"[^a-z0-9]+", "", searchable.casefold()) if entity_compact and entity_compact in object_compact: @@ -211,7 +421,9 @@ def mapped_container(obj: dict | None) -> str | None: def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: + out = Path(out_dir) entities = _entities(contract) + relations, relation_source = _relations(contract) mapping, mapping_records = _assign_entities(entities, program) mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} @@ -223,32 +435,61 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | and not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes) ] semantic_text = [] + semantic_reference_regions = [] for entity in entities: obj = mapping.get(entity["id"]) if not obj: continue box = _label_bbox(obj) + reference_text_id = f"semantic_ref_text_{entity['id']}" + center = {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)} + estimated_font_ratio = round(max(0.006, box["h"] * 0.55), 6) semantic_text.append({ "id": f"semantic_text_{entity['id']}", "text": entity["label"], "role": f"paper_{entity['role']}_label", "target_id": obj.get("id"), "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), "semantic_evidence_ids": entity["evidence_ids"], + "source_reference_text_id": reference_text_id, + "reference_binding": "reference_slot_semantic_overlay_geometry", "bbox_percent": box, - "center_percent": {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)}, + "center_percent": center, "width_percent": box["w"], "height_percent": box["h"], + "estimated_font_ratio": estimated_font_ratio, "font_size_pt": 10.0, "color_hex": "#263747", "bold": entity["role"] in {"module", "innovation"}, "align": "center", "font_family_guess": "Arial", + "font_weight_guess": "bold" if entity["role"] in {"module", "innovation"} else "regular", "fit_strategy": "paper_semantic_contract_exact_label", "editable_in": "pptx", "visible": True, "layer_ownership": "editable_text_layer", }) + semantic_reference_regions.append({ + "id": reference_text_id, + "text": entity["label"], + "raw_text": entity["label"], + "role": f"paper_{entity['role']}_label", + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "evidence_ids": entity["evidence_ids"], + "bbox_percent": box, + "center_percent": center, + "width_percent": box["w"], + "height_percent": box["h"], + "estimated_font_ratio": estimated_font_ratio, + "color_hex": "#263747", + "font_family_guess": "Arial", + "font_weight_guess": "bold" if entity["role"] in {"module", "innovation"} else "regular", + "source": "reference_slot_semantic_overlay_geometry", + "editable_in": "pptx", + }) text_program = { **text_program, "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", @@ -257,13 +498,53 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | } program["text_program"] = text_program program["text_program_path"] = "text_program.json" + program["reference_text_geometry_path"] = "reference_text_geometry.json" + program["text_alignment_report_path"] = "text_alignment_report.json" + + geometry_path = out / "reference_text_geometry.json" + if geometry_path.exists(): + reference_geometry = read_json(geometry_path) + else: + reference_geometry = { + "summary": "Reference-first text geometry with semantic overlay bindings.", + "policy": "reference_image_is_highest_authority_for_text_size_position_color_and_hierarchy", + "detection_mode": "reference_geometry_and_semantic_overlay", + "text_regions": [], + } + geometry_regions = [ + item for item in reference_geometry.get("text_regions", []) + if isinstance(item, dict) and not str(item.get("id") or "").startswith("semantic_ref_text_") + ] + reference_geometry["text_regions"] = geometry_regions + semantic_reference_regions + reference_geometry["semantic_overlay_policy"] = "exact paper terminology bound to reference-derived slot geometry" + + alignment_items = [] + for item in text_program["items"]: + alignment_items.append({ + "label_id": item.get("id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "target_id": item.get("target_id"), + "center_delta_percent": 0.0, + "width_delta_percent": 0.0, + "height_delta_percent": 0.0, + "font_ratio_delta": 0.0, + "color_match": True, + "editable_in": "pptx", + "status": "pass", + }) + alignment_report = { + "summary": "Reference alignment report after applying exact paper-semantic overlay labels.", + "policy": "reference_alignment_only; semantic labels inherit their mapped reference slot geometry", + "status": "pass", + "max_center_delta_percent": 0.0, + "max_width_delta_percent": 0.0, + "max_height_delta_percent": 0.0, + "items": alignment_items, + } - spec = contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract semantic_arrows = [] skipped_relations = [] - for index, relation in enumerate(spec.get("relations") if isinstance(spec.get("relations"), list) else [], start=1): - if not isinstance(relation, dict): - continue + for index, relation in enumerate(relations, start=1): source_id = str(relation.get("source") or relation.get("source_id") or "") target_id = str(relation.get("target") or relation.get("target_id") or "") source, target = mapping.get(source_id), mapping.get(target_id) @@ -276,18 +557,34 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "target_id": str(target.get("id")), "semantic_source_id": source_id, "semantic_target_id": target_id, + "semantic_panel_id": relation.get("panel_id"), "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), "label": str(relation.get("label") or relation.get("statement") or ""), "evidence_ids": list(relation.get("evidence_ids") or []), "path_percent": _anchor(source, target), + "source_anchor": "auto", + "target_anchor": "auto", + "control_kind": "elbow_connector", "render_style": "elbow_connector", "route_intent": "paper_semantic_relation", "stroke_color": "#3F5063", "stroke_width_pt": 1.8, "arrowhead": "triangle", "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", }) preserved_reference_arrows, replaced_reference_arrow_ids = _preserve_local_reference_arrows(program, mapping) + if relation_source == "panel_graphs": + heuristic_defaults = [ + item for item in preserved_reference_arrows + if str(item.get("binding_source") or "").startswith("heuristic_default_") + ] + if heuristic_defaults: + replaced_reference_arrow_ids.extend(str(item.get("id") or "heuristic_default_arrow") for item in heuristic_defaults) + preserved_reference_arrows = [ + item for item in preserved_reference_arrows + if not str(item.get("binding_source") or "").startswith("heuristic_default_") + ] if mapping: program["arrows"] = [*preserved_reference_arrows, *semantic_arrows] @@ -296,10 +593,11 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "summary": "Paper semantic contract application report.", "status": "pass" if len(mapping) == len(entities) and not skipped_relations and not low_confidence_mappings else "warning", "semantic_authority": "paper_ground_truth", + "contract_graph_source": relation_source, "visual_authority": "generated_reference_image", "entity_count": len(entities), "mapped_entity_count": len(mapping), - "relation_count": len(spec.get("relations") or []), + "relation_count": len(relations), "mapped_relation_count": len(semantic_arrows), "preserved_reference_arrow_count": len(preserved_reference_arrows), "replaced_reference_arrow_ids": replaced_reference_arrow_ids, @@ -308,8 +606,10 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "skipped_relations": skipped_relations, "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", } - out = Path(out_dir) write_json(out / "paper_semantic_contract.json", contract) write_json(out / "semantic_binding_report.json", report) + write_json(out / "reference_text_geometry.json", reference_geometry) write_json(out / "text_program.json", text_program) + write_json(out / "text_alignment_report.json", alignment_report) + write_json(out / "figure_program.json", program) return program, report diff --git a/rfs/layout_locator.py b/rfs/layout_locator.py index 2eb9967..38fd6ee 100644 --- a/rfs/layout_locator.py +++ b/rfs/layout_locator.py @@ -146,6 +146,7 @@ def _default_arrows(panels: list[dict]) -> list[dict]: "path_percent": [], "label": "", "editable_in": "pptx", + "binding_source": "heuristic_default_panel_flow", }) if any(p["id"] == "shared_resource_library" for p in panels): arrows.append({ @@ -156,6 +157,7 @@ def _default_arrows(panels: list[dict]) -> list[dict]: "path_percent": [], "label": "shared resources", "editable_in": "pptx", + "binding_source": "heuristic_default_resource_bus", }) return arrows diff --git a/rfs/reference_analyzer.py b/rfs/reference_analyzer.py index f6e5c6b..2605d29 100644 --- a/rfs/reference_analyzer.py +++ b/rfs/reference_analyzer.py @@ -607,17 +607,21 @@ def _supplement_reference_slots(slot_specs: list[Any], target_count: int) -> lis panel = str(_spec_value(base, "macro_panel") or "Paper Method") concept = str(_spec_value(base, "paper_concept") or base_id) composition = str(_spec_value(base, "composition_type") or "full_bleed_card") - supplemented.append({ - "id": f"{base_id}_detail_{detail_index:02d}", + detail = dict(base) if isinstance(base, dict) else {} + detail.update({ + "id": f"{base_id}_reference_detail_{detail_index:02d}", "macro_panel": panel, - "paper_concept": f"{concept} local detail {detail_index}", + "paper_concept": concept, "composition_type": "scene_thumbnail" if composition != "symbol_cutout" else "full_frame_icon", "visual_metaphor": str(_spec_value(base, "visual_metaphor") or ""), "must_show": _spec_value(base, "must_show") if isinstance(_spec_value(base, "must_show"), list) else [], "avoid_showing": _spec_value(base, "avoid_showing") if isinstance(_spec_value(base, "avoid_showing"), list) else [], "display_label": "", "show_slot_caption": False, + "reference_detail_index": detail_index, + "slot_origin": "reference_visual_split", }) + supplemented.append(detail) detail_index += 1 return supplemented supplemented = list(slot_specs) @@ -629,8 +633,8 @@ def _supplement_reference_slots(slot_specs: list[Any], target_count: int) -> lis col = quad % 2 row = quad // 2 new_spec = dict(base) - new_spec["id"] = f"{base['id']}_detail_{detail_index:02d}" - new_spec["paper_concept"] = f"{base.get('paper_concept', base['id'])} local detail {detail_index}" + new_spec["id"] = f"{base['id']}_reference_detail_{detail_index:02d}" + new_spec["paper_concept"] = str(base.get("paper_concept") or base["id"]) new_spec["composition_type"] = "scene_thumbnail" if base.get("composition_type") != "symbol_cutout" else "full_frame_icon" new_spec["bbox_percent"] = { "x": _round4(float(bbox["x"]) + float(bbox["w"]) * (0.04 + col * 0.50)), @@ -640,6 +644,8 @@ def _supplement_reference_slots(slot_specs: list[Any], target_count: int) -> lis } new_spec["display_label"] = "" new_spec["show_slot_caption"] = False + new_spec["reference_detail_index"] = detail_index + new_spec["slot_origin"] = "reference_visual_split" supplemented.append(new_spec) detail_index += 1 return supplemented @@ -1295,7 +1301,16 @@ def _generic_slot_specs(paper_brief: dict, slot_count: int) -> tuple[list[dict], "visual_metaphor": str(item.get("visual_metaphor") or "").strip(), "must_show": item.get("must_show") if isinstance(item.get("must_show"), list) else [], "avoid_showing": item.get("avoid_showing") if isinstance(item.get("avoid_showing"), list) else [], + "semantic_instance_id": item.get("semantic_instance_id"), + "semantic_id": item.get("semantic_id"), + "semantic_panel_id": item.get("semantic_panel_id"), + "semantic_role": item.get("semantic_role"), + "evidence_ids": item.get("evidence_ids") if isinstance(item.get("evidence_ids"), list) else [], + "contract_source": item.get("contract_source"), + "slot_origin": item.get("slot_origin") or "semantic_contract_node", }) + if specs and str(paper_brief.get("semantic_authority") or "") in {"panel_graphs", "global_graph"}: + return specs[:slot_count], panel_layout if len(specs) >= min(25, slot_count): return specs[:slot_count], panel_layout @@ -1504,6 +1519,13 @@ def analyze_reference( "avoid_showing": _spec_value(spec, "avoid_showing") if isinstance(_spec_value(spec, "avoid_showing"), list) else [], "display_label": str(_spec_value(spec, "display_label") or "").strip(), "show_slot_caption": bool(_spec_value(spec, "show_slot_caption")), + "semantic_instance_id": str(_spec_value(spec, "semantic_instance_id") or "").strip() or None, + "semantic_id": str(_spec_value(spec, "semantic_id") or "").strip() or None, + "semantic_panel_id": str(_spec_value(spec, "semantic_panel_id") or "").strip() or None, + "semantic_role": str(_spec_value(spec, "semantic_role") or "").strip() or None, + "semantic_evidence_ids": [str(value) for value in (_spec_value(spec, "evidence_ids") or []) if value], + "slot_origin": str(_spec_value(spec, "slot_origin") or "semantic_contract_node").strip(), + "reference_detail_index": _spec_value(spec, "reference_detail_index"), }) panel_geometry = [] diff --git a/rfs/workflow.py b/rfs/workflow.py index f1be057..4169a6f 100644 --- a/rfs/workflow.py +++ b/rfs/workflow.py @@ -1,6 +1,8 @@ from __future__ import annotations +import time from pathlib import Path +from typing import Any from .asset_generator import generate_assets from .asset_reviewer import review_assets @@ -11,6 +13,7 @@ from .paper_analyzer import analyze_paper from .presentations_qa import run_presentations_qa from .composition import compile_ppt +from .contracts import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract, load_paper_semantic_contract from .prompt_planner import plan_slot_prompts from .layout_locator import locate_layout from .program_builder import build_figure_program @@ -22,6 +25,21 @@ from .visual_critic import apply_layout_corrections, run_visual_critic +def _resolve_semantic_contract( + semantic_contract: dict[str, Any] | str | Path | None, + paper_contract_dir: str | Path | None, +) -> tuple[dict[str, Any] | None, str | None]: + if semantic_contract is not None and paper_contract_dir is not None: + raise ValueError("use either semantic_contract or paper_contract_dir, not both") + source = semantic_contract if semantic_contract is not None else paper_contract_dir + if source is None: + return None, None + if isinstance(source, dict): + return source, "inline" + path = Path(source) + return load_paper_semantic_contract(path), str(path.resolve()) + + def _write_alignment_review(out_dir: Path, paper_brief: dict, inventory: dict, layout_plan: dict, export_result: dict) -> None: text = "\n".join([ "# Summary", @@ -114,14 +132,35 @@ def make_framework( presentations_workspace: str | Path | None = None, presentations_scale: int = 2, export: bool = True, + semantic_contract: dict[str, Any] | str | Path | None = None, + paper_contract_dir: str | Path | None = None, ) -> dict: + started = time.perf_counter() + stage_started = started + stage_timings: dict[str, float] = {} + + def mark(stage: str) -> None: + nonlocal stage_started + now = time.perf_counter() + stage_timings[stage] = round(now - stage_started, 3) + stage_started = now + out_dir = ensure_dir(out) prompt_plan_workers = max(1, min(12, int(prompt_plan_workers))) + contract, contract_source = _resolve_semantic_contract(semantic_contract, paper_contract_dir) input_manifest = archive_inputs(paper, reference, out_dir) + input_manifest["semantic_contract_source"] = contract_source + input_manifest["paper_planner_bypassed"] = bool(contract) + write_json(out_dir / "input_manifest.json", input_manifest) archived_paper = input_manifest.get("paper_archived") or str(paper) archived_reference = input_manifest.get("reference_archived") or str(reference) - loaded = load_text(archived_paper) - paper_brief = analyze_paper(loaded, out_dir) + mark("archive_inputs") + if contract: + paper_brief = compile_paper_brief_from_semantic_contract(contract, out_dir, source_path=archived_paper) + else: + loaded = load_text(archived_paper) + paper_brief = analyze_paper(loaded, out_dir) + mark("paper_brief") inventory = analyze_reference( archived_reference, paper_brief, @@ -130,9 +169,14 @@ def make_framework( slot_source=slot_source, control_localizer_mode=control_localizer_mode, ) + mark("reference_analysis") style = build_style_sheet(paper_brief, inventory, out_dir) layout_plan = locate_layout(archived_reference, inventory, style, out_dir, mode=locator_mode, model=locator_model) + mark("style_and_layout") program = build_figure_program(paper_brief, inventory, style, out_dir, layout_plan=layout_plan) + semantic_binding_report = None + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) _slot_prompt_plan, program = plan_slot_prompts( archived_reference, @@ -147,6 +191,7 @@ def make_framework( complexity_profile=complexity_profile, ) program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) + mark("program_and_prompt_plan") program = build_text_layer( archived_reference, program, @@ -156,6 +201,10 @@ def make_framework( ocr_engine=ocr_engine, ocr_lang=ocr_lang, ) + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) + program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) + mark("editable_text_and_connectors") generate_assets( program, style, @@ -166,10 +215,12 @@ def make_framework( asset_retries=asset_retries, ) asset_review = review_assets(out_dir, mode=asset_review_mode, model=critic_model) + mark("assets_and_review") pptx = compile_ppt(program, out_dir) export_result = {"status": "skipped", "pdf": None, "png": None} if export: export_result = export_outputs(pptx, out_dir) + mark("composition_and_export") visual_critic = run_visual_critic( out_dir=out_dir, @@ -190,6 +241,8 @@ def make_framework( break write_json(out_dir / "layout_plan.json", layout_plan) program = build_figure_program(paper_brief, inventory, style, out_dir, layout_plan=layout_plan) + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) program = build_text_layer( archived_reference, @@ -200,6 +253,9 @@ def make_framework( ocr_engine=ocr_engine, ocr_lang=ocr_lang, ) + if contract: + program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_dir) + program = style_and_route_arrows(program, out_dir, mode=arrow_style_mode) pptx = compile_ppt(program, out_dir) if export: export_result = export_outputs(pptx, out_dir) @@ -213,6 +269,7 @@ def make_framework( model=critic_model, iteration=iteration, ) + mark("visual_critic") _write_alignment_review(out_dir, paper_brief, inventory, layout_plan, export_result) write_text(out_dir / "critic_report.md", "# Summary\nCritic review placeholder created before final validation.\n") validation_for_critic = validate_output(out_dir) @@ -227,6 +284,18 @@ def make_framework( scale=presentations_scale, run_inspect=True, ) + mark("validation_and_qa") + run_report = { + "summary": "Timed make-framework execution report.", + "status": "completed" if validation.get("ok") else "completed_with_validation_errors", + "paper_planner": paper_brief.get("planner"), + "semantic_contract_source": contract_source, + "semantic_contract_active": bool(contract), + "semantic_binding_status": semantic_binding_report.get("status") if semantic_binding_report else None, + "stage_timings_seconds": stage_timings, + "total_seconds": round(time.perf_counter() - started, 3), + } + write_json(out_dir / "run_report.json", run_report) return { "summary": "ResearchFigureStudio make-framework run result.", "ok": validation.get("ok", False), @@ -253,5 +322,9 @@ def make_framework( "presentations_qa": presentations_report, "slot_count": inventory.get("slot_count"), "slot_source": slot_source, + "paper_planner": paper_brief.get("planner"), + "semantic_contract_source": contract_source, + "semantic_binding": semantic_binding_report, + "run_report": run_report, "validation": validation, } diff --git a/rfs/workflows/paper_to_editable.py b/rfs/workflows/paper_to_editable.py index 556a54f..bed31f1 100644 --- a/rfs/workflows/paper_to_editable.py +++ b/rfs/workflows/paper_to_editable.py @@ -8,6 +8,7 @@ from ..paper_to_image import run_paper_to_image from ..rebuild_vlm_adapters import build_rebuild_vlm_adapters from ..utils import ensure_dir, write_json +from ..workflow import make_framework def run_paper_to_editable( @@ -39,6 +40,11 @@ def run_paper_to_editable( allow_engineering_preview: bool = False, ocr_engine: str = "auto", ocr_lang: str = "en_ch", + editable_route: str = "rebuild", + framework_slot_count: int = 25, + framework_asset_mode: str = "image2", + framework_candidates_per_slot: int = 1, + framework_asset_workers: int = 3, critic_adapter=None, ) -> dict[str, Any]: root = ensure_dir(out).resolve() @@ -85,6 +91,55 @@ def run_paper_to_editable( return blocked contract = load_paper_semantic_contract(image_dir) + if editable_route == "framework": + framework_ocr_engine = ocr_engine if ocr_engine in {"rapidocr", "paddle", "easyocr", "off"} else "off" + framework_result = make_framework( + paper=paper, + reference=reference, + out=editable_dir, + slot_count=framework_slot_count, + slot_source="reference-primary", + asset_mode=framework_asset_mode, + candidates_per_slot=framework_candidates_per_slot, + asset_workers=framework_asset_workers, + asset_retries=image_retries, + asset_review_mode="vlm" if review_mode == "vlm" else "heuristic", + locator_mode="vlm" if layout_mode in {"vlm", "hybrid"} else "heuristic", + control_localizer_mode="hybrid" if control_mode in {"vlm", "hybrid"} else "heuristic", + arrow_style_mode="reference", + prompt_plan_mode="vlm" if planner_mode == "vlm" else "heuristic", + prompt_plan_model=planner_model, + prompt_plan_workers=max(1, min(12, framework_asset_workers)), + complexity_profile="reference-dense", + critic_mode="vlm" if review_mode == "vlm" else "heuristic", + critic_model=review_model, + text_extractor_mode="ocr" if text_mode == "ocr" else "heuristic", + ocr_engine=framework_ocr_engine, + ocr_lang=ocr_lang, + export=export_preview, + semantic_contract=contract, + ) + result = { + "summary": "Paper-to-editable workflow completed through the authoritative multi-asset framework compiler.", + "ok": bool(framework_result.get("ok")), + "delivery_blocked": False, + "editable_route": "framework", + "out_dir": str(root), + "reference_status": reference_status, + "reference_image": str(reference), + "paper_to_image_dir": str(image_dir), + "editable_dir": str(editable_dir), + "pptx": framework_result.get("pptx"), + "preview": framework_result.get("png"), + "semantic_binding_status": (framework_result.get("semantic_binding") or {}).get("status"), + "paper_to_image": image_result, + "framework_compiler": framework_result, + } + write_json(root / "paper_to_editable_result.json", result) + return result + if editable_route != "rebuild": + raise ValueError(f"Unsupported editable_route: {editable_route}") + adapters = build_rebuild_vlm_adapters(editable_dir) rebuild_ocr_engine = ocr_engine if ocr_engine in {"paddle", "easyocr", "off"} else "paddle" editable_result = rebuild_editable( @@ -109,6 +164,7 @@ def run_paper_to_editable( "summary": "Paper-to-editable workflow completed with paper-grounded semantic bindings.", "ok": bool(editable_result.get("ok")), "delivery_blocked": False, + "editable_route": "rebuild", "out_dir": str(root), "reference_status": reference_status, "reference_image": str(reference), diff --git a/tests/unit/test_contract_framework_bridge.py b/tests/unit/test_contract_framework_bridge.py new file mode 100644 index 0000000..d0ebea8 --- /dev/null +++ b/tests/unit/test_contract_framework_bridge.py @@ -0,0 +1,99 @@ +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from rfs.contracts import apply_paper_semantic_contract, compile_paper_brief_from_semantic_contract +from rfs.reference_analyzer import analyze_reference + + +def _panel_contract() -> dict: + return { + "summary": "Test paper contract.", + "figure_specification": { + "summary": "Panel-local contract.", + "topology": "dense_multiframe", + "figure_goal": "Show training and architecture without merging repeated model instances.", + "panel_graphs": [ + { + "id": "panel_a", + "label": "Training and usage", + "nodes": [ + {"instance_id": "a_input", "name": "Dataset", "role": "input", "evidence_ids": ["E1"]}, + {"instance_id": "a_model", "name": "Model", "role": "module", "evidence_ids": ["E2"]}, + {"instance_id": "a_output", "name": "Prediction", "role": "output", "evidence_ids": ["E3"]}, + ], + "relations": [ + {"source": "a_input", "target": "a_model", "type": "data_flow", "evidence_ids": ["E4"]}, + {"source": "a_model", "target": "a_output", "type": "prediction", "evidence_ids": ["E5"]}, + ], + }, + { + "id": "panel_b", + "label": "Architecture", + "nodes": [ + {"instance_id": "b_group", "name": "Model block", "role": "group", "evidence_ids": ["E6"]}, + {"instance_id": "b_model", "name": "Model", "role": "module", "evidence_ids": ["E7"]}, + {"instance_id": "b_output", "name": "Output distribution", "role": "output", "evidence_ids": ["E8"]}, + ], + "relations": [ + {"source": "b_model", "target": "b_output", "type": "prediction", "evidence_ids": ["E9"]}, + ], + }, + ], + "relations": [ + {"source": "a_input", "target": "b_output", "type": "incorrect_global_compatibility_edge"} + ], + }, + } + + +class ContractFrameworkBridgeTests(unittest.TestCase): + def test_panel_graph_contract_drives_slots_and_relations_without_summary_fillers(self): + contract = _panel_contract() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = root / "reference.png" + Image.new("RGB", (1200, 800), "white").save(reference) + + brief = compile_paper_brief_from_semantic_contract(contract, root, source_path="paper.pdf") + inventory = analyze_reference( + reference, + brief, + root, + slot_count=25, + slot_source="reference-primary", + control_localizer_mode="off", + ) + + self.assertEqual(brief["planner"], "semantic_contract") + self.assertEqual(brief["semantic_authority"], "panel_graphs") + self.assertEqual(len(brief["slot_suggestions"]), 5) + self.assertNotIn("b_group", {item["id"] for item in brief["slot_suggestions"]}) + self.assertEqual(inventory["slot_count"], 25) + self.assertFalse(any("visual unit" in item["paper_concept"] for item in inventory["slots"])) + self.assertFalse(any("local detail" in item["paper_concept"] for item in inventory["slots"])) + self.assertTrue(all(item.get("semantic_instance_id") for item in inventory["slots"])) + self.assertTrue(any(item.get("slot_origin") == "reference_visual_split" for item in inventory["slots"])) + + program = { + "summary": "Test program.", + "slots": inventory["slots"], + "cards": [], + "panels": [], + "arrows": [], + "text_program": {"summary": "Empty text.", "items": []}, + } + updated, report = apply_paper_semantic_contract(program, contract, root) + + mappings = {item["entity_id"]: item["object_id"] for item in report["mappings"]} + self.assertEqual(report["contract_graph_source"], "panel_graphs") + self.assertEqual(report["mapped_relation_count"], 3) + self.assertNotEqual(mappings["a_model"], mappings["b_model"]) + self.assertEqual({item["semantic_panel_id"] for item in updated["arrows"]}, {"panel_a", "panel_b"}) + self.assertFalse(any(item["relation_type"] == "incorrect_global_compatibility_edge" for item in updated["arrows"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_paper_to_editable_route.py b/tests/unit/test_paper_to_editable_route.py new file mode 100644 index 0000000..f05c7f8 --- /dev/null +++ b/tests/unit/test_paper_to_editable_route.py @@ -0,0 +1,51 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from rfs.workflows.paper_to_editable import run_paper_to_editable + + +class PaperToEditableRouteTests(unittest.TestCase): + def test_framework_route_reuses_paper_to_image_contract_as_authority(self): + contract = {"figure_specification": {"panel_graphs": [{"id": "p", "label": "Panel", "nodes": []}]}} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + reference = root / "selected.png" + reference.write_bytes(b"not-read-by-mocked-compiler") + image_result = {"ok": True, "selected_image": str(reference)} + framework_result = { + "ok": True, + "pptx": str(root / "editable" / "editable_composition.pptx"), + "png": None, + "semantic_binding": {"status": "pass"}, + } + + with ( + patch("rfs.workflows.paper_to_editable.run_paper_to_image", return_value=image_result), + patch("rfs.workflows.paper_to_editable.load_paper_semantic_contract", return_value=contract), + patch("rfs.workflows.paper_to_editable.make_framework", return_value=framework_result) as compiler, + ): + result = run_paper_to_editable( + paper=root / "paper.pdf", + out=root / "run", + editable_route="framework", + framework_asset_mode="placeholder", + framework_slot_count=25, + framework_candidates_per_slot=1, + framework_asset_workers=3, + export_preview=False, + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["editable_route"], "framework") + self.assertEqual(result["semantic_binding_status"], "pass") + kwargs = compiler.call_args.kwargs + self.assertIs(kwargs["semantic_contract"], contract) + self.assertEqual(kwargs["slot_count"], 25) + self.assertEqual(kwargs["asset_mode"], "placeholder") + self.assertEqual(kwargs["slot_source"], "reference-primary") + + +if __name__ == "__main__": + unittest.main() From 7fb9d2339a5c29b966c46bab80014b9438222d4f Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 06:46:49 +0800 Subject: [PATCH 70/74] Add evidence-grounded iterative image review --- rfs/cli.py | 4 +- rfs/paper_to_image/critics.py | 90 ++++++- rfs/paper_to_image/generator.py | 169 +++++++++++-- rfs/paper_to_image/preparation.py | 20 +- rfs/paper_to_image/review_contract.py | 306 +++++++++++++++++++++++ rfs/paper_to_image/workflow.py | 9 +- rfs/workflows/paper_to_editable.py | 2 +- tests/integration/test_paper_to_image.py | 134 +++++++++- tests/unit/test_review_contract.py | 70 ++++++ 9 files changed, 772 insertions(+), 32 deletions(-) create mode 100644 rfs/paper_to_image/review_contract.py create mode 100644 tests/unit/test_review_contract.py diff --git a/rfs/cli.py b/rfs/cli.py index ba5379d..80ae71a 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -220,7 +220,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--image-retries", type=int, default=1, help="Retries per image candidate for retryable HTTP/provider errors, clamped to 0-5. Network timeouts are not blindly retried. Default: 1.") paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - paper_image.add_argument("--repair-rounds", type=int, default=1, help="Localized Image2 edit repair rounds, clamped to 0-1. Default: 1.") + paper_image.add_argument("--repair-rounds", type=int, default=3, help="Localized Image2 edit repair rounds, clamped to 0-4 with no-regression acceptance. Default: 3.") paper_image.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") paper_image.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") @@ -245,7 +245,7 @@ def build_parser() -> argparse.ArgumentParser: paper_editable.add_argument("--review-model") paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") paper_editable.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") - paper_editable.add_argument("--repair-rounds", type=int, default=1) + paper_editable.add_argument("--repair-rounds", type=int, default=3) paper_editable.add_argument("--editable-route", choices=["rebuild", "framework"], default="rebuild", help="Editable compiler. Use framework for the authoritative 25-50 slot paper-contract route; rebuild preserves the legacy image reconstruction route.") paper_editable.add_argument("--framework-slot-count", type=int, default=25, help="Non-arrow slot count for --editable-route framework. Default: 25.") paper_editable.add_argument("--framework-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset backend for --editable-route framework.") diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 51826ba..6847e20 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -51,6 +51,7 @@ def _local_ocr(path: Path, engine: str, lang: str, adapter: Callable | None = No def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: list[str], repeatable_labels: list[str], optional_visible_labels: list[str], forbidden_labels: list[str], model: str | None, adapter: Callable | None = None) -> dict: + review_ground_truth = plan.get("review_ground_truth") if isinstance(plan.get("review_ground_truth"), dict) else {} prompt = f""" # Summary @@ -71,6 +72,9 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: Scientific specification: {json.dumps(plan.get('figure_specification', {}), ensure_ascii=False, indent=2)} +Paper-evidence review ground truth. Treat the quoted source excerpts and page numbers as higher authority than visual plausibility. Evaluate every required priority by ID. Do not infer a common architecture when the evidence does not state it: +{json.dumps(review_ground_truth, ensure_ascii=False, indent=2)} + Mandatory visual connector checklist: {json.dumps(collect_visual_relations(plan.get('figure_specification', {})), ensure_ascii=False, indent=2)} @@ -89,6 +93,7 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: "summary": "Production candidate review.", "ocr": {{"detected_labels": [], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 0.0, "passed": false}}, "scientific": {{"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": true, "score": 0.0, "passed": false}}, + "paper_alignment": {{"priorities": [{{"id": "...", "status": "supported|missing|conflicting|uncertain", "visible_evidence": "...", "evidence_ids": []}}], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": [], "repair": [], "score": 0.0, "passed": false}}, "template": {{"macro_panel_match": 0.0, "reading_order_match": 0.0, "connector_rhythm_match": 0.0, "visual_density_match": 0.0, "copied_reference_content": [], "score": 0.0, "passed": false}}, "aesthetic": {{"hierarchy": 0.0, "balance": 0.0, "whitespace": 0.0, "color": 0.0, "icon_consistency": 0.0, "readability": 0.0, "visual_information_density": 0.0, "mechanism_visualization": 0.0, "publication_polish": 0.0, "score": 0.0, "passed": false}}, "preserve": [], @@ -100,7 +105,14 @@ def _vlm_critic(path: Path, blueprint: Path, plan: dict, template: dict, labels: "overall_score": 0.0 }} -Hard failures: any missing/misspelled critical label, any duplicate label not explicitly allowed above, copied reference term, missing core module, reversed relation, invented mechanism, semantic role/containment mismatch, or template score below 0.72. Aesthetic quality cannot compensate for scientific or OCR failure. +Hard failures: any missing/misspelled critical label, any duplicate label not explicitly allowed above, copied reference term, missing paper priority, conflict with a quoted paper passage, unsupported visual claim, missing core module, reversed relation, invented mechanism, semantic role/containment mismatch, or template score below 0.72. Aesthetic quality cannot compensate for scientific, evidence-alignment, topology, or OCR failure. + +Paper-alignment rules: +- Return exactly one priorities record for every required priority ID in review_ground_truth. +- A priority is supported only when the visible image agrees with both its label/relationship and the quoted evidence. +- Put visually asserted content with no support in the specification/evidence into unsupported_visual_claims. +- Put any contradiction with a quoted passage into evidence_conflicts and cite evidence_ids. +- Unknown items must remain unknown; resolving them through a plausible-looking visual is unsupported invention. Aesthetic scoring rules: - A neat text-only flowchart is not a complete Image-2 scientific figure. Empty rounded rectangles containing only labels must score below 0.70 for mechanism_visualization and visual_information_density. @@ -135,6 +147,64 @@ def _normalize_section(raw: Any, name: str) -> dict: return section +def _normalize_paper_alignment(raw: Any, ground_truth: dict, scientific: dict, adapter_used: bool) -> dict: + required = [item for item in ground_truth.get("priorities", []) if isinstance(item, dict) and item.get("required")] + strict = bool(ground_truth.get("strict_evidence_required")) + if not isinstance(raw, dict): + if adapter_used or not strict: + return { + "summary": "Paper alignment inherited from a trusted adapter or legacy contract without source excerpts.", + "priorities": [], + "missing_priorities": [], + "evidence_conflicts": [], + "unsupported_visual_claims": [], + "score": float(scientific.get("score") or 0.0), + "passed": bool(scientific.get("passed")), + "legacy_fallback": True, + } + return { + "summary": "Paper-evidence alignment was not returned by the production critic.", + "priorities": [], + "missing_priorities": [ + {"id": item.get("id"), "label": item.get("label"), "evidence_ids": item.get("evidence_ids", []), "reason": "not evaluated by critic"} + for item in required + ], + "evidence_conflicts": [], + "unsupported_visual_claims": [], + "score": 0.0, + "passed": False, + } + alignment = _normalize_section(raw, "Paper alignment") + for field in ("priorities", "missing_priorities", "evidence_conflicts", "unsupported_visual_claims", "repair"): + if not isinstance(alignment.get(field), list): + alignment[field] = [] + evaluated = { + str(item.get("id") or "").strip(): str(item.get("status") or "").strip().casefold() + for item in alignment["priorities"] if isinstance(item, dict) and str(item.get("id") or "").strip() + } + missing_ids = { + str(item.get("id") or "").strip() if isinstance(item, dict) else str(item).strip() + for item in alignment["missing_priorities"] + } + if strict: + for item in required: + item_id = str(item.get("id") or "").strip() + status = evaluated.get(item_id) + if status not in {"supported", "present", "correct"} and item_id not in missing_ids: + alignment["missing_priorities"].append({ + "id": item_id, + "label": item.get("label"), + "evidence_ids": item.get("evidence_ids", []), + "reason": "required priority was not explicitly verified as supported", + }) + missing_ids.add(item_id) + issue_count = sum(len(alignment[field]) for field in ("missing_priorities", "evidence_conflicts", "unsupported_visual_claims")) + if strict and required and not issue_count and all(evaluated.get(str(item.get("id") or "").strip()) in {"supported", "present", "correct"} for item in required): + alignment["score"] = 1.0 + alignment["passed"] = bool(alignment.get("passed")) and alignment["score"] >= 0.95 and issue_count == 0 + return alignment + + def _vlm_topology_critic(path: Path, plan: dict, model: str | None, adapter: Callable | None = None) -> dict: specification = plan.get("figure_specification", {}) relations = collect_visual_relations(specification) @@ -395,6 +465,8 @@ def review_candidate( if scientific_issues == 0 and innovation_ok: scientific["score"] = 1.0 scientific["passed"] = scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok + ground_truth = plan.get("review_ground_truth") if isinstance(plan.get("review_ground_truth"), dict) else {} + paper_alignment = _normalize_paper_alignment(vlm_raw.get("paper_alignment"), ground_truth, scientific, critic_adapter is not None) copied = template_review.get("copied_reference_content", []) template_review["passed"] = template_review["score"] >= 0.70 and not copied aesthetic_floor = min(aesthetic["visual_information_density"], aesthetic["mechanism_visualization"], aesthetic["publication_polish"]) @@ -403,6 +475,7 @@ def review_candidate( template_review.update({"score": 1.0 if basic["passed"] else 0.0, "passed": basic["passed"], "engineering_only": True}) aesthetic.update({"score": 0.0, "passed": False, "engineering_only": True}) scientific.update({"score": 0.0, "passed": False, "engineering_only": True}) + paper_alignment.update({"score": 0.0, "passed": False, "engineering_only": True}) ocr.update({"score": 0.0, "passed": False, "engineering_only": True}) hard_errors = [] for value in vlm_raw.get("hard_errors", []): @@ -419,8 +492,15 @@ def review_candidate( hard_errors.extend(_issue_text(value) for value in topology_review.get("invented_relations", [])) if topology_warning: hard_errors.append(f"Focused topology review unavailable: {topology_warning}") + if not paper_alignment["passed"]: + hard_errors.extend(_issue_text(value) for value in paper_alignment.get("missing_priorities", []) if _issue_text(value)) + hard_errors.extend(_issue_text(value) for value in paper_alignment.get("evidence_conflicts", []) if _issue_text(value)) + hard_errors.extend(_issue_text(value) for value in paper_alignment.get("unsupported_visual_claims", []) if _issue_text(value)) + if ground_truth.get("strict_evidence_required") and not isinstance(vlm_raw.get("paper_alignment"), dict) and critic_adapter is None: + hard_errors.append("Production critic did not return paper-evidence alignment") hard_errors = list(dict.fromkeys(value for value in hard_errors if value)) repair = list(vlm_raw.get("repair", [])) + repair.extend(_issue_text(value) for value in paper_alignment.get("repair", []) if _issue_text(value)) repair.extend(_issue_text(value) for value in topology_review.get("repair", []) if _issue_text(value)) repair_regions = list(vlm_raw.get("repair_regions", [])) repair_regions.extend(_issue_text(value) for value in topology_review.get("repair_regions", []) if _issue_text(value)) @@ -432,15 +512,16 @@ def review_candidate( repair.append("Enrich node interiors with paper-grounded visual examples, mechanism cues, and consistent scientific icons while preserving all node boxes and connectors.") hard_errors.append(f"Image-2 visual enrichment ratio {blueprint_enrichment_ratio:.3f} is below required {minimum_enrichment_ratio:.3f}; candidate is too close to the bare layout blueprint") hard_errors = list(dict.fromkeys(hard_errors)) - production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and topology_review["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) - score = 0.25 * scientific["score"] + 0.20 * ocr["score"] + 0.20 * topology_review["score"] + 0.20 * template_review["score"] + 0.15 * aesthetic["score"] + production_pass = bool(basic["passed"] and ocr["passed"] and scientific["passed"] and paper_alignment["passed"] and topology_review["passed"] and template_review["passed"] and aesthetic["passed"] and not hard_errors) + score = 0.22 * scientific["score"] + 0.18 * paper_alignment["score"] + 0.17 * ocr["score"] + 0.17 * topology_review["score"] + 0.14 * template_review["score"] + 0.12 * aesthetic["score"] return { - "summary": "Four-layer production candidate review.", + "summary": "Evidence-grounded production candidate review.", "path": str(candidate), "mode": mode, "basic": basic, "ocr": ocr, "scientific": scientific, + "paper_alignment": paper_alignment, "topology": topology_review, "template": template_review, "aesthetic": aesthetic, @@ -463,6 +544,7 @@ def collect(section: str) -> dict: return { "ocr_review": collect("ocr"), "scientific_critic_report": collect("scientific"), + "paper_alignment_report": collect("paper_alignment"), "topology_critic_report": collect("topology"), "template_alignment_report": collect("template"), "aesthetic_critic_report": collect("aesthetic"), diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index 2acadc0..64c3a01 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -18,6 +18,7 @@ from ..utils import ensure_dir, write_json, write_text from .critics import aggregate_reviews, review_candidate from .planner import collect_visual_relations, compile_image_prompt +from .review_contract import build_repair_plan def _parse_ratio(value: str) -> float: @@ -108,6 +109,7 @@ def _candidate_failure_modes(record: dict) -> list[str]: modes = [] ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + paper_alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} @@ -125,6 +127,12 @@ def _candidate_failure_modes(record: dict) -> list[str]: modes.append("invented_content") if scientific.get("role_mismatches") or scientific.get("containment_mismatches"): modes.append("semantic_role_mismatch") + if paper_alignment.get("missing_priorities"): + modes.append("paper_missing_priorities") + if paper_alignment.get("evidence_conflicts"): + modes.append("paper_evidence_conflicts") + if paper_alignment.get("unsupported_visual_claims"): + modes.append("unsupported_visual_claims") if not template.get("passed", False): modes.append("template_alignment") if not aesthetic.get("passed", False): @@ -241,7 +249,7 @@ def _generate_one(prompt: str, plan: dict, blueprint: Path, path: Path, aspect_r return _call_image2_edit(blueprint, prompt, path, aspect_ratio, model, retries) -def _repair_prompt(review: dict, base_prompt: str, plan: dict) -> str: +def _repair_prompt(review: dict, repair_plan: dict, base_prompt: str, plan: dict) -> str: connector_checklist = "\n".join( f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" for index, item in enumerate(collect_visual_relations(plan.get("figure_specification", {})), start=1) @@ -255,7 +263,7 @@ def _repair_prompt(review: dict, base_prompt: str, plan: dict) -> str: {json.dumps(review.get('preserve', []), ensure_ascii=False)} Repair: -{json.dumps(review.get('repair', []), ensure_ascii=False)} +{json.dumps(repair_plan.get('items', []), ensure_ascii=False, indent=2)} Remove: {json.dumps(review.get('remove', []), ensure_ascii=False)} @@ -276,6 +284,39 @@ def _repair_prompt(review: dict, base_prompt: str, plan: dict) -> str: """.strip() +def _review_regressions(source: dict, candidate: dict, tolerance: float = 0.001) -> list[str]: + regressions = [] + for section in ("basic", "ocr", "scientific", "paper_alignment", "topology", "template", "aesthetic"): + previous = source.get(section, {}) if isinstance(source.get(section), dict) else {} + current = candidate.get(section, {}) if isinstance(candidate.get(section), dict) else {} + if previous.get("passed") and not current.get("passed"): + regressions.append(f"{section} changed from pass to fail") + if section in {"ocr", "scientific", "paper_alignment", "topology"}: + previous_score = float(previous.get("score") or 0.0) + current_score = float(current.get("score") or 0.0) + if current_score + tolerance < previous_score: + regressions.append(f"{section} score decreased from {previous_score:.4f} to {current_score:.4f}") + if len(candidate.get("hard_errors", [])) > len(source.get("hard_errors", [])): + regressions.append("hard error count increased") + return regressions + + +def _strictly_improves(source: dict, candidate: dict) -> tuple[bool, list[str]]: + regressions = _review_regressions(source, candidate) + if regressions: + return False, regressions + source_score = float(source.get("overall_score") or 0.0) + candidate_score = float(candidate.get("overall_score") or 0.0) + source_hard = len(source.get("hard_errors", [])) + candidate_hard = len(candidate.get("hard_errors", [])) + improved = bool( + candidate.get("production_pass") and not source.get("production_pass") + or candidate_score > source_score + 0.001 + or candidate_hard < source_hard + ) + return improved, [] if improved else ["candidate did not strictly improve score, pass status, or hard-error count"] + + def generate_and_select( plan: dict, preferences: dict, @@ -288,7 +329,7 @@ def generate_and_select( image_retries: int = 2, review_mode: str = "vlm", review_model: str | None = None, - repair_rounds: int = 1, + repair_rounds: int = 3, repair_source: str | Path | None = None, ocr_engine: str = "auto", ocr_lang: str = "en_ch", @@ -301,6 +342,7 @@ def generate_and_select( root = ensure_dir(out_dir) candidate_dir = ensure_dir(root / "candidates") prompt_dir = ensure_dir(root / "prompts") + review_dir = ensure_dir(root / "reviews") blueprint = Path(blueprint_path) aspect_ratio = str(preferences.get("aspect_ratio") or "16:9") count = max(1, min(4, int(candidates))) @@ -434,32 +476,119 @@ def generate_candidate(index: int) -> dict[str, Any]: passing = [item for item in records if item["production_pass"]] repair_records = [] - for repair_index in range(max(0, min(1, int(repair_rounds)))): - if passing or asset_mode != "image2": + repair_history = [] + no_improvement_streak = 0 + repair_stop_reason = "not_requested" if int(repair_rounds) <= 0 else "pending" + max_repair_rounds = max(0, min(4, int(repair_rounds))) + for repair_index in range(max_repair_rounds): + if passing: + repair_stop_reason = "all_gates_passed" break - source = max(records, key=lambda item: item["score"]) + if asset_mode != "image2": + repair_stop_reason = "image2_required" + break + accepted_records = [item for item in records if item.get("accepted", True)] + source = max(accepted_records, key=lambda item: item["score"]) repaired_id = f"repair_{repair_index + 1:02d}" repaired_path = candidate_dir / f"{repaired_id}.png" - base_prompt = Path(source["prompt_path"]).read_text(encoding="utf-8") - prompt = _repair_prompt(source["review"], base_prompt, plan) prompt_path = prompt_dir / f"{repaired_id}_prompt.txt" + repair_plan_path = prompt_dir / f"{repaired_id}_plan.json" + review_path = review_dir / f"{repaired_id}_review.json" + repair_plan = build_repair_plan(source["review"], source["candidate_id"], repair_index + 1) + write_json(repair_plan_path, repair_plan) + write_json(root / "repair_plan.json", repair_plan) + if not repair_plan["items"]: + repair_stop_reason = "no_actionable_repairs" + break + base_prompt = Path(source["prompt_path"]).read_text(encoding="utf-8") + prompt = _repair_prompt(source["review"], repair_plan, base_prompt, plan) write_text(prompt_path, prompt) repair_started = time.monotonic() try: - generation_started = time.monotonic() - generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) - generation_seconds = round(time.monotonic() - generation_started, 3) + if resume_candidates and repaired_path.exists() and review_path.exists(): + generation = {"mode": "existing_repair_resume", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + generation_seconds = 0.0 + review = json.loads(review_path.read_text(encoding="utf-8")) + review_seconds = 0.0 + else: + generation_started = time.monotonic() + generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) + generation_seconds = round(time.monotonic() - generation_started, 3) + review_started = time.monotonic() + review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) + review_seconds = round(time.monotonic() - review_started, 3) + write_json(review_path, review) requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) - review_started = time.monotonic() - review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) - review_seconds = round(time.monotonic() - review_started, 3) - repaired = {"candidate_id": repaired_id, "path": str(repaired_path), "prompt_path": str(prompt_path), "generation": generation, "review": review, "production_pass": review["production_pass"], "score": review["overall_score"], "repair_source": source["candidate_id"], "timings": {"generation_seconds": generation_seconds, "review_seconds": review_seconds, "total_seconds": round(time.monotonic() - repair_started, 3)}} + accepted, rejection_reasons = _strictly_improves(source["review"], review) + repaired = { + "candidate_id": repaired_id, + "path": str(repaired_path), + "prompt_path": str(prompt_path), + "repair_plan_path": str(repair_plan_path), + "review_path": str(review_path), + "generation": generation, + "review": review, + "production_pass": bool(review["production_pass"] and accepted), + "raw_production_pass": bool(review["production_pass"]), + "score": review["overall_score"], + "repair_source": source["candidate_id"], + "accepted": accepted, + "rejection_reasons": rejection_reasons, + "timings": {"generation_seconds": generation_seconds, "review_seconds": review_seconds, "total_seconds": round(time.monotonic() - repair_started, 3)}, + } repair_records.append(repaired) records.append(repaired) - if repaired["production_pass"]: - passing.append(repaired) + repair_history.append({ + "round": repair_index + 1, + "candidate_id": repaired_id, + "source_candidate_id": source["candidate_id"], + "source_score": source["score"], + "candidate_score": repaired["score"], + "accepted": accepted, + "production_pass": repaired["production_pass"], + "rejection_reasons": rejection_reasons, + }) + if accepted: + no_improvement_streak = 0 + if repaired["production_pass"]: + passing.append(repaired) + repair_stop_reason = "all_gates_passed" + break + else: + no_improvement_streak += 1 + if no_improvement_streak >= 2: + repair_stop_reason = "two_consecutive_rounds_without_improvement" + break except Exception as exc: - failures.append({"candidate_id": repaired_id, "error": str(exc), "prompt_path": str(prompt_path)}) + no_improvement_streak += 1 + failure = {"candidate_id": repaired_id, "error": str(exc), "prompt_path": str(prompt_path), "repair_source": source["candidate_id"]} + failures.append(failure) + repair_history.append({"round": repair_index + 1, "candidate_id": repaired_id, "source_candidate_id": source["candidate_id"], "accepted": False, "error": str(exc)}) + if no_improvement_streak >= 2: + repair_stop_reason = "two_consecutive_rounds_without_improvement" + break + else: + if max_repair_rounds: + repair_stop_reason = "maximum_repair_rounds_reached" + if repair_stop_reason == "pending": + repair_stop_reason = "maximum_repair_rounds_reached" if repair_records else "not_needed" + write_json(root / "repair_history.json", { + "summary": "Accepted/rejected multi-round Image2 repairs with no-regression checks.", + "requested_rounds": int(repair_rounds), + "maximum_rounds": max_repair_rounds, + "completed_rounds": len(repair_history), + "stop_reason": repair_stop_reason, + "rounds": repair_history, + }) + if not (root / "repair_plan.json").exists(): + write_json(root / "repair_plan.json", { + "summary": "No repair plan was required for the selected candidate set.", + "source_candidate_id": None, + "round_index": None, + "item_count": 0, + "preserve": [], + "items": [], + }) aggregates = aggregate_reviews(records) for name, data in aggregates.items(): @@ -481,8 +610,8 @@ def generate_candidate(index: int) -> dict[str, Any]: stability = _build_stability_report(records, failures, count, repair_source_path) write_json(root / "stability_report.json", stability) - report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "resume_candidates": bool(resume_candidates), "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "failures": failures, "candidates": records, "stability": stability, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} + report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "resume_candidates": bool(resume_candidates), "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "repair_stop_reason": repair_stop_reason, "failures": failures, "candidates": records, "stability": stability, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} write_json(root / "candidate_review.json", report) if report["delivery_blocked"]: - raise RuntimeError("No Image2 candidate passed scientific, OCR, template, and aesthetic production gates") + raise RuntimeError("No Image2 candidate passed paper alignment, scientific, OCR, topology, template, and aesthetic production gates") return report diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index d99f490..2179f9c 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -15,6 +15,7 @@ from .document_cache import DOCUMENT_CACHE_VERSION, read_document_cache, write_document_cache from .planner import build_review_grounded_plan, compile_image_prompt, merge_preferences, plan_fast_paper_contract, plan_paper_image, validate_plan_grounding from .review import build_paper_review, detect_domain_profile, validate_review_coverage +from .review_contract import compile_review_ground_truth def _load_preferences(path: str | Path | None) -> dict[str, Any]: @@ -3062,10 +3063,25 @@ def prepare_paper_figure_contract( write_text(root / "image_prompt.md", final_prompt) write_text(root / "image_prompt.txt", final_prompt) referenced_ids = _collect_evidence_ids({"paper_review": paper_review, "figure_specification": plan["figure_specification"]}) + provisional_ground_truth = compile_review_ground_truth(plan, parsed["evidence"]) + review_evidence_ids = { + str(evidence_id) + for priority in provisional_ground_truth.get("priorities", []) + for evidence_id in priority.get("evidence_ids", []) + if str(evidence_id).strip() + } + referenced_ids.update(review_evidence_ids) + key_evidence = [ + item for item in parsed["evidence"] + if str(item.get("id") or "") in referenced_ids or str(item.get("legacy_id") or "") in referenced_ids + ] write_json(root / "key_evidence.json", { "summary": "Evidence records referenced by the paper review and figure contract.", - "evidence": [item for item in parsed["evidence"] if item["id"] in referenced_ids], + "evidence": key_evidence, }) + review_ground_truth = compile_review_ground_truth(plan, key_evidence) + plan["review_ground_truth"] = review_ground_truth + write_json(root / "review_ground_truth.json", review_ground_truth) elapsed = round(time.monotonic() - started, 3) deadline_reached = time.monotonic() >= deadline_at @@ -3161,7 +3177,7 @@ def prepare_paper_figure_contract( "total_seconds": elapsed, }, "uncertainties": plan["figure_specification"].get("uncertainties", []), - "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "paper_review.json", "figure_specification.json", "contract_completion_report.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], + "artifacts": ["paper.md", "document_model.json", "extraction_report.json", "section_index.json", "section_summary.md", "key_evidence.json", "review_ground_truth.json", "paper_review.json", "figure_specification.json", "contract_completion_report.json", "planning_validation_report.json", "image_prompt.md", "overlay_spec.json", "run_report.json"], } write_json(root / "run_report.json", result) return { diff --git a/rfs/paper_to_image/review_contract.py b/rfs/paper_to_image/review_contract.py new file mode 100644 index 0000000..d6a8057 --- /dev/null +++ b/rfs/paper_to_image/review_contract.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import re +from typing import Any + + +def _text(value: Any) -> str: + if isinstance(value, dict): + for key in ("text", "statement", "name", "label", "title", "description", "claim"): + text = str(value.get(key) or "").strip() + if text: + return text + return "" + return str(value or "").strip() + + +def _list(value: Any) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.casefold()).strip("_") + return slug[:48] or "item" + + +def _normalized(value: str) -> str: + return re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "", value.casefold()) + + +def _excerpt(value: Any, limit: int) -> str: + text = re.sub(r"\s+", " ", str(value or "")).strip() + return text if len(text) <= limit else f"{text[: limit - 1].rstrip()}…" + + +def _tokens(value: Any) -> set[str]: + stop = {"the", "and", "for", "with", "from", "into", "that", "this", "are", "was", "were", "has", "have", "via", "only", "shown", "show", "high", "level"} + return {token for token in re.findall(r"[a-z0-9\u4e00-\u9fff]+", str(value or "").casefold()) if len(token) >= 3 and token not in stop} + + +def compile_review_ground_truth(plan: dict, evidence_records: list[dict] | None = None) -> dict: + """Compile a compact, evidence-bearing contract used only for image review. + + The figure specification remains the semantic source of truth. This view + adds the original evidence excerpts needed by a visual critic to check the + candidate against the paper instead of judging only a summarized plan. + """ + + specification = plan.get("figure_specification") if isinstance(plan.get("figure_specification"), dict) else {} + evidence_records = [item for item in (evidence_records or []) if isinstance(item, dict)] + evidence_index: dict[str, dict] = {} + for item in evidence_records: + for key in (item.get("id"), item.get("legacy_id")): + if key: + evidence_index[str(key)] = item + searchable_evidence = [ + (item, _normalized(str(item.get("text") or ""))) + for item in evidence_records + if _normalized(str(item.get("text") or "")) + ] + + entity_labels: dict[str, str] = {} + for field in ("inputs", "modules", "outputs", "innovations"): + for item in _list(specification.get(field)): + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "").strip() + label = _text(item) + if item_id and label: + entity_labels[item_id] = label + + priorities: list[dict] = [] + seen: set[tuple[str, str]] = set() + + def add_priority(kind: str, value: Any, *, importance: str = "required", visual_expectation: str | None = None) -> None: + source_id = "" + target_id = "" + if isinstance(value, dict): + label = _text(value) + evidence_ids = [str(item) for item in _list(value.get("evidence_ids")) if str(item).strip()] + source_id = str(value.get("source") or "").strip() + target_id = str(value.get("target") or "").strip() + if kind in {"relation", "feedback_loop", "training_flow", "inference_flow"} and (source_id or target_id): + source = entity_labels.get(source_id, source_id) + target = entity_labels.get(target_id, target_id) + relation_type = str(value.get("type") or value.get("relation") or "directed relation").strip() + label = f"{source} -> {target} [{relation_type}]" + item_id = str(value.get("id") or "").strip() + status = str(value.get("status") or importance).strip().casefold() + required = status not in {"optional", "unknown", "uncertain"} + else: + label = _text(value) + evidence_ids = [] + item_id = "" + required = importance != "optional" + if not label: + return + semantic_kind = "relation" if kind in {"relation", "feedback_loop"} else "flow" if kind in {"training_flow", "inference_flow"} and (source_id or target_id) else "concept" + key = (semantic_kind, _normalized(label)) + if key in seen: + return + seen.add(key) + excerpts = [] + for evidence_id in evidence_ids: + record = evidence_index.get(evidence_id) + if not record: + continue + excerpts.append({ + "evidence_id": str(record.get("id") or evidence_id), + "page": record.get("page"), + "section": record.get("section_hint") or record.get("section"), + "text": _excerpt(record.get("text"), 240), + "source": record.get("source"), + "confidence": record.get("confidence"), + }) + link_method = "explicit" if excerpts else "none" + if not excerpts: + needle = _normalized(label) + if len(needle) >= 3: + matches = [] + needle_tokens = _tokens(label) + for item, haystack in searchable_evidence: + reverse_fragment = len(haystack) >= 8 and len(haystack) / len(needle) >= 0.45 and haystack in needle + evidence_tokens = _tokens(item.get("text")) + token_coverage = len(needle_tokens & evidence_tokens) / max(1, len(needle_tokens)) + if needle == haystack: + rank = 0 + elif needle in haystack: + rank = 1 + elif reverse_fragment: + rank = 2 + elif len(needle_tokens) >= 3 and token_coverage >= 0.8: + rank = 3 + else: + continue + matches.append((rank, len(str(item.get("text") or "")), int(item.get("page") or 0), item)) + matches.sort(key=lambda match: match[:3]) + for _, _, _, record in matches[:2]: + record_id = str(record.get("id") or record.get("legacy_id") or "").strip() + if record_id and record_id not in evidence_ids: + evidence_ids.append(record_id) + excerpts.append({ + "evidence_id": record_id, + "page": record.get("page"), + "section": record.get("section_hint") or record.get("section"), + "text": _excerpt(record.get("text"), 240), + "source": record.get("source"), + "confidence": record.get("confidence"), + }) + if excerpts: + link_method = "deterministic_text_match" + priorities.append({ + "id": item_id or f"GT_{kind.upper()}_{_slug(label)}", + "kind": kind, + "label": label, + "importance": "critical" if importance == "critical" else "required" if required else "optional", + "required": required, + "visual_expectation": visual_expectation or ("visible and scientifically correct" if required else "may be visible when useful"), + "evidence_ids": evidence_ids, + "evidence": excerpts, + "evidence_link_method": link_method, + }) + + add_priority("research_problem", specification.get("research_problem"), importance="critical", visual_expectation="the problem context is visually recognizable") + add_priority("central_claim", specification.get("central_claim"), importance="critical", visual_expectation="the figure's main visual story supports this claim") + for field, kind, expectation in ( + ("inputs", "input", "shown with the correct semantic role as an input or source"), + ("modules", "module", "shown as the named paper-specific component"), + ("relations", "relation", "shown with the correct source, target, and direction"), + ("feedback_loops", "feedback_loop", "shown as a closed and correctly directed loop"), + ("outputs", "output", "shown as an output rather than an internal module"), + ("training_flow", "training_flow", "shown only in the training path"), + ("inference_flow", "inference_flow", "shown only in the inference path"), + ("innovations", "innovation", "the paper's novelty is visually identifiable"), + ("must_show", "must_show", "explicitly visible in the figure"), + ): + for item in _list(specification.get(field)): + add_priority(kind, item, importance="critical" if kind in {"module", "relation", "feedback_loop"} else "required", visual_expectation=expectation) + + for label in _list(specification.get("required_labels")): + add_priority("required_label", label, importance="required", visual_expectation="appears as exact visible text") + for label in _list(specification.get("optional_visible_labels")): + add_priority("optional_label", label, importance="optional", visual_expectation="allowed but not required") + + forbidden = [] + for index, item in enumerate(_list(specification.get("forbidden_inventions")), start=1): + label = _text(item) + if label: + forbidden.append({"id": f"FORBIDDEN_{index:03d}", "label": label, "rule": "must not be asserted or visually implied"}) + + unknown = [] + for index, item in enumerate(_list(specification.get("uncertainties")), start=1): + label = _text(item) + if label: + unknown.append({"id": f"UNKNOWN_{index:03d}", "label": label, "rule": "do not resolve by invention"}) + + evidence_catalog: dict[str, dict] = {} + for priority in priorities: + for item in priority.get("evidence", []): + evidence_id = str(item.get("evidence_id") or "").strip() + record = evidence_index.get(evidence_id) + if evidence_id and record and evidence_id not in evidence_catalog: + evidence_catalog[evidence_id] = { + "evidence_id": evidence_id, + "page": record.get("page"), + "section": record.get("section_hint") or record.get("section"), + "text": _excerpt(record.get("text"), 900), + "source": record.get("source"), + "confidence": record.get("confidence"), + } + required_priorities = [item for item in priorities if item["required"]] + grounded_required = [item for item in required_priorities if item["evidence"]] + return { + "summary": "Paper-evidence ground truth for visual candidate review.", + "source": "figure_specification + key_evidence", + "evidence_available": bool(grounded_required), + "strict_evidence_required": bool(evidence_records and required_priorities), + "required_priority_count": len(required_priorities), + "grounded_required_priority_count": len(grounded_required), + "priorities": priorities, + "evidence_catalog": list(evidence_catalog.values()), + "forbidden": forbidden, + "unknown": unknown, + } + + +def build_repair_plan(review: dict, source_candidate_id: str | None = None, round_index: int | None = None) -> dict: + """Turn critic output into a deterministic, auditable Image2 edit plan.""" + + items: list[dict] = [] + seen: set[tuple[str, str]] = set() + preserve = [str(item).strip() for item in review.get("preserve", []) if str(item).strip()] + + def add(issue_type: str, issue: Any, severity: str, expected_check: str, evidence_ids: list[str] | None = None) -> None: + if isinstance(issue, dict): + instruction = str(issue.get("repair") or issue.get("instruction") or issue.get("reason") or issue.get("label") or issue).strip() + region = issue.get("region") or issue.get("visible_region") or issue.get("target") or "auto-localize" + ids = [str(item) for item in issue.get("evidence_ids", []) if str(item).strip()] + else: + instruction = str(issue).strip() + region = "auto-localize" + ids = [] + if not instruction: + return + key = (issue_type, instruction.casefold()) + if key in seen: + return + seen.add(key) + items.append({ + "id": f"RP_{len(items) + 1:03d}", + "region": region, + "issue_type": issue_type, + "severity": severity, + "evidence_ids": list(dict.fromkeys([*(evidence_ids or []), *ids])), + "preserve": preserve, + "instruction": instruction, + "expected_check": expected_check, + }) + + alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} + for issue in alignment.get("missing_priorities", []): + ids = issue.get("evidence_ids", []) if isinstance(issue, dict) else [] + add("paper_missing_priority", issue, "critical", "paper_alignment.missing_priorities no longer contains this priority", ids) + for issue in alignment.get("evidence_conflicts", []): + ids = issue.get("evidence_ids", []) if isinstance(issue, dict) else [] + add("paper_evidence_conflict", issue, "critical", "paper_alignment.evidence_conflicts is empty for this claim", ids) + for issue in alignment.get("unsupported_visual_claims", []): + add("unsupported_visual_claim", issue, "critical", "paper_alignment.unsupported_visual_claims no longer contains this claim") + + scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + for field in ("missing_modules", "missing_relations", "reversed_relations", "invented_items", "role_mismatches", "containment_mismatches"): + for issue in scientific.get(field, []): + add(f"scientific_{field}", issue, "critical", f"scientific.{field} is empty for this issue") + + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} + for field in ("missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"): + for issue in topology.get(field, []): + add(f"topology_{field}", issue, "critical", f"topology.{field} is empty for this connector") + + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + for field in ("missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", "unexpected_labels"): + for issue in ocr.get(field, []): + add(f"ocr_{field}", issue, "high", f"ocr.{field} is empty for this label") + + for instruction in review.get("repair", []): + add("critic_instruction", instruction, "medium", "the originating critic issue is resolved") + for issue in review.get("hard_errors", []): + add("hard_error", issue, "critical", "the hard error is absent after full re-review") + + template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} + if not template.get("passed", False): + add("template_alignment", "Improve macro-panel alignment, reading order, connector rhythm, and density without changing scientific content.", "medium", "template.passed is true") + aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + if not aesthetic.get("passed", False): + add("aesthetic_quality", "Improve hierarchy, balance, whitespace, color consistency, mechanism visualization, and publication polish while preserving every correct scientific element.", "medium", "aesthetic.passed is true") + basic = review.get("basic", {}) if isinstance(review.get("basic"), dict) else {} + if not basic.get("visual_enrichment_passed", True): + add("visual_enrichment", "Add paper-grounded mechanism cues and scientific visual detail inside existing regions; do not alter labels or connector topology.", "high", "basic.visual_enrichment_passed is true") + + return { + "summary": "Machine-executable repair plan compiled from all candidate critics.", + "source_candidate_id": source_candidate_id, + "round_index": round_index, + "item_count": len(items), + "preserve": preserve, + "items": items, + } diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index ba9ee7e..8346026 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -31,7 +31,7 @@ def run_paper_to_image( review_model: str | None = None, domain_profile: str = "auto", template: str = "auto", - repair_rounds: int = 1, + repair_rounds: int = 3, repair_source: str | Path | None = None, ocr_engine: str = "auto", ocr_lang: str = "en_ch", @@ -146,7 +146,7 @@ def run_paper_to_image( "review_model": review_model, "domain_profile": selected_domain["id"], "template": selected_template["template_id"], - "repair_rounds": max(0, min(1, int(repair_rounds))), + "repair_rounds": max(0, min(4, int(repair_rounds))), "repair_source": str(Path(repair_source).resolve()) if repair_source else None, "resume_candidates": bool(resume_candidates), "ocr_engine": ocr_engine, @@ -205,6 +205,8 @@ def run_paper_to_image( "evidence_map.json", "document_index.json", "paper_review.json", + "key_evidence.json", + "review_ground_truth.json", "review_coverage_report.json", "domain_profile.json", "template_profiles/", @@ -225,8 +227,11 @@ def run_paper_to_image( "ocr_review.json", "template_alignment_report.json", "scientific_critic_report.json", + "paper_alignment_report.json", "topology_critic_report.json", "aesthetic_critic_report.json", + "repair_plan.json", + "repair_history.json", "selected_image.png" if generation.get("selected_image") else "engineering_preview.png", ], } diff --git a/rfs/workflows/paper_to_editable.py b/rfs/workflows/paper_to_editable.py index bed31f1..3831341 100644 --- a/rfs/workflows/paper_to_editable.py +++ b/rfs/workflows/paper_to_editable.py @@ -29,7 +29,7 @@ def run_paper_to_editable( review_model: str | None = None, domain_profile: str = "auto", template: str = "auto", - repair_rounds: int = 1, + repair_rounds: int = 3, rebuild_asset_mode: str = "api", rebuild_asset_policy: str = "smart-api", layout_mode: str = "hybrid", diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index ec910bb..712cd79 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -122,7 +122,7 @@ def test_cli_production_defaults(self): self.assertEqual(args.aspect_ratio, "auto") self.assertEqual(args.domain_profile, "auto") self.assertEqual(args.template, "auto") - self.assertEqual(args.repair_rounds, 1) + self.assertEqual(args.repair_rounds, 3) def test_image2_native_canvas_ratio_normalization(self): self.assertEqual(native_image2_aspect_ratio("16:9"), "3:2") @@ -1239,6 +1239,138 @@ def test_local_ocr_can_block_vlm_pass(self): self.assertIn("Document Parser", review["ocr"]["missing_labels"]) self.assertFalse(review["production_pass"]) + def test_paper_evidence_is_sent_to_critic_and_alignment_is_a_hard_gate(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + image = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (800, 600), "white").save(image) + Image.new("RGB", (800, 600), "white").save(blueprint) + plan = _plan() + plan["review_ground_truth"] = { + "summary": "Evidence contract.", + "strict_evidence_required": True, + "priorities": [{ + "id": "GT_MODULE_parser", + "kind": "module", + "label": "Document Parser", + "required": True, + "evidence_ids": ["E0001"], + "evidence": [{"evidence_id": "E0001", "page": 2, "section": "Method", "text": "The Document Parser splits the input paper into page-aware passages."}], + }], + "forbidden": [], + "unknown": [], + } + captured = {} + + def critic(path, blueprint_path, prompt): + captured["prompt"] = prompt + result = _passing_critic(path, blueprint_path, prompt) + result["paper_alignment"] = { + "priorities": [{"id": "GT_MODULE_parser", "status": "supported", "visible_evidence": "Document Parser is shown", "evidence_ids": ["E0001"]}], + "missing_priorities": [], + "evidence_conflicts": [], + "unsupported_visual_claims": ["A memory bank is shown although the paper does not support it"], + "score": 0.8, + "passed": False, + } + return result + + review = review_candidate(image, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertIn("page-aware passages", captured["prompt"]) + self.assertIn("E0001", captured["prompt"]) + self.assertFalse(review["paper_alignment"]["passed"]) + self.assertFalse(review["production_pass"]) + self.assertTrue(any("memory bank" in item.casefold() for item in review["hard_errors"])) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_multi_round_repair_can_succeed_on_second_round(self, post): + calls = {"count": 0} + + def critic(path, blueprint, prompt): + calls["count"] += 1 + result = _passing_critic(path, blueprint, prompt) + if calls["count"] == 1: + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.3, "passed": False}) + result["repair"] = ["Restore the exact Document Parser label"] + result["hard_errors"] = ["Document Parser is missing"] + elif calls["count"] == 2: + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.7, "passed": False}) + result["repair"] = ["Correct the remaining Document Parser spelling"] + result["hard_errors"] = ["Document Parser is still misspelled"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + self.assertEqual(result["selected_candidate_id"], "repair_02") + self.assertEqual(result["repair_candidates"], 2) + self.assertEqual(result["repair_stop_reason"], "all_gates_passed") + self.assertEqual(post.call_count, 3) + history = json.loads((root / "repair_history.json").read_text(encoding="utf-8")) + self.assertTrue(all(item["accepted"] for item in history["rounds"])) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_scientific_regressions_are_rejected_and_two_failures_stop_repairs(self, post): + calls = {"count": 0} + + def critic(path, blueprint, prompt): + calls["count"] += 1 + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + if calls["count"] >= 2: + result["ocr"].update({"missing_labels": [], "score": 1.0, "passed": True}) + result["scientific"].update({"missing_modules": ["Evidence Graph Builder"], "score": 0.6, "passed": False}) + result["hard_errors"] = ["Evidence Graph Builder disappeared"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + history = json.loads((root / "repair_history.json").read_text(encoding="utf-8")) + self.assertEqual(history["completed_rounds"], 2) + self.assertEqual(history["stop_reason"], "two_consecutive_rounds_without_improvement") + self.assertTrue(all(not item["accepted"] for item in history["rounds"])) + self.assertTrue(any("scientific score decreased" in reason for reason in history["rounds"][0]["rejection_reasons"])) + self.assertEqual(post.call_count, 3) + + @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + def test_resume_reuses_completed_repair_rounds_without_new_image_calls(self, post): + def critic(path, blueprint, prompt): + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value", "RFS_IMAGE_MODEL": "image-2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + self.assertEqual(post.call_count, 3) + + post.reset_mock() + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "1.5:1", "language": "English"}, template, root / "layout_blueprint.png", root, asset_mode="image2", candidates=1, review_mode="vlm", repair_rounds=4, ocr_engine="off", critic_adapter=critic, resume_candidates=True, require_visual_enrichment=False) + + post.assert_not_called() + manifest = json.loads((root / "image2_request_manifest.json").read_text(encoding="utf-8")) + resumed = [item for item in manifest["requests"] if item.get("candidate_id", "").startswith("repair_")] + self.assertTrue(resumed) + self.assertTrue(all(item["mode"] == "existing_repair_resume" for item in resumed)) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_review_contract.py b/tests/unit/test_review_contract.py new file mode 100644 index 0000000..62c2d11 --- /dev/null +++ b/tests/unit/test_review_contract.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import unittest + +from rfs.paper_to_image.review_contract import build_repair_plan, compile_review_ground_truth + + +class ReviewContractTests(unittest.TestCase): + def test_ground_truth_embeds_page_section_and_original_excerpt(self): + plan = { + "figure_specification": { + "central_claim": {"text": "Evidence-constrained reasoning reduces unsupported conclusions.", "evidence_ids": ["E1"]}, + "modules": [{"id": "parser", "name": "Document Parser", "evidence_ids": ["E2"]}], + "relations": [{"source": "parser", "target": "answer", "type": "data_flow", "evidence_ids": ["E2"]}], + "outputs": [{"id": "answer", "name": "Evidence-linked Answer", "evidence_ids": ["E2"]}], + "training_flow": ["Training loss"], + "must_show": ["Training loss", "Document Parser"], + "forbidden_inventions": ["External memory bank"], + "uncertainties": ["The internal decoder architecture is not specified"], + } + } + evidence = [ + {"id": "E1", "page": 1, "section_hint": "Abstract", "text": "The method reduces unsupported conclusions.", "confidence": 1.0, "source": "pymupdf"}, + {"id": "E2", "page": 3, "section_hint": "Method", "text": "The Document Parser creates page-aware passages used for the final answer.", "confidence": 0.99, "source": "pymupdf"}, + {"id": "E3", "page": 4, "section_hint": "Training", "text": "Training loss is optimized over synthetic tasks.", "confidence": 0.98, "source": "pymupdf"}, + ] + + contract = compile_review_ground_truth(plan, evidence) + + parser = next(item for item in contract["priorities"] if item["label"] == "Document Parser") + self.assertEqual(parser["evidence"][0]["page"], 3) + self.assertEqual(parser["evidence"][0]["section"], "Method") + self.assertIn("page-aware passages", parser["evidence"][0]["text"]) + self.assertTrue(contract["strict_evidence_required"]) + self.assertEqual(contract["forbidden"][0]["label"], "External memory bank") + self.assertIn("not specified", contract["unknown"][0]["label"]) + training = next(item for item in contract["priorities"] if item["label"] == "Training loss") + self.assertEqual(training["evidence_link_method"], "deterministic_text_match") + self.assertEqual(training["evidence"][0]["evidence_id"], "E3") + self.assertEqual(sum(item["label"] == "Training loss" for item in contract["priorities"]), 1) + self.assertEqual(sum(item["label"] == "Document Parser" for item in contract["priorities"]), 1) + + def test_repair_plan_prioritizes_paper_conflicts_and_carries_evidence_ids(self): + review = { + "preserve": ["correct left panel"], + "paper_alignment": { + "missing_priorities": [{"id": "GT_MODULE_parser", "label": "Document Parser", "reason": "missing", "evidence_ids": ["E2"]}], + "evidence_conflicts": [], + "unsupported_visual_claims": ["Unsupported memory bank"], + }, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": []}, + "topology": {"missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": []}, + "ocr": {"missing_labels": ["Document Parser"], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "unexpected_labels": []}, + "template": {"passed": True}, + "aesthetic": {"passed": True}, + "basic": {"visual_enrichment_passed": True}, + "repair": [], + "hard_errors": [], + } + + repair = build_repair_plan(review, "candidate_01", 1) + + self.assertEqual(repair["items"][0]["issue_type"], "paper_missing_priority") + self.assertEqual(repair["items"][0]["severity"], "critical") + self.assertEqual(repair["items"][0]["evidence_ids"], ["E2"]) + self.assertEqual(repair["items"][0]["preserve"], ["correct left panel"]) + + +if __name__ == "__main__": + unittest.main() From d3a84cb86d710b7c5c2a983f9f4cdb7acea98f3b Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 08:02:46 +0800 Subject: [PATCH 71/74] Add evidence-aware benchmark re-auditing --- .../schema/paper_image_review.schema.json | 10 +- docs/paper-to-image.md | 55 ++++- rfs/cli.py | 10 +- rfs/evaluation/__init__.py | 3 +- rfs/evaluation/benchmarking.py | 184 +++++++++++++++- rfs/paper_to_image/critics.py | 143 +++++++++++-- rfs/paper_to_image/generator.py | 86 +++++++- rfs/paper_to_image/planner.py | 38 ++-- rfs/paper_to_image/review_contract.py | 15 +- rfs/paper_to_image/workflow.py | 2 + tests/integration/test_paper_to_image.py | 201 +++++++++++++++++- tests/unit/test_benchmarking.py | 156 ++++++++++++++ tests/unit/test_review_contract.py | 22 +- 13 files changed, 845 insertions(+), 80 deletions(-) diff --git a/benchmarks/schema/paper_image_review.schema.json b/benchmarks/schema/paper_image_review.schema.json index 2ac8b84..e42afd9 100644 --- a/benchmarks/schema/paper_image_review.schema.json +++ b/benchmarks/schema/paper_image_review.schema.json @@ -2,12 +2,20 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Frozen judge or human-calibrated paper-image review", "type": "object", - "required": ["scientific", "ocr", "information", "clarity", "aesthetic", "stability"], + "required": ["scientific", "paper_alignment", "topology", "ocr", "information", "clarity", "aesthetic", "stability"], "properties": { "scientific": { "type": "object", "required": ["score", "missing_modules", "missing_relations", "reversed_relations", "invented_items"] }, + "paper_alignment": { + "type": "object", + "required": ["score", "passed", "priorities", "missing_priorities", "evidence_conflicts", "unsupported_visual_claims"] + }, + "topology": { + "type": "object", + "required": ["score", "passed", "missing_relations", "reversed_relations", "invented_relations"] + }, "ocr": { "type": "object", "required": ["score", "missing_labels", "misspelled_labels", "forbidden_labels_found"] diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 77c11c0..0e5501c 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -129,6 +129,23 @@ reviewed against the current exact-label and scientific contract. If it fails, the best source is edited once and reviewed again. The request manifest marks the reused source separately from the real Image2 repair request. +Re-audit an existing complete or failed run without regenerating an image: + +```powershell +rfs benchmark audit ` + --run output/paper_to_image/existing_run ` + --case benchmarks/paper-to-image/cases/101_vit_linear ` + --candidate-image output/paper_to_image/existing_run/candidates/repair_02.png ` + --json +``` + +The audit rebuilds `review_ground_truth.json`, scans the selected image against +quoted paper evidence, exact labels, visible connector topology, and aesthetics, +then writes `benchmark_review.json`, `paper_alignment_report.json`, +`reaudit_repair_plan.json`, and `paper_image_reaudit.json`. If neither +`selected_image.png` nor a selected candidate exists, the highest-scoring +recorded candidate with an existing image is audited automatically. + Never reuse a key that has appeared in chat, source control, an issue, or a terminal transcript. The command records only `api_key_present`; it never records a key value. ## Paper Review Contract @@ -252,7 +269,8 @@ removes isolated appendix-only diagram labels, innovations that cannot be grounded to novelty evidence, duplicate aliases such as `MLP`/`MLP Head`, and input shortcuts that bypass an explicit intermediate representation. -The general production review and focused topology review execute concurrently. +The general production review and focused topology review execute concurrently +for every contract that declares visible relations, including linear pipelines. Their default request timeout is 90 seconds with no automatic retry, preventing two independent judges from creating a multi-minute sequential stall. Configure `RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and @@ -260,11 +278,25 @@ two independent judges from creating a multi-minute sequential stall. Configure budget. Existing candidates can be passed through `--repair-source` so review or localized repair does not regenerate the correct parts of the image. +The focused topology review is authoritative only for visible connector +geometry. When it cleanly passes every declared relation, connector-only false +positives from the general critic are suppressed and recorded in +`cross_critic_reconciliation`; paper semantics, entity roles, OCR, and +aesthetics remain under their original hard gates. If both critics identify the +same missing, bypassed, reversed, or invented connector, the failure remains +blocking. + Image-2 edit requests default to a 120-second timeout controlled by `RFS_IMAGE2_TIMEOUT`. Network timeouts stop the candidate immediately instead of blindly regenerating the entire image. Retryable HTTP or provider failures may retry up to `--image-retries`; the default is one. +A full provider read/connect timeout also stops the multi-round repair loop with +`repair_stop_reason=provider_timeout`; another nominal repair round must not +repeat the same 120-second blind wait. Fast failures such as HTTP 429 remain +provider failures rather than semantic non-improvement and may use later repair +rounds. + ## Stability Audit Production stability requires repeated independent candidates rather than a @@ -287,10 +319,19 @@ candidates and reused `--repair-source` images are excluded because they are not independent generations. `rfs benchmark score` consumes the generated report automatically. -The review gate also rejects `unexpected_labels`: visible scientific labels -outside `figure_specification.required_labels` and `repeatable_labels`. This is -stricter than checking hallucinations alone because a term can be supported by -the paper yet still be outside the intended overview contract. +The review records every `unexpected_label`, but only +`unsupported_unexpected_labels` are blocking. A visible detail outside +`required_labels` may remain when the scientific critic or paper-evidence critic +supports it; a label explicitly identified as invented or unsupported still +fails OCR/scientific production gates. Duplicate required labels remain errors, +while repeated non-required implementation details are recorded as +`noncritical_duplicate_labels` instead of causing a false failure. + +`training_flow` and `inference_flow` remain part of the paper contract, but they +are contextual by default. They become mandatory visual priorities only when an +item explicitly sets `required`, `must_appear_in_figure`, or +`visible_label_required`, or when the same content is listed in `must_show` or +`required_labels`. The focused topology judge also supports explicit containment semantics. If a declared operation container visibly contains an evidence-supported repeatable @@ -325,8 +366,8 @@ Every Image2 candidate must pass: - exact-label OCR with no missing, misspelled, duplicate, or copied reference labels; - scientific module and relation checks with no invention or reversal; -- for feedback, branch, multimodal, and dense layouts, a second focused topology - review that verifies visible connector endpoints and arrowheads, rejects +- for every declared visible relation, a second focused topology review that + verifies visible connector endpoints and arrowheads, rejects bypasses, and writes `topology_critic_report.json`; - template alignment score of at least `0.70` with no copied reference content; - aesthetic score of at least `0.70`; diff --git a/rfs/cli.py b/rfs/cli.py index 80ae71a..9958034 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -12,7 +12,7 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution from .editable_rebuild import rebuild_editable -from .evaluation import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, run_pdf_extraction_stress_suite, score_benchmark_case, validate_benchmark_case +from .evaluation import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, run_pdf_extraction_stress_suite, score_benchmark_case, validate_benchmark_case from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image from .workflows import run_paper_to_editable from .professional_rebuild import rebuild_editable_pro @@ -336,7 +336,7 @@ def build_parser() -> argparse.ArgumentParser: coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") - benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score"]) + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score", "audit"]) benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") @@ -347,6 +347,8 @@ def build_parser() -> argparse.ArgumentParser: benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") + benchmark.add_argument("--review-model", help="Optional frozen VLM model for benchmark audit.") + benchmark.add_argument("--candidate-image", help="Optional explicit image to re-audit inside an existing paper-to-image run.") benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") benchmark.add_argument("--rasterize-dpi", type=int, help="For benchmark fast/fast-suite, convert local PDF inputs to image-only scanned PDFs at this DPI before extraction.") benchmark.add_argument("--json", action="store_true", help="Emit JSON.") @@ -625,6 +627,10 @@ def main(argv: list[str] | None = None) -> int: if not args.out: parser.error("benchmark pdf-suite requires --out") result = run_pdf_extraction_stress_suite(args.out, ocr_engine=args.ocr_engine) + elif args.benchmark_action == "audit": + if not args.run_dir: + parser.error("benchmark audit requires --run; --case is optional but enables benchmark scoring") + result = audit_paper_image_run(args.run_dir, case_dir=args.case, candidate_image=args.candidate_image, review_model=args.review_model, ocr_engine=args.ocr_engine) else: if not args.case or not args.run_dir: parser.error("benchmark score requires --case and --run") diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index 44582bd..8df7580 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -1,11 +1,12 @@ """Deterministic and model-assisted quality evaluation APIs.""" from .rebuild_visual import run_rebuild_visual_quality_check -from .benchmarking import fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case +from .benchmarking import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case from .pdf_extraction_benchmark import run_pdf_extraction_stress_suite __all__ = [ "list_benchmark_cases", + "audit_paper_image_run", "fetch_benchmark_case", "run_rebuild_visual_quality_check", "run_benchmark_case", diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 22b08e9..1ab8ca4 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -46,6 +46,9 @@ def _threshold_failures(metrics: dict, thresholds: dict) -> list[str]: "plan_forbidden_content_count", "full_slide_image_count", "blocking_visual_issue_count", + "evidence_conflict_count", + "unsupported_visual_claim_count", + "missing_paper_priority_count", } failures = [] for key, threshold in thresholds.items(): @@ -340,6 +343,11 @@ def _select_candidate_review(run: Path) -> dict: def _unexpected_visible_labels(review: dict, specification: dict) -> list[str]: + ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} + if "unsupported_unexpected_labels" in ocr: + return [str(value) for value in ocr.get("unsupported_unexpected_labels", []) if str(value).strip()] + if ocr.get("unexpected_labels"): + return [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] allowed = { _normalized_text(value) for value in specification.get("required_labels", []) if _normalized_text(value) @@ -352,7 +360,7 @@ def _unexpected_visible_labels(review: dict, specification: dict) -> list[str]: _normalized_text(item.get("label")) for item in specification.get("relations", []) if isinstance(item, dict) and _normalized_text(item.get("label")) ) - detected = review.get("ocr", {}).get("detected_labels", []) if isinstance(review.get("ocr"), dict) else [] + detected = ocr.get("detected_labels", []) return [str(value) for value in detected if _normalized_text(value) and _normalized_text(value) not in allowed] @@ -400,6 +408,8 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, review = {**review, **benchmark_review} scientific = review.get("scientific", {}) if isinstance(review.get("scientific"), dict) else {} + paper_alignment = review.get("paper_alignment", {}) if isinstance(review.get("paper_alignment"), dict) else {} + topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} clarity = review.get("clarity", {}) if isinstance(review.get("clarity"), dict) else {} @@ -410,6 +420,8 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, if isinstance(benchmark_review.get("stability"), dict): stability = benchmark_review["stability"] specification = _load(run / "figure_specification.json") + review_ground_truth = _load(run / "review_ground_truth.json") + repair_history = _load(run / "repair_history.json") planning_contract = _score_planning_contract(expected, specification) stability_audit = _audit_candidate_stability(candidate_report, specification) @@ -435,9 +447,35 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, if stability_audit.get("available") else stability.get("production_pass_rate", stability.get("score", 0.0)) ) - unexpected_labels = list(review.get("ocr", {}).get("unexpected_labels", []) if isinstance(review.get("ocr"), dict) else []) - if not unexpected_labels: - unexpected_labels = _unexpected_visible_labels(review, specification) + paper_alignment_score = _clamp(paper_alignment.get("score")) if paper_alignment else None + topology_score = _clamp(topology.get("score")) if topology else None + required_priority_ids = { + str(item.get("id") or "").strip() + for item in review_ground_truth.get("priorities", []) + if isinstance(item, dict) and item.get("required") and str(item.get("id") or "").strip() + } + supported_priority_ids = { + str(item.get("id") or "").strip() + for item in paper_alignment.get("priorities", []) + if isinstance(item, dict) + and str(item.get("status") or "").strip().casefold() in {"supported", "present", "correct"} + and str(item.get("id") or "").strip() + } + evidence_priority_coverage = ( + _clamp(len(required_priority_ids & supported_priority_ids) / max(1, len(required_priority_ids))) + if required_priority_ids else None + ) + evidence_conflicts = list(paper_alignment.get("evidence_conflicts", [])) if paper_alignment else [] + unsupported_visual_claims = list(paper_alignment.get("unsupported_visual_claims", [])) if paper_alignment else [] + missing_priorities = list(paper_alignment.get("missing_priorities", [])) if paper_alignment else [] + repair_rounds = [item for item in repair_history.get("rounds", []) if isinstance(item, dict)] + accepted_repairs = [item for item in repair_rounds if item.get("accepted")] + rejected_repairs = [item for item in repair_rounds if not item.get("accepted")] + regression_rejections = [ + item for item in rejected_repairs + if any("decreased" in str(reason).casefold() or "pass to fail" in str(reason).casefold() for reason in item.get("rejection_reasons", [])) + ] + unexpected_labels = _unexpected_visible_labels(review, specification) hard_failures = list(review.get("hard_errors", [])) if not scientific or not aesthetic or scientific.get("engineering_only") or aesthetic.get("engineering_only"): hard_failures.append("frozen benchmark judge did not provide scientific and aesthetic sections") @@ -451,6 +489,17 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, hard_failures.append("required labels missing or misspelled") if unexpected_labels: hard_failures.append("unexpected visible labels outside the normalized figure contract") + evidence_review_required = bool(review_ground_truth.get("strict_evidence_required")) + if evidence_review_required and not paper_alignment: + hard_failures.append("paper-evidence alignment review is missing") + if paper_alignment and (not paper_alignment.get("passed") or missing_priorities): + hard_failures.append("required paper priorities are missing or unverified") + if evidence_conflicts: + hard_failures.append("candidate conflicts with quoted paper evidence") + if unsupported_visual_claims: + hard_failures.append("candidate contains unsupported visual claims") + if topology and not topology.get("passed", False): + hard_failures.append("visible connector topology failed review") if stability_audit.get("available") and not stability_audit.get("stable"): hard_failures.append("repeated generation stability below production threshold") plan_entity_min = float(case.get("thresholds", {}).get("plan_entity_recall", 1.0)) @@ -471,6 +520,16 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "clarity_score": clarity_score, "aesthetic_score": aesthetic_score, "stability_score": stability_score, + "paper_alignment_score": paper_alignment_score, + "evidence_priority_coverage": evidence_priority_coverage, + "evidence_conflict_count": len(evidence_conflicts), + "unsupported_visual_claim_count": len(unsupported_visual_claims), + "missing_paper_priority_count": len(missing_priorities), + "topology_score": topology_score, + "repair_attempt_count": len(repair_rounds), + "accepted_repair_count": len(accepted_repairs), + "rejected_repair_count": len(rejected_repairs), + "repair_regression_rejection_count": len(regression_rejections), "hallucination_count": len(invented), "forbidden_content_count": len(forbidden_found), "unexpected_label_count": len(unexpected_labels), @@ -480,7 +539,10 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, } thresholds = case.get("thresholds", {}) threshold_failures = _threshold_failures(metrics, thresholds) - total_score = round(0.40 * scientific_score + 0.15 * information_score + 0.15 * clarity_score + 0.20 * aesthetic_score + 0.10 * stability_score, 4) + if paper_alignment and topology: + total_score = round(0.28 * scientific_score + 0.20 * float(paper_alignment_score or 0.0) + 0.12 * information_score + 0.12 * clarity_score + 0.10 * float(topology_score or 0.0) + 0.13 * aesthetic_score + 0.05 * stability_score, 4) + else: + total_score = round(0.40 * scientific_score + 0.15 * information_score + 0.15 * clarity_score + 0.20 * aesthetic_score + 0.10 * stability_score, 4) return { "summary": "Paper-to-image benchmark score.", "suite": "paper-to-image", @@ -489,13 +551,123 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "run_dir": str(run), "metrics": metrics, "planning_contract": planning_contract, + "paper_alignment": paper_alignment, + "repair_history": repair_history, "stability_audit": stability_audit, "total_score": total_score, "hard_failures": sorted(set(hard_failures)), "threshold_failures": threshold_failures, "passed": not hard_failures and not threshold_failures, - "policy": "scientific, relation, terminology, and hallucination failures cannot be offset by aesthetics", + "policy": "paper-evidence alignment, scientific content, topology, terminology, and hallucination failures cannot be offset by aesthetics", + } + + +def audit_paper_image_run( + run_dir: str | Path, + case_dir: str | Path | None = None, + candidate_image: str | Path | None = None, + review_model: str | None = None, + ocr_engine: str = "off", + ocr_lang: str = "en_ch", + critic_adapter=None, + topology_adapter=None, +) -> dict[str, Any]: + """Re-scan an existing generated image against its paper evidence contract. + + This deliberately does not regenerate the image. It upgrades legacy runs to + the current frozen review gates and leaves an actionable repair plan when + evidence, topology, OCR, or aesthetic checks fail. + """ + + run = Path(run_dir).resolve() + if not run.exists(): + raise FileNotFoundError(f"Paper-to-image run does not exist: {run}") + from ..paper_to_image.critics import review_candidate + from ..paper_to_image.review_contract import build_repair_plan, compile_review_ground_truth + + plan = {} + for name in ("paper_summary", "figure_specification", "design_plan", "layout_intent", "visual_metaphors", "style_plan"): + value = _load(run / f"{name}.json") + if value: + plan[name] = value + if not isinstance(plan.get("figure_specification"), dict): + raise ValueError(f"figure_specification.json is missing or invalid in {run}") + + document_model = _load(run / "document_model.json") + key_evidence = _load(run / "key_evidence.json") + evidence_records = document_model.get("evidence", []) if isinstance(document_model.get("evidence"), list) else key_evidence.get("evidence", []) + ground_truth = compile_review_ground_truth(plan, evidence_records) + plan["review_ground_truth"] = ground_truth + write_json(run / "review_ground_truth.json", ground_truth) + + candidate_report = _load(run / "candidate_review.json") + selected_id = str(candidate_report.get("selected_candidate_id") or "").strip() + candidate = Path(candidate_image).resolve() if candidate_image else run / "selected_image.png" + if candidate_image: + selected_id = "explicit_reaudit_candidate" + elif not candidate.exists(): + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict) and str(item.get("path") or "").strip()] + selected_record = next((item for item in candidate_records if str(item.get("candidate_id")) == selected_id), None) + if selected_record is None and candidate_records: + selected_record = max(candidate_records, key=lambda item: (bool(item.get("production_pass")), float(item.get("score") or 0.0))) + if selected_record is None: + raise FileNotFoundError(f"No reviewable candidate image is recorded in {run}") + selected_id = str(selected_record.get("candidate_id") or "reaudit_candidate") + candidate = Path(str(selected_record.get("path"))) + if not candidate.is_absolute(): + candidate = run / candidate + blueprint = run / "layout_blueprint.png" + if not candidate.exists(): + raise FileNotFoundError(f"Selected candidate image is missing in {run}") + if not blueprint.exists(): + raise FileNotFoundError(f"layout_blueprint.png is missing in {run}") + template = _load(run / "selected_template.json") + preferences = _load(run / "preferences.json") + acceptable_ratios = [ + str(value) for value in (preferences.get("requested_aspect_ratio"), preferences.get("aspect_ratio")) + if value and str(value) != "auto" + ] + generation_parameters = _load(run / "generation_parameters.json") + review = review_candidate( + candidate, + blueprint, + plan, + template, + mode="vlm", + model=review_model, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + critic_adapter=critic_adapter, + topology_adapter=topology_adapter, + acceptable_aspect_ratios=acceptable_ratios, + require_visual_enrichment=str(generation_parameters.get("asset_mode") or "image2") == "image2", + ) + repair_plan = build_repair_plan(review, selected_id, 0) + write_json(run / "benchmark_review.json", review) + write_json(run / "paper_alignment_report.json", { + "summary": "Independent paper-evidence re-audit for the selected complete image.", + "candidates": [{"candidate_id": selected_id, **review.get("paper_alignment", {})}], + }) + write_json(run / "reaudit_repair_plan.json", repair_plan) + benchmark = score_benchmark_case(case_dir, run, run) if case_dir else None + result = { + "summary": "Existing complete image re-audited against quoted paper evidence and current visual gates.", + "ok": True, + "run_dir": str(run), + "candidate_id": selected_id, + "candidate": str(candidate), + "review_ground_truth": str(run / "review_ground_truth.json"), + "required_priority_count": ground_truth.get("required_priority_count"), + "grounded_required_priority_count": ground_truth.get("grounded_required_priority_count"), + "production_pass": bool(review.get("production_pass")), + "overall_score": review.get("overall_score"), + "paper_alignment_score": review.get("paper_alignment", {}).get("score"), + "topology_score": review.get("topology", {}).get("score"), + "repair_item_count": repair_plan.get("item_count"), + "benchmark": benchmark, } + write_json(run / "paper_image_reaudit.json", result) + return result def _image_similarity(reference: Path, preview: Path) -> dict[str, float | None]: diff --git a/rfs/paper_to_image/critics.py b/rfs/paper_to_image/critics.py index 6847e20..bf2358f 100644 --- a/rfs/paper_to_image/critics.py +++ b/rfs/paper_to_image/critics.py @@ -30,6 +30,76 @@ def _issue_text(value: object) -> str: return f"{relation}: {details}" if details else relation +def _is_connector_issue(value: object) -> bool: + text = _issue_text(value).casefold() + return any(token in text for token in ( + "arrow", "connector", "connection", "direct edge", "shortcut", "bypass", + "relation", "reversed", "source ->", "target ->", "连线", "箭头", "直连", + )) + + +def _reconcile_with_focused_topology(vlm_raw: dict, topology_review: dict, ground_truth: dict) -> dict: + """Let the focused connector verifier arbitrate connector-only disagreements. + + The general critic remains authoritative for paper semantics, roles, labels, + and aesthetics. This only suppresses connector claims when the independent + topology pass inspected all declared relations and found no connector issue. + """ + + if not topology_review.get("passed") or any(topology_review.get(field) for field in ("missing_relations", "reversed_relations", "bypassed_relations", "invented_relations")): + return {"applied": False, "reason": "focused topology did not pass cleanly", "suppressed": []} + + suppressed: list[dict] = [] + scientific = vlm_raw.get("scientific") if isinstance(vlm_raw.get("scientific"), dict) else {} + for field in ("missing_relations", "reversed_relations"): + for issue in scientific.get(field, []) if isinstance(scientific.get(field), list) else []: + suppressed.append({"section": "scientific", "field": field, "issue": issue}) + scientific[field] = [] + invented = scientific.get("invented_items", []) if isinstance(scientific.get("invented_items"), list) else [] + scientific["invented_items"] = [issue for issue in invented if not _is_connector_issue(issue)] + for issue in invented: + if _is_connector_issue(issue): + suppressed.append({"section": "scientific", "field": "invented_items", "issue": issue}) + vlm_raw["scientific"] = scientific + + if isinstance(vlm_raw.get("paper_alignment"), dict): + alignment = vlm_raw["paper_alignment"] + for field in ("evidence_conflicts", "unsupported_visual_claims", "repair"): + values = alignment.get(field, []) if isinstance(alignment.get(field), list) else [] + alignment[field] = [issue for issue in values if not _is_connector_issue(issue)] + for issue in values: + if _is_connector_issue(issue): + suppressed.append({"section": "paper_alignment", "field": field, "issue": issue}) + + priority_kinds = { + str(item.get("id") or "").strip(): str(item.get("kind") or "").strip() + for item in ground_truth.get("priorities", []) if isinstance(item, dict) + } + for priority in alignment.get("priorities", []) if isinstance(alignment.get("priorities"), list) else []: + if not isinstance(priority, dict): + continue + status = str(priority.get("status") or "").strip().casefold() + issue = priority.get("visible_evidence") or priority.get("reason") or "" + priority_id = str(priority.get("id") or "").strip() + if status not in {"supported", "present", "correct"} and _is_connector_issue(issue): + priority["status"] = "supported" + priority["reconciled_by"] = "focused_topology" + suppressed.append({"section": "paper_alignment", "field": "priorities", "priority_id": priority_id, "kind": priority_kinds.get(priority_id), "issue": issue}) + if not any(alignment.get(field) for field in ("missing_priorities", "evidence_conflicts", "unsupported_visual_claims")): + alignment["passed"] = True + for field in ("repair", "remove"): + values = vlm_raw.get(field, []) if isinstance(vlm_raw.get(field), list) else [] + vlm_raw[field] = [issue for issue in values if not _is_connector_issue(issue)] + for issue in values: + if _is_connector_issue(issue): + suppressed.append({"section": "critic", "field": field, "issue": issue}) + return { + "applied": bool(suppressed), + "reason": "focused topology is authoritative for visible connector geometry", + "suppressed": suppressed, + } + + def required_labels(plan: dict) -> list[str]: return collect_visible_labels(plan.get("figure_specification", {})) @@ -370,16 +440,19 @@ def review_candidate( local_records, local_engine, local_warning = _local_ocr(candidate, requested_engine, ocr_lang, adapter=ocr_adapter) topology_name = str(plan.get("figure_specification", {}).get("topology") or "unknown") + declared_relations = collect_visual_relations(plan.get("figure_specification", {})) visual_roles = collect_visual_roles(plan.get("figure_specification", {})) has_structured_provenance = any(str(item.get("role") or "").casefold() == "data_source" for item in visual_roles) - topology_required = topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} or has_structured_provenance + topology_required = bool(declared_relations) or topology_name in {"feedback", "branch", "multimodal", "dense_multiframe"} or str(template.get("template_id") or "") in {"feedback", "arbor"} or has_structured_provenance + focused_topology_available = bool(topology_adapter or (critic_adapter is None and vlm_credentials_available())) + focused_topology_required = topology_required and focused_topology_available vlm_raw: dict = {} topology_raw: dict = {} vlm_warning = None topology_warning = None review_available = vlm_credentials_available() or critic_adapter or topology_adapter if mode == "vlm" and review_available: - with ThreadPoolExecutor(max_workers=2 if topology_required else 1) as executor: + with ThreadPoolExecutor(max_workers=2 if focused_topology_required else 1) as executor: critic_future = executor.submit( _vlm_critic, candidate, @@ -393,7 +466,7 @@ def review_candidate( model, critic_adapter, ) - topology_future = executor.submit(_vlm_topology_critic, candidate, plan, model, topology_adapter) if topology_required else None + topology_future = executor.submit(_vlm_topology_critic, candidate, plan, model, topology_adapter) if focused_topology_required else None try: vlm_raw = critic_future.result() except Exception as exc: @@ -405,7 +478,7 @@ def review_candidate( topology_warning = str(exc) elif mode == "vlm": vlm_warning = "VLM review credentials unavailable" - if topology_required: + if focused_topology_required: topology_warning = vlm_warning detected_local = [str(item.get("text") or "").strip() for item in local_records if str(item.get("text") or "").strip()] @@ -430,20 +503,32 @@ def review_candidate( required_normalized = {_normalize_text(value) for value in labels if _normalize_text(value)} duplicate_labels = [str(value) for value in ocr.get("duplicate_labels", [])] allowed_duplicate_labels = [value for value in duplicate_labels if _normalize_text(value) in repeatable_normalized] + noncritical_duplicate_labels = [ + value for value in duplicate_labels + if _normalize_text(value) not in repeatable_normalized and _normalize_text(value) not in required_normalized + ] ocr["allowed_duplicate_labels"] = allowed_duplicate_labels - ocr["duplicate_labels"] = [value for value in duplicate_labels if _normalize_text(value) not in repeatable_normalized] + ocr["noncritical_duplicate_labels"] = noncritical_duplicate_labels + ocr["duplicate_labels"] = [ + value for value in duplicate_labels + if _normalize_text(value) not in repeatable_normalized and _normalize_text(value) in required_normalized + ] detected_labels = [str(value).strip() for value in ocr.get("detected_labels", []) if str(value).strip()] allowed_visible_labels = required_normalized | repeatable_normalized | optional_normalized ocr["unexpected_labels"] = [ value for value in detected_labels if _normalize_text(value) and _normalize_text(value) not in allowed_visible_labels ] + topology_review = _normalize_section(topology_raw, "Topology") if focused_topology_required else {"summary": "Focused topology review not available or not required for this run.", "score": 1.0, "passed": True, "skipped": True} + topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) + if focused_topology_required: + topology_review["passed"] = bool(topology_review.get("passed")) and topology_review["score"] >= 0.9 and topology_issues == 0 and not topology_warning + ground_truth = plan.get("review_ground_truth") if isinstance(plan.get("review_ground_truth"), dict) else {} + cross_critic_reconciliation = _reconcile_with_focused_topology(vlm_raw, topology_review, ground_truth) if focused_topology_required else {"applied": False, "reason": "focused topology was not run", "suppressed": []} scientific = _normalize_section(vlm_raw.get("scientific"), "Scientific") unexpected_visible_labels = [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] scientific["unexpected_visible_labels"] = unexpected_visible_labels - if unexpected_visible_labels: - invented_items = [str(value) for value in scientific.get("invented_items", []) if str(value).strip()] - scientific["invented_items"] = list(dict.fromkeys([*invented_items, *unexpected_visible_labels])) + invented_items = [str(value) for value in scientific.get("invented_items", []) if str(value).strip()] template_review = _normalize_section(vlm_raw.get("template"), "Template") aesthetic = _normalize_section(vlm_raw.get("aesthetic"), "Aesthetic") for field in ("visual_information_density", "mechanism_visualization", "publication_polish"): @@ -451,22 +536,27 @@ def review_candidate( aesthetic[field] = max(0.0, min(1.0, float(aesthetic.get(field, aesthetic["score"])))) except Exception: aesthetic[field] = aesthetic["score"] - topology_review = _normalize_section(topology_raw, "Topology") if topology_required else {"summary": "Focused topology review not required for this topology.", "score": 1.0, "passed": True, "skipped": True} - topology_issues = sum(len(topology_review.get(field, [])) for field in ["missing_relations", "reversed_relations", "bypassed_relations", "invented_relations"]) - if topology_required: - topology_review["passed"] = bool(topology_review.get("passed")) and topology_review["score"] >= 0.9 and topology_issues == 0 and not topology_warning - ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", "unexpected_labels"]) - if ocr_issues == 0: - ocr["score"] = 1.0 - ocr["passed"] = ocr["score"] >= 0.999 and ocr_issues == 0 scientific_issues = sum(len(scientific.get(field, [])) for field in ["missing_modules", "missing_relations", "reversed_relations", "invented_items", "role_mismatches", "containment_mismatches"]) has_innovations = bool(plan.get("figure_specification", {}).get("innovations")) innovation_ok = not has_innovations or bool(scientific.get("innovation_visible", False)) if scientific_issues == 0 and innovation_ok: scientific["score"] = 1.0 scientific["passed"] = scientific["score"] >= 0.95 and scientific_issues == 0 and innovation_ok - ground_truth = plan.get("review_ground_truth") if isinstance(plan.get("review_ground_truth"), dict) else {} paper_alignment = _normalize_paper_alignment(vlm_raw.get("paper_alignment"), ground_truth, scientific, critic_adapter is not None) + unsupported_sources = [*invented_items, *(_issue_text(value) for value in paper_alignment.get("unsupported_visual_claims", []))] + unsupported_unexpected_labels = [ + value for value in unexpected_visible_labels + if any( + _normalize_text(value) + and (_normalize_text(value) in _normalize_text(source) or _normalize_text(source) in _normalize_text(value)) + for source in unsupported_sources if _normalize_text(source) + ) + ] + ocr["unsupported_unexpected_labels"] = unsupported_unexpected_labels + ocr_issues = sum(len(ocr.get(field, [])) for field in ["missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", "unsupported_unexpected_labels"]) + if ocr_issues == 0: + ocr["score"] = 1.0 + ocr["passed"] = ocr["score"] >= 0.999 and ocr_issues == 0 copied = template_review.get("copied_reference_content", []) template_review["passed"] = template_review["score"] >= 0.70 and not copied aesthetic_floor = min(aesthetic["visual_information_density"], aesthetic["mechanism_visualization"], aesthetic["publication_polish"]) @@ -482,10 +572,20 @@ def review_candidate( text = str(value) normalized_error = _normalize_text(text) allowed_duplicate_error = "duplicate" in text.casefold() and any(label in normalized_error for label in repeatable_normalized) + noncritical_duplicate_error = "duplicate" in text.casefold() and any(_normalize_text(label) in normalized_error for label in noncritical_duplicate_labels) + reconciled_connector_error = bool(cross_critic_reconciliation.get("applied")) and _is_connector_issue(text) + benign_unexpected_error = ( + any(token in text.casefold() for token in ("unexpected", "whitelist", "outside the contract", "not in the contract")) + and not unsupported_unexpected_labels + ) or ( + any(token in text.casefold() for token in ("unexpected", "whitelist", "outside the contract", "not in the contract")) + and any(_normalize_text(label) in normalized_error for label in unexpected_visible_labels) + and not any(_normalize_text(label) in normalized_error for label in unsupported_unexpected_labels) + ) contradicted_template_error = "template" in text.casefold() and template_review["passed"] and template_review["score"] >= 0.70 - if not allowed_duplicate_error and not contradicted_template_error: + if not allowed_duplicate_error and not noncritical_duplicate_error and not reconciled_connector_error and not benign_unexpected_error and not contradicted_template_error: hard_errors.append(text) - if topology_required and not topology_review["passed"]: + if focused_topology_required and not topology_review["passed"]: hard_errors.extend(_issue_text(value) for value in topology_review.get("missing_relations", [])) hard_errors.extend(_issue_text(value) for value in topology_review.get("reversed_relations", [])) hard_errors.extend(_issue_text(value) for value in topology_review.get("bypassed_relations", [])) @@ -504,8 +604,8 @@ def review_candidate( repair.extend(_issue_text(value) for value in topology_review.get("repair", []) if _issue_text(value)) repair_regions = list(vlm_raw.get("repair_regions", [])) repair_regions.extend(_issue_text(value) for value in topology_review.get("repair_regions", []) if _issue_text(value)) - unexpected_labels = [str(value) for value in ocr.get("unexpected_labels", []) if str(value).strip()] - repair.extend(f"Remove unexpected visible label: {value}" for value in unexpected_labels) + unexpected_labels = [str(value) for value in ocr.get("unsupported_unexpected_labels", []) if str(value).strip()] + repair.extend(f"Remove unsupported visible label: {value}" for value in unexpected_labels) remove = list(vlm_raw.get("remove", [])) remove.extend(unexpected_labels) if require_visual_enrichment and not visual_enrichment_passed: @@ -523,6 +623,7 @@ def review_candidate( "scientific": scientific, "paper_alignment": paper_alignment, "topology": topology_review, + "cross_critic_reconciliation": cross_critic_reconciliation, "template": template_review, "aesthetic": aesthetic, "hard_errors": hard_errors, diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index 64c3a01..aad6e9d 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -113,7 +113,8 @@ def _candidate_failure_modes(record: dict) -> list[str]: topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} - if ocr.get("missing_labels") or ocr.get("misspelled_labels") or ocr.get("duplicate_labels") or ocr.get("unexpected_labels"): + unexpected_labels = ocr.get("unsupported_unexpected_labels") if "unsupported_unexpected_labels" in ocr else ocr.get("unexpected_labels") + if ocr.get("missing_labels") or ocr.get("misspelled_labels") or ocr.get("duplicate_labels") or unexpected_labels: modes.append("label_accuracy") if scientific.get("missing_modules"): modes.append("missing_modules") @@ -360,6 +361,55 @@ def generate_and_select( if value and str(value) != "auto" ] + try: + review_attempt_limit = max(1, min(3, int(os.getenv("RFS_PAPER_IMAGE_REVIEW_ATTEMPTS", "2")))) + except ValueError: + review_attempt_limit = 2 + + def review_is_inconclusive(review: dict[str, Any]) -> bool: + return bool( + review_mode == "vlm" + and critic_adapter is None + and ( + review.get("vlm_warning") + or review.get("topology_warning") + or "Production critic did not return paper-evidence alignment" in review.get("hard_errors", []) + ) + ) + + def run_review(path: Path) -> dict[str, Any]: + attempts = [] + final_review: dict[str, Any] = {} + for attempt in range(1, review_attempt_limit + 1): + final_review = review_candidate( + path, + blueprint, + plan, + selected_template, + mode=review_mode, + model=review_model, + ocr_engine=ocr_engine, + ocr_lang=ocr_lang, + ocr_adapter=ocr_adapter, + critic_adapter=critic_adapter, + topology_adapter=topology_adapter, + acceptable_aspect_ratios=acceptable_aspect_ratios, + require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment, + ) + inconclusive = review_is_inconclusive(final_review) + attempts.append({ + "attempt": attempt, + "inconclusive": inconclusive, + "vlm_warning": final_review.get("vlm_warning"), + "topology_warning": final_review.get("topology_warning"), + }) + if not inconclusive or attempt >= review_attempt_limit: + break + time.sleep(min(4.0, float(attempt))) + final_review["review_attempts"] = len(attempts) + final_review["review_attempt_history"] = attempts + return final_review + repair_source_path = Path(repair_source).resolve() if repair_source else None if repair_source_path and not repair_source_path.exists(): raise FileNotFoundError(f"Repair source image does not exist: {repair_source_path}") @@ -380,7 +430,7 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) review_started = time.monotonic() - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) + review = run_review(path) review_seconds = round(time.monotonic() - review_started, 3) return { "record": { @@ -412,7 +462,7 @@ def generate_candidate(index: int) -> dict[str, Any]: prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" write_text(prompt_path, prompt) review_started = time.monotonic() - review = review_candidate(path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) + review = run_review(path) review_seconds = round(time.monotonic() - review_started, 3) generation = {"mode": "existing_candidate", "source": str(repair_source_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} records.append({ @@ -506,16 +556,24 @@ def generate_candidate(index: int) -> dict[str, Any]: repair_started = time.monotonic() try: if resume_candidates and repaired_path.exists() and review_path.exists(): - generation = {"mode": "existing_repair_resume", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + stored_review = json.loads(review_path.read_text(encoding="utf-8")) generation_seconds = 0.0 - review = json.loads(review_path.read_text(encoding="utf-8")) - review_seconds = 0.0 + if review_is_inconclusive(stored_review): + generation = {"mode": "existing_repair_reaudit", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + review_started = time.monotonic() + review = run_review(repaired_path) + review_seconds = round(time.monotonic() - review_started, 3) + write_json(review_path, review) + else: + generation = {"mode": "existing_repair_resume", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + review = stored_review + review_seconds = 0.0 else: generation_started = time.monotonic() generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) review_started = time.monotonic() - review = review_candidate(repaired_path, blueprint, plan, selected_template, mode=review_mode, model=review_model, ocr_engine=ocr_engine, ocr_lang=ocr_lang, ocr_adapter=ocr_adapter, critic_adapter=critic_adapter, topology_adapter=topology_adapter, acceptable_aspect_ratios=acceptable_aspect_ratios, require_visual_enrichment=asset_mode == "image2" and require_visual_enrichment) + review = run_review(repaired_path) review_seconds = round(time.monotonic() - review_started, 3) write_json(review_path, review) requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) @@ -560,10 +618,18 @@ def generate_candidate(index: int) -> dict[str, Any]: repair_stop_reason = "two_consecutive_rounds_without_improvement" break except Exception as exc: - no_improvement_streak += 1 - failure = {"candidate_id": repaired_id, "error": str(exc), "prompt_path": str(prompt_path), "repair_source": source["candidate_id"]} + error_text = str(exc) + normalized_error = error_text.casefold() + provider_timeout = any(token in normalized_error for token in ("timed out", "read timeout", "connect timeout")) + provider_failure = provider_timeout or any(token in normalized_error for token in ("http 429", "http 5", "provider", "connection")) + if not provider_failure: + no_improvement_streak += 1 + failure = {"candidate_id": repaired_id, "error": error_text, "prompt_path": str(prompt_path), "repair_source": source["candidate_id"], "provider_failure": provider_failure, "provider_timeout": provider_timeout} failures.append(failure) - repair_history.append({"round": repair_index + 1, "candidate_id": repaired_id, "source_candidate_id": source["candidate_id"], "accepted": False, "error": str(exc)}) + repair_history.append({"round": repair_index + 1, "candidate_id": repaired_id, "source_candidate_id": source["candidate_id"], "accepted": False, "provider_failure": provider_failure, "provider_timeout": provider_timeout, "error": error_text}) + if provider_timeout: + repair_stop_reason = "provider_timeout" + break if no_improvement_streak >= 2: repair_stop_reason = "two_consecutive_rounds_without_improvement" break diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index e2b9da0..9a51fa6 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -544,7 +544,6 @@ def collect_visible_labels(spec: dict, limit: int = 40) -> list[str]: labels: list[str] = [] normalized_labels: set[str] = set() panel_graphs = [item for item in spec.get("panel_graphs", []) if isinstance(item, dict) and item.get("nodes")] if isinstance(spec.get("panel_graphs"), list) else [] - panel_graph_active = len(panel_graphs) >= 2 def add(value: object) -> None: if isinstance(value, dict): @@ -555,16 +554,17 @@ def add(value: object) -> None: labels.append(label) normalized_labels.add(normalized) - for item in spec.get("required_labels", []) if isinstance(spec.get("required_labels"), list) else []: + explicit_required = spec.get("required_labels", []) if isinstance(spec.get("required_labels"), list) else [] + for item in explicit_required: add(item) - terminology = spec.get("terminology", {}) - if isinstance(terminology, dict): - for value in terminology.values(): - add(value) - elif isinstance(terminology, list): - for item in terminology: - add(item) - if not panel_graph_active: + if not explicit_required: + terminology = spec.get("terminology", {}) + if isinstance(terminology, dict): + for value in terminology.values(): + add(value) + elif isinstance(terminology, list): + for item in terminology: + add(item) for field in ("inputs", "modules", "outputs"): for item in spec.get(field, []) if isinstance(spec.get(field), list) else []: add(item) @@ -574,15 +574,15 @@ def add(value: object) -> None: for relation in spec.get("relations", []) if isinstance(spec.get("relations"), list) else []: if isinstance(relation, dict): add(relation.get("label")) - for panel in panel_graphs: - if not isinstance(panel, dict): - continue - add(panel.get("label")) - for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: - add(node) - for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: - if isinstance(relation, dict): - add(relation.get("label")) + for panel in panel_graphs: + if not isinstance(panel, dict): + continue + add(panel.get("label")) + for node in panel.get("nodes", []) if isinstance(panel.get("nodes"), list) else []: + add(node) + for relation in panel.get("relations", []) if isinstance(panel.get("relations"), list) else []: + if isinstance(relation, dict): + add(relation.get("label")) return labels[: max(1, int(limit))] diff --git a/rfs/paper_to_image/review_contract.py b/rfs/paper_to_image/review_contract.py index d6a8057..387b0ce 100644 --- a/rfs/paper_to_image/review_contract.py +++ b/rfs/paper_to_image/review_contract.py @@ -6,7 +6,7 @@ def _text(value: Any) -> str: if isinstance(value, dict): - for key in ("text", "statement", "name", "label", "title", "description", "claim"): + for key in ("text", "statement", "name", "label", "title", "description", "claim", "step"): text = str(value.get(key) or "").strip() if text: return text @@ -86,7 +86,9 @@ def add_priority(kind: str, value: Any, *, importance: str = "required", visual_ label = f"{source} -> {target} [{relation_type}]" item_id = str(value.get("id") or "").strip() status = str(value.get("status") or importance).strip().casefold() - required = status not in {"optional", "unknown", "uncertain"} + explicitly_required = bool(value.get("required") is True or value.get("must_appear_in_figure") or value.get("visible_label_required")) + explicitly_optional = bool(value.get("required") is False or value.get("optional") or status in {"optional", "unknown", "uncertain"}) + required = explicitly_required or (not explicitly_optional and importance != "optional") else: label = _text(value) evidence_ids = [] @@ -168,13 +170,15 @@ def add_priority(kind: str, value: Any, *, importance: str = "required", visual_ ("relations", "relation", "shown with the correct source, target, and direction"), ("feedback_loops", "feedback_loop", "shown as a closed and correctly directed loop"), ("outputs", "output", "shown as an output rather than an internal module"), - ("training_flow", "training_flow", "shown only in the training path"), - ("inference_flow", "inference_flow", "shown only in the inference path"), ("innovations", "innovation", "the paper's novelty is visually identifiable"), ("must_show", "must_show", "explicitly visible in the figure"), ): for item in _list(specification.get(field)): add_priority(kind, item, importance="critical" if kind in {"module", "relation", "feedback_loop"} else "required", visual_expectation=expectation) + for item in _list(specification.get("training_flow")): + add_priority("training_flow", item, importance="optional", visual_expectation="when visible, shown only in the training path") + for item in _list(specification.get("inference_flow")): + add_priority("inference_flow", item, importance="optional", visual_expectation="when visible, shown only in the inference path") for label in _list(specification.get("required_labels")): add_priority("required_label", label, importance="required", visual_expectation="appears as exact visible text") @@ -277,7 +281,8 @@ def add(issue_type: str, issue: Any, severity: str, expected_check: str, evidenc add(f"topology_{field}", issue, "critical", f"topology.{field} is empty for this connector") ocr = review.get("ocr", {}) if isinstance(review.get("ocr"), dict) else {} - for field in ("missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", "unexpected_labels"): + unexpected_field = "unsupported_unexpected_labels" if "unsupported_unexpected_labels" in ocr else "unexpected_labels" + for field in ("missing_labels", "misspelled_labels", "duplicate_labels", "forbidden_labels_found", unexpected_field): for issue in ocr.get(field, []): add(f"ocr_{field}", issue, "high", f"ocr.{field} is empty for this label") diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 8346026..8ac221a 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -195,6 +195,8 @@ def run_paper_to_image( "engineering_preview": generation.get("engineering_preview"), "selected_candidate_id": generation["selected_candidate_id"], "selected_passed_all_checks": generation["selected_passed_all_checks"], + "repair_candidates": generation.get("repair_candidates", 0), + "repair_stop_reason": generation.get("repair_stop_reason"), "repair_source": str(Path(repair_source).resolve()) if repair_source else None, "resume_candidates": bool(resume_candidates), "elapsed_seconds": elapsed, diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index 712cd79..ed75bde 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -16,7 +16,7 @@ from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import required_labels, review_candidate from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio -from rfs.paper_to_image.planner import collect_visual_relations, collect_visual_roles, compile_image_prompt +from rfs.paper_to_image.planner import collect_visible_labels, collect_visual_relations, collect_visual_roles, compile_image_prompt from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint from rfs.paper_to_image.templates import build_template_profiles, render_layout_blueprint, select_template @@ -345,7 +345,7 @@ def critic(path, template, prompt): self.assertEqual(review["ocr"]["allowed_duplicate_labels"], ["Document Parser"]) self.assertEqual(review["ocr"]["duplicate_labels"], []) - def test_candidate_review_rejects_visible_labels_outside_contract_whitelist(self): + def test_candidate_review_records_but_allows_paper_supported_labels_outside_required_list(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) candidate = root / "candidate.png" @@ -360,12 +360,72 @@ def critic(path, template, prompt): review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) - self.assertFalse(review["production_pass"]) + self.assertTrue(review["production_pass"]) self.assertEqual(review["ocr"]["unexpected_labels"], ["Contrastive Loss"]) - self.assertFalse(review["scientific"]["passed"]) - self.assertIn("Contrastive Loss", review["scientific"]["invented_items"]) - self.assertIn("Remove unexpected visible label: Contrastive Loss", review["repair"]) - self.assertIn("Contrastive Loss", review["remove"]) + self.assertEqual(review["ocr"]["unsupported_unexpected_labels"], []) + self.assertTrue(review["scientific"]["passed"]) + self.assertNotIn("Contrastive Loss", review["remove"]) + + def test_candidate_review_rejects_extra_label_when_evidence_review_marks_it_unsupported(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].append("External Memory Bank") + result["scientific"]["invented_items"] = ["External Memory Bank"] + result["paper_alignment"] = { + "priorities": [], + "missing_priorities": [], + "evidence_conflicts": [], + "unsupported_visual_claims": ["External Memory Bank is not supported by the paper"], + "repair": ["Remove External Memory Bank"], + "score": 0.8, + "passed": False, + } + result["hard_errors"] = ["Unexpected visible label: External Memory Bank"] + return result + + review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertFalse(review["production_pass"]) + self.assertEqual(review["ocr"]["unsupported_unexpected_labels"], ["External Memory Bank"]) + self.assertIn("External Memory Bank", review["remove"]) + + def test_candidate_review_allows_duplicate_nonrequired_detail_label(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["ocr"]["detected_labels"].append("Add & Norm") + result["ocr"]["duplicate_labels"] = ["Add & Norm"] + result["hard_errors"] = ["Duplicate label: Add & Norm"] + return result + + review = review_candidate(candidate, blueprint, _plan(), {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic) + + self.assertTrue(review["production_pass"]) + self.assertEqual(review["ocr"]["noncritical_duplicate_labels"], ["Add & Norm"]) + self.assertEqual(review["ocr"]["duplicate_labels"], []) + + def test_explicit_required_labels_are_the_only_forced_visible_labels(self): + specification = { + "required_labels": ["Image Patches", "Transformer Encoder"], + "modules": [{"name": "MLP Head"}, {"name": "Classifier"}], + "terminology": {"patches": "Patches"}, + "panel_graphs": [{"id": "a", "nodes": [{"name": "Add & Norm"}]}], + } + + self.assertEqual(collect_visible_labels(specification), ["Image Patches", "Transformer Encoder"]) def test_candidate_review_allows_evidence_backed_optional_panel_and_symbol_labels(self): with tempfile.TemporaryDirectory() as temp: @@ -427,6 +487,50 @@ def topology(_path, _prompt): prompt = compile_image_prompt(plan, {"aspect_ratio": "3:2", "language": "English"}) self.assertIn("source -> modality -> method chain", prompt) + def test_focused_topology_arbitrates_false_general_connector_claim(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.png" + blueprint = root / "blueprint.png" + Image.new("RGB", (1536, 1024), "white").save(candidate) + Image.new("RGB", (1536, 1024), "white").save(blueprint) + plan = _plan() + plan["figure_specification"]["topology"] = "linear" + plan["review_ground_truth"] = { + "strict_evidence_required": True, + "priorities": [{"id": "claim", "kind": "central_claim", "label": "Parser feeds graph", "required": True, "evidence_ids": ["E1"], "evidence": [{"evidence_id": "E1", "text": "Parser feeds graph."}]}], + } + + def critic(path, template, prompt): + result = _passing_critic(path, template, prompt) + result["scientific"].update({"invented_items": ["Invented shortcut arrow from Document Parser directly to output"], "score": 0.0, "passed": False}) + result["paper_alignment"] = { + "priorities": [{"id": "claim", "status": "conflicting", "visible_evidence": "A direct arrow bypasses the Evidence Graph Builder", "evidence_ids": ["E1"]}], + "missing_priorities": [], + "evidence_conflicts": ["The shortcut arrow bypasses Evidence Graph Builder"], + "unsupported_visual_claims": ["Direct connection from Document Parser to output"], + "repair": ["Remove the shortcut arrow"], + "score": 0.0, + "passed": False, + } + result["hard_errors"] = ["Invented relation: shortcut arrow bypasses Evidence Graph Builder"] + return result + + def topology(path, prompt): + return { + "relations": [{"source": "parser", "target": "graph", "status": "present", "visible_evidence": "Visible arrow enters graph"}], + "missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": [], + "repair": [], "repair_regions": [], "score": 1.0, "passed": True, + } + + review = review_candidate(candidate, blueprint, plan, {"template_id": "linear", "forbidden_copy_terms": []}, mode="vlm", ocr_engine="off", critic_adapter=critic, topology_adapter=topology) + + self.assertTrue(review["production_pass"]) + self.assertTrue(review["cross_critic_reconciliation"]["applied"]) + self.assertEqual(review["scientific"]["invented_items"], []) + self.assertEqual(review["paper_alignment"]["evidence_conflicts"], []) + self.assertEqual(review["paper_alignment"]["priorities"][0]["status"], "supported") + def test_visual_role_contract_merges_duplicate_innovation_with_existing_node(self): spec = { "inputs": [{"id": "image", "name": "Input Image", "role": "input"}], @@ -1371,6 +1475,89 @@ def critic(path, blueprint, prompt): self.assertTrue(resumed) self.assertTrue(all(item["mode"] == "existing_repair_resume" for item in resumed)) + @patch("rfs.paper_to_image.generator.review_candidate") + def test_transient_inconclusive_vlm_review_is_retried_without_regenerating_image(self, review_mock): + base = { + "basic": {"passed": True}, + "ocr": {"score": 1.0, "passed": True}, + "scientific": {"score": 1.0, "passed": True}, + "paper_alignment": {"score": 1.0, "passed": True}, + "topology": {"score": 1.0, "passed": True}, + "template": {"score": 1.0, "passed": True}, + "aesthetic": {"score": 1.0, "passed": True}, + "hard_errors": [], "preserve": [], "repair": [], "remove": [], "repair_regions": [], + "production_pass": True, "overall_score": 1.0, + } + review_mock.side_effect = [ + {**base, "vlm_warning": "temporary provider timeout", "production_pass": False, "overall_score": 0.0}, + dict(base), + ] + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"RFS_PAPER_IMAGE_REVIEW_ATTEMPTS": "2"}, clear=False): + root = Path(temp) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": []} + render_layout_blueprint({**template, "source_aspect_ratio": 1.5}, root / "layout_blueprint.png", "1.5:1") + + result = generate_and_select(_plan(), {"aspect_ratio": "1.5:1"}, template, root / "layout_blueprint.png", root, asset_mode="placeholder", candidates=1, review_mode="vlm", repair_rounds=0, ocr_engine="off", require_visual_enrichment=False) + + self.assertEqual(review_mock.call_count, 2) + self.assertEqual(result["candidates"][0]["review"]["review_attempts"], 2) + self.assertEqual(result["candidates"][0]["review"]["review_attempt_history"][0]["inconclusive"], True) + + @patch("rfs.paper_to_image.generator.requests.post") + def test_provider_failures_do_not_count_as_semantic_no_improvement(self, post): + class RateLimitedResponse: + status_code = 429 + + post.return_value = RateLimitedResponse() + + def critic(path, blueprint, prompt): + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + return result + + with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "secret-value"}, clear=False): + root = Path(temp) + blueprint = root / "blueprint.png" + source = root / "source.png" + Image.new("RGB", (1536, 1024), "white").save(blueprint) + Image.new("RGB", (1536, 1024), "white").save(source) + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "3:2"}, {"template_id": "linear", "forbidden_copy_terms": []}, blueprint, root / "run", asset_mode="image2", candidates=1, image_retries=0, review_mode="vlm", repair_rounds=3, repair_source=source, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + history = json.loads((root / "run" / "repair_history.json").read_text(encoding="utf-8")) + self.assertEqual(history["completed_rounds"], 3) + self.assertEqual(history["stop_reason"], "maximum_repair_rounds_reached") + self.assertTrue(all(item["provider_failure"] for item in history["rounds"])) + self.assertEqual(post.call_count, 3) + + @patch("rfs.paper_to_image.generator._call_image2_edit") + def test_provider_read_timeout_stops_repair_loop_without_blind_second_wait(self, edit_mock): + edit_mock.side_effect = RuntimeError("HTTPSConnectionPool: Read timed out. (read timeout=120)") + + def critic(path, blueprint, prompt): + result = _passing_critic(path, blueprint, prompt) + result["ocr"].update({"missing_labels": ["Document Parser"], "score": 0.5, "passed": False}) + result["repair"] = ["Restore Document Parser"] + result["hard_errors"] = ["Document Parser is missing"] + return result + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + blueprint = root / "blueprint.png" + source = root / "source.png" + Image.new("RGB", (1536, 1024), "white").save(blueprint) + Image.new("RGB", (1536, 1024), "white").save(source) + with self.assertRaises(RuntimeError): + generate_and_select(_plan(), {"aspect_ratio": "3:2"}, {"template_id": "linear", "forbidden_copy_terms": []}, blueprint, root / "run", asset_mode="image2", candidates=1, image_retries=0, review_mode="vlm", repair_rounds=4, repair_source=source, ocr_engine="off", critic_adapter=critic, require_visual_enrichment=False) + + history = json.loads((root / "run" / "repair_history.json").read_text(encoding="utf-8")) + self.assertEqual(history["completed_rounds"], 1) + self.assertEqual(history["stop_reason"], "provider_timeout") + self.assertTrue(history["rounds"][0]["provider_timeout"]) + self.assertEqual(edit_mock.call_count, 1) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_benchmarking.py b/tests/unit/test_benchmarking.py index c3bcd3c..6055d0b 100644 --- a/tests/unit/test_benchmarking.py +++ b/tests/unit/test_benchmarking.py @@ -10,6 +10,7 @@ from rfs.evaluation.benchmarking import ( _score_planning_contract, + audit_paper_image_run, list_benchmark_cases, run_fast_benchmark_case, run_fast_benchmark_suite, @@ -228,6 +229,161 @@ def test_paper_to_image_score_uses_hard_scientific_gates(self): self.assertTrue(result["passed"]) self.assertEqual(result["metrics"]["relation_recall"], 1.0) + def test_paper_to_image_score_blocks_evidence_conflicts_and_reports_repair_metrics(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "paper_alignment": { + "score": 0.8, + "passed": False, + "priorities": [{"id": "GT_1", "status": "conflicting"}], + "missing_priorities": [], + "evidence_conflicts": [{"id": "GT_1", "evidence_ids": ["E1"], "reason": "visible claim reverses the paper statement"}], + "unsupported_visual_claims": [], + }, + "topology": {"score": 1.0, "passed": True, "missing_relations": [], "reversed_relations": [], "invented_relations": []}, + "ocr": {"score": 1.0, "missing_labels": [], "misspelled_labels": [], "forbidden_labels_found": []}, + "aesthetic": {"score": 0.9, "readability": 0.9}, + "clarity": {"score": 0.9}, + "information": {"score": 0.95}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": "candidate_01", + "candidates": [{"candidate_id": "candidate_01", "score": 0.9, "review": review}], + }), encoding="utf-8") + (run / "review_ground_truth.json").write_text(json.dumps({ + "strict_evidence_required": True, + "priorities": [{"id": "GT_1", "required": True}], + }), encoding="utf-8") + (run / "repair_history.json").write_text(json.dumps({ + "rounds": [ + {"accepted": True}, + {"accepted": False, "rejection_reasons": ["scientific score decreased from 1.0 to 0.8"]}, + ] + }), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertFalse(result["passed"]) + self.assertEqual(result["metrics"]["evidence_conflict_count"], 1) + self.assertEqual(result["metrics"]["evidence_priority_coverage"], 0.0) + self.assertEqual(result["metrics"]["accepted_repair_count"], 1) + self.assertEqual(result["metrics"]["repair_regression_rejection_count"], 1) + self.assertIn("candidate conflicts with quoted paper evidence", result["hard_failures"]) + + def test_new_review_does_not_block_supported_extra_visible_labels(self): + case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + review = { + "scientific": {"score": 1.0, "missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": []}, + "paper_alignment": {"score": 1.0, "passed": True, "priorities": [], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": []}, + "topology": {"score": 1.0, "passed": True, "missing_relations": [], "reversed_relations": [], "invented_relations": []}, + "ocr": { + "score": 1.0, + "detected_labels": ["Raw Document", "Document Encoding", "Final Prediction", "Add & Norm"], + "missing_labels": [], + "misspelled_labels": [], + "forbidden_labels_found": [], + "unexpected_labels": ["Add & Norm"], + "unsupported_unexpected_labels": [], + }, + "aesthetic": {"score": 1.0, "readability": 1.0}, + "clarity": {"score": 1.0}, + "information": {"score": 1.0}, + "stability": {"production_pass_rate": 1.0}, + "hard_errors": [], + } + (run / "candidate_review.json").write_text(json.dumps({"selected_candidate_id": "candidate_01", "requested_candidates": 1, "candidates": [{"candidate_id": "candidate_01", "production_pass": True, "score": 1.0, "review": review}]}), encoding="utf-8") + (run / "figure_specification.json").write_text(json.dumps({"required_labels": ["Raw Document", "Document Encoding", "Final Prediction"]}), encoding="utf-8") + + result = score_benchmark_case(case_dir, run) + + self.assertNotIn("unexpected visible labels outside the normalized figure contract", result["hard_failures"]) + self.assertEqual(result["metrics"]["unexpected_label_count"], 0) + + def test_audit_existing_image_compiles_ground_truth_and_writes_frozen_review(self): + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + Image.new("RGB", (800, 600), "white").save(run / "selected_image.png") + Image.new("RGB", (800, 600), "white").save(run / "layout_blueprint.png") + specification = { + "summary": "Scientific contract.", + "inputs": [{"id": "raw", "name": "Raw Document", "evidence_ids": ["E1"]}], + "modules": [{"id": "encode", "name": "Document Encoding", "evidence_ids": ["E1"]}], + "outputs": [{"id": "prediction", "name": "Final Prediction", "evidence_ids": ["E1"]}], + "relations": [{"source": "raw", "target": "encode", "type": "data_flow", "evidence_ids": ["E1"]}], + "required_labels": ["Raw Document", "Document Encoding", "Final Prediction"], + } + (run / "figure_specification.json").write_text(json.dumps(specification), encoding="utf-8") + (run / "key_evidence.json").write_text(json.dumps({ + "evidence": [{"id": "E1", "page": 2, "section_hint": "Method", "text": "The Raw Document is transformed by Document Encoding into the Final Prediction.", "confidence": 1.0, "source": "plain"}] + }), encoding="utf-8") + (run / "selected_template.json").write_text(json.dumps({"template_id": "linear", "forbidden_copy_terms": []}), encoding="utf-8") + (run / "preferences.json").write_text(json.dumps({"aspect_ratio": "4:3"}), encoding="utf-8") + (run / "generation_parameters.json").write_text(json.dumps({"asset_mode": "placeholder"}), encoding="utf-8") + (run / "candidate_review.json").write_text(json.dumps({"selected_candidate_id": "candidate_01", "candidates": []}), encoding="utf-8") + + def critic(path, blueprint, prompt): + ground_truth = json.loads((run / "review_ground_truth.json").read_text(encoding="utf-8")) + return { + "ocr": {"detected_labels": ["Raw Document", "Document Encoding", "Final Prediction"], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 1.0, "passed": True}, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": True, "score": 1.0, "passed": True}, + "paper_alignment": {"priorities": [{"id": item["id"], "status": "supported", "evidence_ids": item.get("evidence_ids", [])} for item in ground_truth["priorities"] if item.get("required")], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": [], "score": 1.0, "passed": True}, + "template": {"copied_reference_content": [], "score": 0.9, "passed": True}, + "aesthetic": {"visual_information_density": 0.9, "mechanism_visualization": 0.9, "publication_polish": 0.9, "score": 0.9, "passed": True}, + "preserve": [], "repair": [], "remove": [], "repair_regions": [], "hard_errors": [], + } + + result = audit_paper_image_run(run, critic_adapter=critic) + + self.assertTrue(result["production_pass"]) + self.assertEqual(result["required_priority_count"], result["grounded_required_priority_count"]) + self.assertTrue((run / "review_ground_truth.json").exists()) + self.assertTrue((run / "benchmark_review.json").exists()) + self.assertTrue((run / "paper_image_reaudit.json").exists()) + self.assertTrue((run / "reaudit_repair_plan.json").exists()) + + def test_audit_failed_run_falls_back_to_highest_scoring_recorded_candidate(self): + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + Image.new("RGB", (800, 600), "white").save(run / "layout_blueprint.png") + weaker = run / "candidate_01.png" + stronger = run / "repair_02.png" + Image.new("RGB", (800, 600), "white").save(weaker) + Image.new("RGB", (800, 600), "white").save(stronger) + (run / "figure_specification.json").write_text(json.dumps({"required_labels": ["Input"]}), encoding="utf-8") + (run / "selected_template.json").write_text(json.dumps({"template_id": "linear", "forbidden_copy_terms": []}), encoding="utf-8") + (run / "preferences.json").write_text(json.dumps({"aspect_ratio": "4:3"}), encoding="utf-8") + (run / "generation_parameters.json").write_text(json.dumps({"asset_mode": "placeholder"}), encoding="utf-8") + (run / "candidate_review.json").write_text(json.dumps({ + "selected_candidate_id": None, + "candidates": [ + {"candidate_id": "candidate_01", "path": str(weaker), "score": 0.5, "production_pass": False}, + {"candidate_id": "repair_02", "path": str(stronger), "score": 0.9, "production_pass": False}, + ], + }), encoding="utf-8") + + def critic(path, blueprint, prompt): + self.assertEqual(Path(path), stronger) + return { + "ocr": {"detected_labels": ["Input"], "missing_labels": [], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "score": 1.0, "passed": True}, + "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": [], "innovation_visible": True, "score": 1.0, "passed": True}, + "paper_alignment": {"priorities": [], "missing_priorities": [], "evidence_conflicts": [], "unsupported_visual_claims": [], "score": 1.0, "passed": True}, + "template": {"copied_reference_content": [], "score": 1.0, "passed": True}, + "aesthetic": {"visual_information_density": 1.0, "mechanism_visualization": 1.0, "publication_polish": 1.0, "score": 1.0, "passed": True}, + "hard_errors": [], "preserve": [], "repair": [], "remove": [], "repair_regions": [], + } + + result = audit_paper_image_run(run, critic_adapter=critic) + + self.assertEqual(result["candidate_id"], "repair_02") + self.assertEqual(Path(result["candidate"]), stronger) + def test_paper_to_image_score_checks_planning_contract(self): case_dir = ROOT / "benchmarks" / "paper-to-image" / "cases" / "001_linear_pipeline" with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/unit/test_review_contract.py b/tests/unit/test_review_contract.py index 62c2d11..f4429df 100644 --- a/tests/unit/test_review_contract.py +++ b/tests/unit/test_review_contract.py @@ -39,6 +39,26 @@ def test_ground_truth_embeds_page_section_and_original_excerpt(self): self.assertEqual(training["evidence"][0]["evidence_id"], "E3") self.assertEqual(sum(item["label"] == "Training loss" for item in contract["priorities"]), 1) self.assertEqual(sum(item["label"] == "Document Parser" for item in contract["priorities"]), 1) + self.assertTrue(training["required"], "must_show must keep an otherwise optional training item required") + + def test_training_and_inference_context_are_optional_unless_explicitly_visible(self): + plan = { + "figure_specification": { + "training_flow": [ + {"id": "train_context", "step": "Optimize auxiliary loss", "evidence_ids": ["E1"]}, + {"id": "train_visible", "step": "Supervised fine-tuning", "required": True, "evidence_ids": ["E1"]}, + ], + "inference_flow": [{"id": "infer_context", "step": "Run classifier", "evidence_ids": ["E1"]}], + } + } + evidence = [{"id": "E1", "page": 2, "section_hint": "Method", "text": "The model uses an auxiliary loss, supervised fine-tuning, and a classifier.", "confidence": 1.0, "source": "pymupdf"}] + + contract = compile_review_ground_truth(plan, evidence) + priorities = {item["id"]: item for item in contract["priorities"]} + + self.assertFalse(priorities["train_context"]["required"]) + self.assertTrue(priorities["train_visible"]["required"]) + self.assertFalse(priorities["infer_context"]["required"]) def test_repair_plan_prioritizes_paper_conflicts_and_carries_evidence_ids(self): review = { @@ -50,7 +70,7 @@ def test_repair_plan_prioritizes_paper_conflicts_and_carries_evidence_ids(self): }, "scientific": {"missing_modules": [], "missing_relations": [], "reversed_relations": [], "invented_items": [], "role_mismatches": [], "containment_mismatches": []}, "topology": {"missing_relations": [], "reversed_relations": [], "bypassed_relations": [], "invented_relations": []}, - "ocr": {"missing_labels": ["Document Parser"], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "unexpected_labels": []}, + "ocr": {"missing_labels": ["Document Parser"], "misspelled_labels": [], "duplicate_labels": [], "forbidden_labels_found": [], "unexpected_labels": ["Supported Detail"], "unsupported_unexpected_labels": []}, "template": {"passed": True}, "aesthetic": {"passed": True}, "basic": {"visual_enrichment_passed": True}, From be28bc30563b720454cc92cc5ee60e8bbf8f3ba9 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 08:45:05 +0800 Subject: [PATCH 72/74] Add deterministic labels and connector overlays --- README.md | 12 +- docs/paper-to-image.md | 72 ++++- rfs/evaluation/benchmarking.py | 39 ++- rfs/paper_to_image/generator.py | 142 +++++++-- rfs/paper_to_image/overlay_renderer.py | 279 ++++++++++++++++ rfs/paper_to_image/planner.py | 107 ++++--- rfs/paper_to_image/preparation.py | 2 +- rfs/paper_to_image/review_contract.py | 29 ++ rfs/paper_to_image/semantic_blueprint.py | 390 ++++++++++++++++++++--- rfs/paper_to_image/workflow.py | 25 +- tests/integration/test_paper_to_image.py | 56 +++- tests/unit/test_overlay_renderer.py | 120 +++++++ tests/unit/test_review_contract.py | 3 + tests/unit/test_semantic_blueprint.py | 61 +++- 14 files changed, 1213 insertions(+), 124 deletions(-) create mode 100644 rfs/paper_to_image/overlay_renderer.py create mode 100644 tests/unit/test_overlay_renderer.py diff --git a/README.md b/README.md index 8dfa28b..56dcbd2 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,11 @@ positive references into content-free architecture templates, selects a template renders `layout_blueprint.png`, and uses Image2 edit to create and review three candidates. For evidence-grounded contracts with 2-16 visible nodes, a generic semantic compiler now writes ranked node boxes, distinct input/output ports, -orthogonal paths, branch lanes, and outer feedback loops into -`layout_blueprint.json.semantic_plan`; the rendered blueprint embeds the exact -paper labels and connector geometry. Self-Refine feedback and SAM-style dense +global obstacle-aware orthogonal paths, branch lanes, and outer feedback loops +into `layout_blueprint.json.semantic_plan`. Image2 receives a separate +`visual_substrate_blueprint.png` with no text or arrows. The compiler then adds +exact paper labels and directed connectors to `candidate_XX_raw.png`, producing +the reviewed `candidate_XX.png` plus an auditable overlay report. Self-Refine feedback and SAM-style dense multi-frame contracts retain stricter specialized layouts. It never invokes the PPTX compiler. @@ -187,7 +189,9 @@ rfs paper-to-image ` The main outputs are `paper_review.json`, `review_coverage_report.json`, `domain_profile.json`, `template_profiles/`, `selected_template.json`, -`layout_blueprint.png`, `layout_blueprint.json`, `figure_specification.json`, `image_prompt.txt`, +`layout_blueprint.png`, `layout_blueprint.json`, `visual_substrate_blueprint.png`, +`visual_substrate_blueprint.json`, `overlay_spec.json`, `overlays/`, +`figure_specification.json`, `image_prompt.txt`, `image2_request_manifest.json`, the general and focused-topology critic reports, the `candidates/` directory, and production-only `selected_image.png`. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index 0e5501c..e752ce3 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -1,6 +1,6 @@ # Summary -`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a content-free architecture blueprint, reference-conditioned Image2 edit generation, and four production quality gates. +`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a paper-grounded semantic blueprint, reference-conditioned Image2 visual generation, deterministic exact-label/connector composition, and the full production quality gates. ## Three-Minute Fast Contract @@ -127,7 +127,10 @@ rfs paper-to-image ` `--repair-source` skips fresh initial candidate generation. The source is first reviewed against the current exact-label and scientific contract. If it fails, the best source is edited once and reviewed again. The request manifest marks -the reused source separately from the real Image2 repair request. +the reused source separately from the real Image2 repair request. This remains a +compatibility route for older images whose labels and arrows are already baked +in, so deterministic overlay mode is disabled for an explicit external repair +source. Re-audit an existing complete or failed run without regenerating an image: @@ -177,9 +180,62 @@ density, palette, and forbidden copied content. Automatic selection uses module count, loop/tree/parallel-branch structure, multimodality, retrieval structure, and requested ratio. -The selected profile is rendered as `layout_blueprint.png` and is the only image -supplied to Image2 edit for initial candidates. Reference-derived profiles remain -free of copied reference text and reference-specific objects. +The selected profile is rendered as `layout_blueprint.png` for review. When a +semantic plan is available, Image2 receives `visual_substrate_blueprint.png` +instead. That guide contains the exact node and panel geometry plus reserved +header bands, but no scientific text and no connectors. Reference-derived +profiles remain free of copied reference text and reference-specific objects. + +## Deterministic Structure Overlay + +New semantic-plan runs enable deterministic overlay mode by default: + +1. Image2 generates `candidates/candidate_XX_raw.png`, containing only the + image-rich visual substrate. +2. `overlay_spec.json` supplies exact paper terminology and declared directed + relations. +3. `layout_blueprint.json.semantic_plan` supplies normalized node boxes and + connector `path_percent` geometry. +4. The compiler writes `candidates/candidate_XX.png` and + `overlays/candidate_XX_overlay.json`. +5. OCR, scientific, topology, template, and aesthetic critics review the + composed candidate, not the raw substrate. + +Image2 prompts explicitly forbid words, letters, numbers, formulas, arrows, +lines, loops, and pseudo-text. Exact labels and relation geometry are therefore +owned by the deterministic compiler. The same semantic structure can later be +emitted as editable PowerPoint labels and connectors; the raster overlay exists +so the current whole-image review path scans the same final appearance. + +The connector compiler uses global candidate routing rather than independent +per-edge elbows. It sorts fan-in/fan-out ports by geometry, reserves distinct +lanes, routes skip-layer and feedback edges through outside channels, and scores +every candidate against node obstacles, crossings, near-collinear shared +segments, bend count, and path length. `semantic_plan.route_quality` records: + +- `crossing_count` +- `shared_segment_overlap` +- `node_obstacle_intersections` +- per-connector `lane_assignments` +- the routing algorithm and pass state + +Overlay compilation is a hard contract step. Missing label targets, missing +connector geometry, or incomplete paths fail as `failure_stage=overlay_compilation` +and are not misreported as Image2 provider failures. + +Repair ownership is also separated. OCR, spelling, exact-label, arrow, and +topology findings are assigned to `overlay_compiler`; visual density, mechanism +illustration, hierarchy, balance, and substrate aesthetics remain assigned to +`image2_visual_substrate`. Image2 repair prompts never ask the model to modify +text or connectors in deterministic overlay mode. If only compiler-owned issues +remain, the repair loop stops with `overlay_recompile_required` instead of +waiting on an unnecessary image-provider edit. + +`--resume-candidates` prefers the raw/composed pair. A raw candidate is +re-overlaid and re-reviewed using the current semantic routes. A legacy run that +contains only the composed PNG is reviewed without applying the overlay twice; +if it later needs an Image2 substrate repair, the run stops with +`overlay_raw_source_missing` and must regenerate the missing raw candidate. For supported normalized topologies (`linear`, `branch`, `multimodal`, `feedback`, and `dense_multiframe`), contracts with 2-16 visible entities are @@ -193,6 +249,9 @@ generic compiler: - moves source-only conditioning nodes immediately before their target; - routes skip-layer edges around intermediate nodes instead of through them; - routes backward/feedback edges through an outer canvas lane; +- avoids connector/node intersections and near-overlapping long connector lanes; +- assigns fan-in and fan-out ports in geometric order to prevent avoidable crossings; +- records global route quality and per-connector lane ownership; - refuses to force more than 16 visible nodes into one raster guide. When the contract matches a stricter built-in topology, the renderer uses a @@ -351,7 +410,8 @@ rfs paper-to-image ` --resume-candidates ``` -Existing `candidate_XX.png` files are re-reviewed with the current gates and +Existing raw candidates are re-composed and re-reviewed with the current gates; +legacy composed-only candidates are re-reviewed without duplicate overlay, and only missing candidates are generated. A three-seed stability run also permits one bounded replacement for a provider-failed seed; successful candidates are never regenerated by that replacement path. diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 1ab8ca4..4aaca5e 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -403,6 +403,16 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, expected = _load(case_root / str(case.get("expected_semantics") or "expected_semantics.json")) candidate_report = _load(run / "candidate_review.json") review = _select_candidate_review(run) + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict)] + selected_candidate_id = candidate_report.get("selected_candidate_id") + selected_candidate_record = next((item for item in candidate_records if item.get("candidate_id") == selected_candidate_id), None) + if selected_candidate_record is None and candidate_records: + selected_candidate_record = max(candidate_records, key=lambda item: float(item.get("score") or 0.0)) + deterministic_overlay = bool(candidate_report.get("deterministic_overlay")) + overlay_report = selected_candidate_record.get("overlay_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("overlay_report"), dict) else {} + layout_blueprint = _load(run / "layout_blueprint.json") + semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} + route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} benchmark_review = _load(run / "benchmark_review.json") if benchmark_review: review = {**review, **benchmark_review} @@ -510,6 +520,12 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, hard_failures.append("paper planning missed required benchmark relations") if planning_contract.get("forbidden_labels_found"): hard_failures.append("paper planning introduced forbidden benchmark labels") + if deterministic_overlay and not overlay_report: + hard_failures.append("deterministic overlay report is missing for the selected candidate") + if deterministic_overlay and overlay_report and not overlay_report.get("passed", False): + hard_failures.append("deterministic overlay did not compile every required label and connector") + if deterministic_overlay and route_quality and not route_quality.get("passed", False): + hard_failures.append("deterministic connector routing quality gate failed") metrics = { "entity_recall": entity_recall, @@ -536,6 +552,12 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "plan_entity_recall": planning_contract.get("entity_recall"), "plan_relation_recall": planning_contract.get("relation_recall"), "plan_forbidden_content_count": len(planning_contract.get("forbidden_labels_found", [])), + "deterministic_overlay": deterministic_overlay, + "overlay_label_count": overlay_report.get("label_count") if overlay_report else None, + "overlay_connector_count": overlay_report.get("connector_count") if overlay_report else None, + "route_crossing_count": route_quality.get("crossing_count") if route_quality else None, + "route_shared_segment_overlap": route_quality.get("shared_segment_overlap") if route_quality else None, + "route_node_obstacle_intersections": route_quality.get("node_obstacle_intersections") if route_quality else None, } thresholds = case.get("thresholds", {}) threshold_failures = _threshold_failures(metrics, thresholds) @@ -551,6 +573,12 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "run_dir": str(run), "metrics": metrics, "planning_contract": planning_contract, + "deterministic_overlay": { + "enabled": deterministic_overlay, + "selected_candidate_id": selected_candidate_record.get("candidate_id") if isinstance(selected_candidate_record, dict) else None, + "overlay_report": overlay_report, + "route_quality": route_quality, + }, "paper_alignment": paper_alignment, "repair_history": repair_history, "stability_audit": stability_audit, @@ -602,12 +630,12 @@ def audit_paper_image_run( candidate_report = _load(run / "candidate_review.json") selected_id = str(candidate_report.get("selected_candidate_id") or "").strip() + candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict) and str(item.get("path") or "").strip()] + selected_record = next((item for item in candidate_records if str(item.get("candidate_id")) == selected_id), None) candidate = Path(candidate_image).resolve() if candidate_image else run / "selected_image.png" if candidate_image: selected_id = "explicit_reaudit_candidate" elif not candidate.exists(): - candidate_records = [item for item in candidate_report.get("candidates", []) if isinstance(item, dict) and str(item.get("path") or "").strip()] - selected_record = next((item for item in candidate_records if str(item.get("candidate_id")) == selected_id), None) if selected_record is None and candidate_records: selected_record = max(candidate_records, key=lambda item: (bool(item.get("production_pass")), float(item.get("score") or 0.0))) if selected_record is None: @@ -628,6 +656,10 @@ def audit_paper_image_run( if value and str(value) != "auto" ] generation_parameters = _load(run / "generation_parameters.json") + layout_blueprint = _load(run / "layout_blueprint.json") + semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} + route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} + overlay_report = selected_record.get("overlay_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("overlay_report"), dict) else {} review = review_candidate( candidate, blueprint, @@ -663,6 +695,9 @@ def audit_paper_image_run( "overall_score": review.get("overall_score"), "paper_alignment_score": review.get("paper_alignment", {}).get("score"), "topology_score": review.get("topology", {}).get("score"), + "deterministic_overlay": bool(candidate_report.get("deterministic_overlay")), + "overlay_report": overlay_report, + "route_quality": route_quality, "repair_item_count": repair_plan.get("item_count"), "benchmark": benchmark, } diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index aad6e9d..a677e4d 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -17,6 +17,7 @@ from ..asset_generator import _call_gemini from ..utils import ensure_dir, write_json, write_text from .critics import aggregate_reviews, review_candidate +from .overlay_renderer import compose_semantic_overlay from .planner import collect_visual_relations, compile_image_prompt from .review_contract import build_repair_plan @@ -250,11 +251,31 @@ def _generate_one(prompt: str, plan: dict, blueprint: Path, path: Path, aspect_r return _call_image2_edit(blueprint, prompt, path, aspect_ratio, model, retries) -def _repair_prompt(review: dict, repair_plan: dict, base_prompt: str, plan: dict) -> str: +def _repair_prompt( + review: dict, + repair_plan: dict, + base_prompt: str, + plan: dict, + *, + deterministic_overlay: bool = False, +) -> str: connector_checklist = "\n".join( f"{index}. {item['source_label']} -> {item['target_label']} [{item['type']}]" for index, item in enumerate(collect_visual_relations(plan.get("figure_specification", {})), start=1) ) + if deterministic_overlay: + ownership_rules = """Deterministic overlay ownership: +- Do not add, remove, rewrite, move, or repair any text, letter, number, formula, arrow, connector, arrowhead, loop, or line. +- Exact labels and directed relations are compiled after this edit from overlay_spec.json and semantic_plan.path_percent. +- Repair only the image-rich visual substrate inside the existing node regions. +- Keep every reserved header band and connector corridor visually clear.""" + connector_contract = "The overlay compiler will restore the exact connector contract after this visual-only repair." + else: + ownership_rules = "Preserve all correct scientific text and connector geometry while applying the localized repair." + connector_contract = f"""Mandatory directed connector checklist after repair: +{connector_checklist} + +Preserve every currently correct connector. Add or redirect only the connectors named by the repair findings. The repaired image must contain every checklist edge exactly in the stated direction. Never draw a shortcut that bypasses any declared intermediate module.""" return f""" # Summary @@ -275,16 +296,31 @@ def _repair_prompt(review: dict, repair_plan: dict, base_prompt: str, plan: dict Hard errors: {json.dumps(review.get('hard_errors', []), ensure_ascii=False)} -Mandatory directed connector checklist after repair: -{connector_checklist} +{ownership_rules} -Preserve every currently correct connector. Add or redirect only the connectors named by the repair findings. The repaired image must contain every checklist edge exactly in the stated direction. Never draw a shortcut that bypasses any declared intermediate module. +{connector_contract} Original scientific and visual contract: {base_prompt} """.strip() +def _failure_metadata(error: Exception) -> dict[str, Any]: + error_text = str(error) + normalized = error_text.casefold() + overlay_failure = "deterministic overlay compilation failed" in normalized + provider_timeout = not overlay_failure and any(token in normalized for token in ("timed out", "read timeout", "connect timeout")) + provider_failure = not overlay_failure and ( + provider_timeout or any(token in normalized for token in ("http 429", "http 5", "provider", "connection")) + ) + return { + "error": error_text, + "failure_stage": "overlay_compilation" if overlay_failure else "image_provider" if provider_failure else "generation_or_review", + "provider_failure": provider_failure, + "provider_timeout": provider_timeout, + } + + def _review_regressions(source: dict, candidate: dict, tolerance: float = 0.001) -> list[str]: regressions = [] for section in ("basic", "ocr", "scientific", "paper_alignment", "topology", "template", "aesthetic"): @@ -339,12 +375,19 @@ def generate_and_select( topology_adapter: Callable | None = None, resume_candidates: bool = False, require_visual_enrichment: bool = True, + generation_blueprint_path: str | Path | None = None, + overlay_spec: dict[str, Any] | None = None, + deterministic_overlay: bool = False, ) -> dict: root = ensure_dir(out_dir) candidate_dir = ensure_dir(root / "candidates") prompt_dir = ensure_dir(root / "prompts") review_dir = ensure_dir(root / "reviews") + overlay_dir = ensure_dir(root / "overlays") blueprint = Path(blueprint_path) + generation_blueprint = Path(generation_blueprint_path) if generation_blueprint_path else blueprint + semantic_plan = selected_template.get("semantic_plan", {}) if isinstance(selected_template.get("semantic_plan"), dict) else {} + overlay_enabled = bool(deterministic_overlay and semantic_plan.get("applied") and isinstance(overlay_spec, dict)) aspect_ratio = str(preferences.get("aspect_ratio") or "16:9") count = max(1, min(4, int(candidates))) records: list[dict] = [] @@ -414,21 +457,41 @@ def run_review(path: Path) -> dict[str, Any]: if repair_source_path and not repair_source_path.exists(): raise FileNotFoundError(f"Repair source image does not exist: {repair_source_path}") + def apply_overlay(raw_path: Path, composed_path: Path, report_path: Path) -> dict[str, Any]: + try: + return compose_semantic_overlay(raw_path, semantic_plan, overlay_spec or {}, composed_path, report_path=report_path) + except Exception as exc: + raise RuntimeError(f"Deterministic overlay compilation failed: {exc}") from exc + def generate_candidate(index: int) -> dict[str, Any]: candidate_id = f"candidate_{index:02d}" path = candidate_dir / f"{candidate_id}.png" - prompt = compile_image_prompt(plan, preferences, candidate_variant=index, selected_template=selected_template) + raw_path = candidate_dir / f"{candidate_id}_raw.png" if overlay_enabled else path + overlay_report_path = overlay_dir / f"{candidate_id}_overlay.json" + prompt = compile_image_prompt(plan, preferences, candidate_variant=index, selected_template=selected_template, deterministic_overlay=overlay_enabled) prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" write_text(prompt_path, prompt) started = time.monotonic() try: - if resume_candidates and path.exists(): - generation = {"mode": "existing_candidate_resume", "source": str(path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + overlay_report = None + record_raw_path: Path | None = raw_path if overlay_enabled else None + if resume_candidates and raw_path.exists(): + generation = {"mode": "existing_candidate_resume", "source": str(raw_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + generation_seconds = 0.0 + elif resume_candidates and path.exists(): + generation = {"mode": "existing_candidate_resume_legacy_composed", "source": str(path), "overlay_assumed_precomposed": overlay_enabled, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + raw_path = path + record_raw_path = None + if overlay_report_path.exists(): + overlay_report = json.loads(overlay_report_path.read_text(encoding="utf-8")) generation_seconds = 0.0 else: generation_started = time.monotonic() - generation = _generate_one(prompt, plan, blueprint, path, aspect_ratio, asset_mode, image_model, image_retries) + generation = _generate_one(prompt, plan, generation_blueprint, raw_path, aspect_ratio, asset_mode, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) + if overlay_enabled and raw_path != path: + overlay_report = apply_overlay(raw_path, path, overlay_report_path) + generation = {**generation, "raw_path": str(raw_path), "composed_path": str(path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True} review_started = time.monotonic() review = run_review(path) review_seconds = round(time.monotonic() - review_started, 3) @@ -436,6 +499,8 @@ def generate_candidate(index: int) -> dict[str, Any]: "record": { "candidate_id": candidate_id, "path": str(path), + "raw_path": str(record_raw_path) if record_raw_path else None, + "overlay_report": overlay_report, "prompt_path": str(prompt_path), "generation": generation, "review": review, @@ -450,7 +515,7 @@ def generate_candidate(index: int) -> dict[str, Any]: "request": {"candidate_id": candidate_id, "generation_seconds": generation_seconds, **generation}, } except Exception as exc: - return {"failure": {"candidate_id": candidate_id, "error": str(exc), "prompt_path": str(prompt_path), "elapsed_seconds": round(time.monotonic() - started, 3)}} + return {"failure": {"candidate_id": candidate_id, "prompt_path": str(prompt_path), "elapsed_seconds": round(time.monotonic() - started, 3), **_failure_metadata(exc)}} if repair_source_path: candidate_workers = 1 @@ -468,6 +533,7 @@ def generate_candidate(index: int) -> dict[str, Any]: records.append({ "candidate_id": candidate_id, "path": str(path), + "raw_path": None, "prompt_path": str(prompt_path), "generation": generation, "review": review, @@ -541,21 +607,49 @@ def generate_candidate(index: int) -> dict[str, Any]: source = max(accepted_records, key=lambda item: item["score"]) repaired_id = f"repair_{repair_index + 1:02d}" repaired_path = candidate_dir / f"{repaired_id}.png" + repaired_raw_path = candidate_dir / f"{repaired_id}_raw.png" if overlay_enabled else repaired_path + overlay_report_path = overlay_dir / f"{repaired_id}_overlay.json" prompt_path = prompt_dir / f"{repaired_id}_prompt.txt" repair_plan_path = prompt_dir / f"{repaired_id}_plan.json" review_path = review_dir / f"{repaired_id}_review.json" repair_plan = build_repair_plan(source["review"], source["candidate_id"], repair_index + 1) + if overlay_enabled: + all_items = list(repair_plan.get("items", [])) + overlay_owned = [item for item in all_items if item.get("owner") == "overlay_compiler"] + image_owned = [item for item in all_items if item.get("owner") != "overlay_compiler"] + repair_plan = { + **repair_plan, + "summary": "Ownership-aware repair plan: Image2 edits only the visual substrate; exact text and connectors remain compiler-owned.", + "all_items": all_items, + "overlay_owned_items": overlay_owned, + "overlay_owned_item_count": len(overlay_owned), + "image2_item_count": len(image_owned), + "items": image_owned, + "item_count": len(image_owned), + } write_json(repair_plan_path, repair_plan) write_json(root / "repair_plan.json", repair_plan) if not repair_plan["items"]: - repair_stop_reason = "no_actionable_repairs" + repair_stop_reason = "overlay_recompile_required" if repair_plan.get("overlay_owned_item_count") else "no_actionable_repairs" + break + if overlay_enabled and not source.get("raw_path"): + repair_stop_reason = "overlay_raw_source_missing" break base_prompt = Path(source["prompt_path"]).read_text(encoding="utf-8") - prompt = _repair_prompt(source["review"], repair_plan, base_prompt, plan) + prompt = _repair_prompt(source["review"], repair_plan, base_prompt, plan, deterministic_overlay=overlay_enabled) write_text(prompt_path, prompt) repair_started = time.monotonic() try: - if resume_candidates and repaired_path.exists() and review_path.exists(): + overlay_report = None + if overlay_enabled and resume_candidates and repaired_raw_path.exists(): + overlay_report = apply_overlay(repaired_raw_path, repaired_path, overlay_report_path) + generation = {"mode": "existing_repair_reoverlay", "source": str(repaired_raw_path), "raw_path": str(repaired_raw_path), "composed_path": str(repaired_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + generation_seconds = 0.0 + review_started = time.monotonic() + review = run_review(repaired_path) + review_seconds = round(time.monotonic() - review_started, 3) + write_json(review_path, review) + elif resume_candidates and repaired_path.exists() and review_path.exists() and not overlay_enabled: stored_review = json.loads(review_path.read_text(encoding="utf-8")) generation_seconds = 0.0 if review_is_inconclusive(stored_review): @@ -570,8 +664,12 @@ def generate_candidate(index: int) -> dict[str, Any]: review_seconds = 0.0 else: generation_started = time.monotonic() - generation = _call_image2_edit(Path(source["path"]), prompt, repaired_path, aspect_ratio, image_model, image_retries) + source_visual = Path(str(source.get("raw_path") or source["path"])) + generation = _call_image2_edit(source_visual, prompt, repaired_raw_path, aspect_ratio, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) + if overlay_enabled and repaired_raw_path != repaired_path: + overlay_report = apply_overlay(repaired_raw_path, repaired_path, overlay_report_path) + generation = {**generation, "raw_path": str(repaired_raw_path), "composed_path": str(repaired_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True} review_started = time.monotonic() review = run_review(repaired_path) review_seconds = round(time.monotonic() - review_started, 3) @@ -581,6 +679,8 @@ def generate_candidate(index: int) -> dict[str, Any]: repaired = { "candidate_id": repaired_id, "path": str(repaired_path), + "raw_path": str(repaired_raw_path) if overlay_enabled else None, + "overlay_report": overlay_report, "prompt_path": str(prompt_path), "repair_plan_path": str(repair_plan_path), "review_path": str(review_path), @@ -618,15 +718,15 @@ def generate_candidate(index: int) -> dict[str, Any]: repair_stop_reason = "two_consecutive_rounds_without_improvement" break except Exception as exc: - error_text = str(exc) - normalized_error = error_text.casefold() - provider_timeout = any(token in normalized_error for token in ("timed out", "read timeout", "connect timeout")) - provider_failure = provider_timeout or any(token in normalized_error for token in ("http 429", "http 5", "provider", "connection")) + metadata = _failure_metadata(exc) + error_text = str(metadata["error"]) + provider_timeout = bool(metadata["provider_timeout"]) + provider_failure = bool(metadata["provider_failure"]) if not provider_failure: no_improvement_streak += 1 - failure = {"candidate_id": repaired_id, "error": error_text, "prompt_path": str(prompt_path), "repair_source": source["candidate_id"], "provider_failure": provider_failure, "provider_timeout": provider_timeout} + failure = {"candidate_id": repaired_id, "prompt_path": str(prompt_path), "repair_source": source["candidate_id"], **metadata} failures.append(failure) - repair_history.append({"round": repair_index + 1, "candidate_id": repaired_id, "source_candidate_id": source["candidate_id"], "accepted": False, "provider_failure": provider_failure, "provider_timeout": provider_timeout, "error": error_text}) + repair_history.append({"round": repair_index + 1, "candidate_id": repaired_id, "source_candidate_id": source["candidate_id"], "accepted": False, **metadata}) if provider_timeout: repair_stop_reason = "provider_timeout" break @@ -659,7 +759,7 @@ def generate_candidate(index: int) -> dict[str, Any]: aggregates = aggregate_reviews(records) for name, data in aggregates.items(): write_json(root / f"{name}.json", data) - request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "candidate_workers": candidate_workers, "repair_source": str(repair_source_path) if repair_source_path else None, "requests": requests_manifest, "failures": failures} + request_manifest = {"summary": "Safe Image2 request manifest; secrets are never recorded.", "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY")), "model": _image2_model_name(image_model) if asset_mode == "image2" else image_model, "edit_endpoint": _safe_endpoint(_image2_edit_url()) if asset_mode == "image2" and (os.getenv("API_BASE") or os.getenv("RFS_IMAGE_EDIT_URL")) else None, "blueprint": str(blueprint), "generation_blueprint": str(generation_blueprint), "deterministic_overlay": overlay_enabled, "candidate_workers": candidate_workers, "repair_source": str(repair_source_path) if repair_source_path else None, "requests": requests_manifest, "failures": failures} write_json(root / "image2_request_manifest.json", request_manifest) selected_path = None @@ -676,7 +776,7 @@ def generate_candidate(index: int) -> dict[str, Any]: stability = _build_stability_report(records, failures, count, repair_source_path) write_json(root / "stability_report.json", stability) - report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "resume_candidates": bool(resume_candidates), "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "repair_stop_reason": repair_stop_reason, "failures": failures, "candidates": records, "stability": stability, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} + report = {"summary": "Generated and reviewed paper-to-image candidates.", "asset_mode": asset_mode, "review_mode": review_mode, "production_eligible": asset_mode == "image2", "deterministic_overlay": overlay_enabled, "requested_candidates": 0 if repair_source_path else count, "candidate_workers": candidate_workers, "resume_candidates": bool(resume_candidates), "repair_source": str(repair_source_path) if repair_source_path else None, "successful_candidates": len(records), "repair_candidates": len(repair_records), "repair_stop_reason": repair_stop_reason, "failures": failures, "candidates": records, "stability": stability, "selected_candidate_id": selected["candidate_id"] if selected_path and selected else None, "selected_image": str(selected_path) if selected_path else None, "engineering_preview": str(engineering_preview) if engineering_preview else None, "selected_passed_all_checks": bool(selected_path), "delivery_blocked": asset_mode == "image2" and not selected_path} write_json(root / "candidate_review.json", report) if report["delivery_blocked"]: raise RuntimeError("No Image2 candidate passed paper alignment, scientific, OCR, topology, template, and aesthetic production gates") diff --git a/rfs/paper_to_image/overlay_renderer.py b/rfs/paper_to_image/overlay_renderer.py new file mode 100644 index 0000000..415f0bc --- /dev/null +++ b/rfs/paper_to_image/overlay_renderer.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import hashlib +import math +from pathlib import Path +from typing import Any + +from PIL import Image, ImageDraw, ImageFont + +from ..utils import write_json + + +_NODE_COLORS = { + "inputs": ((237, 245, 252, 235), (42, 99, 161, 255), (36, 75, 114, 255)), + "modules": ((237, 249, 247, 235), (31, 117, 126, 255), (25, 91, 87, 255)), + "outputs": ((242, 248, 243, 235), (66, 126, 80, 255), (49, 95, 59, 255)), + "innovations": ((252, 244, 235, 235), (177, 92, 33, 255), (140, 72, 26, 255)), +} + +_CONNECTOR_COLORS = { + "feedback_loop": (177, 92, 33, 255), + "iteration": (177, 92, 33, 255), + "iterate": (177, 92, 33, 255), + "return_flow": (177, 92, 33, 255), + "branch": (98, 84, 147, 255), + "conditioning": (31, 117, 126, 255), + "feature_flow": (42, 99, 161, 255), + "prediction": (66, 126, 80, 255), + "data_flow": (80, 111, 134, 255), +} + + +def _font(size: int, *, bold: bool = False) -> ImageFont.ImageFont: + names = [ + r"C:\Windows\Fonts\arialbd.ttf" if bold else r"C:\Windows\Fonts\arial.ttf", + r"C:\Windows\Fonts\msyhbd.ttc" if bold else r"C:\Windows\Fonts\msyh.ttc", + "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", + ] + for name in names: + try: + return ImageFont.truetype(name, max(10, int(size))) + except OSError: + continue + return ImageFont.load_default() + + +def _bbox_pixels(bbox: dict[str, Any], width: int, height: int) -> tuple[int, int, int, int]: + x = float(bbox.get("x") or 0.0) + y = float(bbox.get("y") or 0.0) + w = float(bbox.get("w") or 0.0) + h = float(bbox.get("h") or 0.0) + return ( + round(x * width), + round(y * height), + round((x + w) * width), + round((y + h) * height), + ) + + +def _wrap(draw: ImageDraw.ImageDraw, text: str, font: ImageFont.ImageFont, max_width: int, max_lines: int = 3) -> list[str]: + tokens = text.split() if " " in text.strip() else list(text.strip()) + separator = " " if " " in text.strip() else "" + lines: list[str] = [] + current = "" + for token in tokens: + candidate = token if not current else f"{current}{separator}{token}" + if not current or draw.textlength(candidate, font=font) <= max_width: + current = candidate + else: + lines.append(current) + current = token + if current: + lines.append(current) + if len(lines) <= max_lines: + return lines + merged = lines[: max_lines - 1] + merged.append(separator.join(lines[max_lines - 1 :])) + return merged + + +def _fit_text(draw: ImageDraw.ImageDraw, text: str, box: tuple[int, int, int, int], *, bold: bool = True) -> tuple[ImageFont.ImageFont, list[str]]: + max_width = max(20, box[2] - box[0] - 14) + max_height = max(18, box[3] - box[1] - 8) + starting = max(12, min(28, round(max_height * 0.42))) + for size in range(starting, 9, -1): + font = _font(size, bold=bold) + lines = _wrap(draw, text, font, max_width) + rendered = "\n".join(lines) + bounds = draw.multiline_textbbox((0, 0), rendered, font=font, spacing=2, align="center") + if bounds[2] - bounds[0] <= max_width and bounds[3] - bounds[1] <= max_height: + return font, lines + return _font(10, bold=bold), _wrap(draw, text, _font(10, bold=bold), max_width) + + +def _draw_centered_text(draw: ImageDraw.ImageDraw, text: str, box: tuple[int, int, int, int], fill: tuple[int, int, int, int]) -> None: + font, lines = _fit_text(draw, text, box) + rendered = "\n".join(lines) + bounds = draw.multiline_textbbox((0, 0), rendered, font=font, spacing=2, align="center") + x = box[0] + ((box[2] - box[0]) - (bounds[2] - bounds[0])) / 2 + y = box[1] + ((box[3] - box[1]) - (bounds[3] - bounds[1])) / 2 - bounds[1] + draw.multiline_text((x, y), rendered, font=font, fill=fill, spacing=2, align="center") + + +def _draw_dashed_line(draw: ImageDraw.ImageDraw, points: list[tuple[int, int]], fill: tuple[int, int, int, int], width: int, dash: int = 12, gap: int = 8) -> None: + for start, end in zip(points[:-1], points[1:]): + dx, dy = end[0] - start[0], end[1] - start[1] + length = math.hypot(dx, dy) + if length <= 0: + continue + distance = 0.0 + while distance < length: + stop = min(length, distance + dash) + x0 = start[0] + dx * distance / length + y0 = start[1] + dy * distance / length + x1 = start[0] + dx * stop / length + y1 = start[1] + dy * stop / length + draw.line((x0, y0, x1, y1), fill=fill, width=width) + distance += dash + gap + + +def _draw_arrow(draw: ImageDraw.ImageDraw, points: list[tuple[int, int]], relation_type: str, canvas_width: int) -> None: + if len(points) < 2: + return + color = _CONNECTOR_COLORS.get(relation_type, _CONNECTOR_COLORS["data_flow"]) + line_width = max(3, round(canvas_width * 0.0024)) + halo_width = line_width + max(3, round(canvas_width * 0.0022)) + draw.line(points, fill=(255, 255, 255, 225), width=halo_width, joint="curve") + dashed = relation_type in {"conditioning", "feedback_loop", "iteration", "iterate", "return_flow"} + if dashed: + _draw_dashed_line(draw, points, color, line_width, dash=max(9, line_width * 3), gap=max(6, line_width * 2)) + else: + draw.line(points, fill=color, width=line_width, joint="curve") + tail, end = points[-2], points[-1] + angle = math.atan2(end[1] - tail[1], end[0] - tail[0]) + arrow = max(10, round(canvas_width * 0.010)) + wing = 0.48 + triangle = [ + end, + (round(end[0] - arrow * math.cos(angle - wing)), round(end[1] - arrow * math.sin(angle - wing))), + (round(end[0] - arrow * math.cos(angle + wing)), round(end[1] - arrow * math.sin(angle + wing))), + ] + draw.polygon(triangle, fill=(255, 255, 255, 235)) + inner_arrow = max(7, round(arrow * 0.78)) + inner = [ + end, + (round(end[0] - inner_arrow * math.cos(angle - wing)), round(end[1] - inner_arrow * math.sin(angle - wing))), + (round(end[0] - inner_arrow * math.cos(angle + wing)), round(end[1] - inner_arrow * math.sin(angle + wing))), + ] + draw.polygon(inner, fill=color) + + +def render_visual_substrate_blueprint( + semantic_plan: dict[str, Any], + out_path: str | Path, + *, + width: int = 1536, + height: int = 1024, +) -> dict[str, Any]: + """Render geometry guides for Image2 without scientific text or connectors.""" + + if not semantic_plan.get("applied"): + raise ValueError("A deterministic visual substrate requires an applied semantic plan") + image = Image.new("RGB", (width, height), (250, 250, 249)) + draw = ImageDraw.Draw(image, "RGBA") + for panel in semantic_plan.get("panels", []): + if not isinstance(panel, dict): + continue + box = _bbox_pixels(panel.get("bbox_percent", {}), width, height) + draw.rounded_rectangle(box, radius=max(10, round(height * 0.018)), fill=(247, 249, 249, 255), outline=(104, 132, 151, 255), width=3) + for node in semantic_plan.get("nodes", []): + if not isinstance(node, dict): + continue + box = _bbox_pixels(node.get("bbox_percent", {}), width, height) + fill, outline, _ = _NODE_COLORS.get(str(node.get("field") or "modules"), _NODE_COLORS["modules"]) + draw.rounded_rectangle(box, radius=max(8, round(height * 0.014)), fill=fill, outline=outline, width=3) + header_height = max(26, round((box[3] - box[1]) * 0.30)) + header = (box[0] + 4, box[1] + 4, box[2] - 4, min(box[3] - 4, box[1] + header_height)) + draw.rounded_rectangle(header, radius=max(6, round(height * 0.008)), fill=(255, 255, 255, 185), outline=(255, 255, 255, 0), width=1) + output = Path(out_path) + output.parent.mkdir(parents=True, exist_ok=True) + image.save(output) + return { + "summary": "Visual-only Image2 substrate with semantic node geometry and reserved editable header bands.", + "path": str(output), + "width": width, + "height": height, + "node_count": len([item for item in semantic_plan.get("nodes", []) if isinstance(item, dict)]), + "panel_count": len([item for item in semantic_plan.get("panels", []) if isinstance(item, dict)]), + "contains_semantic_text": False, + "contains_connectors": False, + "render_policy": "image2_visual_substrate_only", + } + + +def compose_semantic_overlay( + source_path: str | Path, + semantic_plan: dict[str, Any], + overlay_spec: dict[str, Any], + out_path: str | Path, + *, + report_path: str | Path | None = None, +) -> dict[str, Any]: + """Compose exact paper labels and directed connectors over a visual substrate.""" + + if not semantic_plan.get("applied"): + raise ValueError("Deterministic overlay requires an applied semantic plan") + source = Path(source_path) + with Image.open(source) as raw: + image = raw.convert("RGBA") + width, height = image.size + layer = Image.new("RGBA", image.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer, "RGBA") + nodes = {str(item.get("id")): item for item in semantic_plan.get("nodes", []) if isinstance(item, dict) and item.get("id")} + connectors = [item for item in semantic_plan.get("connectors", []) if isinstance(item, dict)] + requested_labels = [item for item in overlay_spec.get("labels", []) if isinstance(item, dict)] + requested_connectors = [item for item in overlay_spec.get("connectors", []) if isinstance(item, dict)] + + connector_index = { + (str(item.get("source") or ""), str(item.get("target") or ""), str(item.get("type") or "data_flow")): item + for item in connectors + } + rendered_connectors = [] + for requested in requested_connectors: + key = (str(requested.get("source") or ""), str(requested.get("target") or ""), str(requested.get("type") or "data_flow")) + connector = connector_index.get(key) + if connector is None: + relaxed = [item for item in connectors if str(item.get("source")) == key[0] and str(item.get("target")) == key[1]] + connector = relaxed[0] if len(relaxed) == 1 else None + if connector is None: + raise ValueError(f"Overlay connector has no semantic geometry: {key[0]} -> {key[1]} [{key[2]}]") + path_pixels = [(round(float(point[0]) * width), round(float(point[1]) * height)) for point in connector.get("path_percent", [])] + if len(path_pixels) < 2: + raise ValueError(f"Overlay connector has fewer than two path points: {key[0]} -> {key[1]}") + _draw_arrow(draw, path_pixels, key[2], width) + rendered_connectors.append({ + "source": key[0], "target": key[1], "type": key[2], + "path_percent": connector.get("path_percent", []), "path_pixels": [list(point) for point in path_pixels], + }) + + rendered_labels = [] + for item in requested_labels: + target_id = str(item.get("target_id") or "") + text = str(item.get("text") or "").strip() + node = nodes.get(target_id) + if not text or node is None: + raise ValueError(f"Overlay label has no semantic node geometry: {target_id or text}") + box = _bbox_pixels(node.get("bbox_percent", {}), width, height) + fill, outline, text_color = _NODE_COLORS.get(str(node.get("field") or "modules"), _NODE_COLORS["modules"]) + draw.rounded_rectangle(box, radius=max(8, round(height * 0.014)), fill=(255, 255, 255, 0), outline=outline, width=max(2, round(width * 0.0018))) + header_height = max(30, min(round((box[3] - box[1]) * 0.42), 74)) + header = (box[0] + 4, box[1] + 4, box[2] - 4, min(box[3] - 4, box[1] + header_height)) + draw.rounded_rectangle(header, radius=max(6, round(height * 0.008)), fill=fill, outline=(255, 255, 255, 200), width=1) + _draw_centered_text(draw, text, header, text_color) + rendered_labels.append({"target_id": target_id, "text": text, "bbox_pixels": list(header), "role": item.get("role")}) + + composed = Image.alpha_composite(image, layer).convert("RGB") + output = Path(out_path) + output.parent.mkdir(parents=True, exist_ok=True) + composed.save(output) + report = { + "summary": "Exact scientific labels and directed connectors composed deterministically over the Image2 visual substrate.", + "source": str(source), + "output": str(output), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "width": width, + "height": height, + "label_count": len(rendered_labels), + "connector_count": len(rendered_connectors), + "expected_label_count": len(requested_labels), + "expected_connector_count": len(requested_connectors), + "labels": rendered_labels, + "connectors": rendered_connectors, + "text_render_policy": "deterministic_raster_overlay; exact source terminology", + "connector_render_policy": "deterministic_raster_overlay; semantic_plan_path_percent", + "passed": len(rendered_labels) == len(requested_labels) and len(rendered_connectors) == len(requested_connectors), + } + if report_path: + write_json(report_path, report) + return report diff --git a/rfs/paper_to_image/planner.py b/rfs/paper_to_image/planner.py index 9a51fa6..952ac3e 100644 --- a/rfs/paper_to_image/planner.py +++ b/rfs/paper_to_image/planner.py @@ -785,7 +785,14 @@ def _topology_specific_prompt_rules(spec: dict) -> str: return "No additional topology-specific rules. Follow the directed connector checklist exactly." -def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = 1, selected_template: dict | None = None) -> str: +def compile_image_prompt( + plan: dict, + preferences: dict, + candidate_variant: int = 1, + selected_template: dict | None = None, + *, + deterministic_overlay: bool = False, +) -> str: spec = plan["figure_specification"] design = plan["design_plan"] layout = plan["layout_intent"] @@ -985,10 +992,68 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = for item in semantic_plan.get("connectors", []) if isinstance(item, dict) ], } + if deterministic_overlay: + visible_label_section = f"""Scientific concept-to-node mapping. These terms describe node meaning only; do not render any of them as visible text in the visual substrate: +{json.dumps(labels, ensure_ascii=False)} + +Reserve a clean translucent or light header band inside every node for later deterministic text composition. Do not render letters, words, numbers, formulas, axis labels, panel IDs, legends, headings, watermarks, or pseudo-text anywhere in the image.""" + connector_section = f"""Post-composition directed connector contract. These relations will be drawn later by the deterministic overlay compiler; do not render any arrow, line, connector, arrowhead, loop, or flow path in the visual substrate: +{connector_checklist or '- No visible connector is rendered by Image2.'} + +Keep the whitespace corridors around the supplied semantic connector paths clear enough for the later overlay. Preserve node bounding boxes, but leave all relationships visually unconnected.""" + overlay_requirements = """- Generate only the image-rich visual substrate inside the supplied node geometry. +- Do not render any scientific text or any connector geometry; exact labels and arrows are owned by overlay_spec.json and semantic_plan.path_percent. +- Reserve a clean header band in each node and clear connector corridors between nodes. +- Enrich every input, core method, innovation, and output node with a paper-grounded visual cue from the role checklist. +- Use roughly 55-85% of each node interior below the reserved header band for a representative sample, mechanism inset, scientific icon, or outcome cue. +- For modality nodes, show a representative sample; for data sources, show archive cues; for method nodes, show the internal operation; for outputs, show a compact outcome cue. +- Preserve every node bounding box and layer assignment. Do not merge, split, move, rename, or add nodes. +- Do not add arrows, lines, loops, legends, headings, architecture submodule names, letters, numbers, fake UI text, formulas, axes, charts, citations, or watermarks. +- Keep all major objects complete and inside the canvas; avoid large blank margins. +- Do not invent datasets, mechanisms, metrics, modules, or results. +- Treat the supplied visual-only blueprint as a hard macro-layout guide and replace only the empty node interiors with paper-grounded visual explanation.""" + output_mode_summary = "Create one publication-quality visual substrate for a scientific framework figure. Exact labels and connectors will be composed deterministically after generation." + else: + visible_label_section = f"""Exact visible label whitelist (render these exactly and do not invent alternatives): +{json.dumps(labels, ensure_ascii=False)} + +Evidence-backed optional visible labels (allowed if useful, but not required): +{json.dumps(optional_visible_labels, ensure_ascii=False)} + +Mandatory visible label checklist: +{numbered_labels} + +Every checklist item must appear at least once as clearly readable text in the completed figure. Labels explicitly marked as repeatable may appear more than once only to show reuse of the same paper-supported component; every other label must appear exactly once. An icon, symbol, equation variable, or unlabeled output card does not satisfy a checklist item. Reserve enough width for long labels and place each output label beside or inside its output node.""" + connector_section = f"""Mandatory directed connector checklist: +{connector_checklist} + +Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows.""" + overlay_requirements = """- Preserve every paper-supported core module and relation. +- Use the exact short labels listed in terminology when labels are necessary. +- Make arrows unambiguous and follow the specified source-to-target direction. +- Emphasize the paper's actual innovation rather than generic AI decoration. +- Use a clean academic composition with dense but readable information hierarchy. +- Enrich every input, core method, innovation, and output node with a paper-grounded visual cue from the role checklist. Do not return a bare text-only flowchart. +- Use roughly 35-60% of each node interior for a representative example, mechanism inset, scientific icon, or outcome cue while keeping the exact label clearly readable. +- For modality nodes, show a representative sample of that modality; for data sources, show dataset/archive cues; for method nodes, show the internal operation; for outputs, show a compact outcome/evaluation cue. +- Preserve the supplied geometry and connectors while adding visual explanation inside the nodes; do not merely add gradients or shadows to the blueprint boxes. +- Keep all major objects complete and inside the canvas; avoid large blank margins. +- Do not invent datasets, mechanisms, formulas, metrics, modules, or results. +- Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. +- Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. +- Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. +- When semantic blueprint geometry is supplied, preserve every node bounding box, layer assignment, connector path, and arrow direction; visual decoration may change but graph geometry may not. +- Template panels are macro spatial regions, not an exact limit on the number of scientific content nodes inside those regions. +- Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. +- The exact visible label whitelist is the only permitted visible text. Do not add a legend, inset title, process verb, architecture submodule, or explanatory heading outside that whitelist. +- Render every mandatory visible label at least once; repeat only labels explicitly marked as evidence-supported shared components, and never replace an output label with an icon-only node. +- Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. +- Include explicit input-boundary and output-boundary connectors.""" + output_mode_summary = "Create one complete publication-quality scientific framework figure as a raster image." return f""" # Summary -Create one complete publication-quality scientific framework figure as a raster image. +{output_mode_summary} Candidate variant: {candidate_variant}. Scientific goal: {spec.get('figure_goal')}. @@ -1007,21 +1072,9 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = Paper-grounded semantic blueprint geometry: {json.dumps(compact_semantic_plan, ensure_ascii=False, separators=(',', ':'))} -Exact visible label whitelist (render these exactly and do not invent alternatives): -{json.dumps(labels, ensure_ascii=False)} - -Evidence-backed optional visible labels (allowed if useful, but not required): -{json.dumps(optional_visible_labels, ensure_ascii=False)} - -Mandatory visible label checklist: -{numbered_labels} - -Every checklist item must appear at least once as clearly readable text in the completed figure. Labels explicitly marked as repeatable may appear more than once only to show reuse of the same paper-supported component; every other label must appear exactly once. An icon, symbol, equation variable, or unlabeled output card does not satisfy a checklist item. Reserve enough width for long labels and place each output label beside or inside its output node. - -Mandatory directed connector checklist: -{connector_checklist} +{visible_label_section} -Every connector above must be visible with the stated direction. Do not bypass an intermediate module, do not replace two required inputs with one shortcut arrow, and do not add direct source-to-output paths that are absent from this checklist. Shared component labels may repeat as visual badges without adding extra implementation-level arrows. +{connector_section} Boundary-role anti-hallucination rules: {boundary_role_rules} @@ -1045,27 +1098,7 @@ def compile_image_prompt(plan: dict, preferences: dict, candidate_variant: int = {json.dumps(template.get('forbidden_copy_terms', []), ensure_ascii=False)} Hard requirements: -- Preserve every paper-supported core module and relation. -- Use the exact short labels listed in terminology when labels are necessary. -- Make arrows unambiguous and follow the specified source-to-target direction. -- Emphasize the paper's actual innovation rather than generic AI decoration. -- Use a clean academic composition with dense but readable information hierarchy. -- Enrich every input, core method, innovation, and output node with a paper-grounded visual cue from the role checklist. Do not return a bare text-only flowchart. -- Use roughly 35-60% of each node interior for a representative example, mechanism inset, scientific icon, or outcome cue while keeping the exact label clearly readable. -- For modality nodes, show a representative sample of that modality; for data sources, show dataset/archive cues; for method nodes, show the internal operation; for outputs, show a compact outcome/evaluation cue. -- Preserve the supplied geometry and connectors while adding visual explanation inside the nodes; do not merely add gradients or shadows to the blueprint boxes. -- Keep all major objects complete and inside the canvas; avoid large blank margins. -- Do not invent datasets, mechanisms, formulas, metrics, modules, or results. -- Do not render paragraphs, citations, fake equations, fake axes, fake charts, or invented numbers. -- Avoid commercial-poster styling, cyberpunk effects, unrelated robots, generic brains, and repeated dashboard cards. -- Treat the supplied blueprint as a hard macro-layout guide, but replace all reference content with this paper's content. -- When semantic blueprint geometry is supplied, preserve every node bounding box, layer assignment, connector path, and arrow direction; visual decoration may change but graph geometry may not. -- Template panels are macro spatial regions, not an exact limit on the number of scientific content nodes inside those regions. -- Every visible scientific word must come from the exact label whitelist; do not paraphrase labels. -- The exact visible label whitelist is the only permitted visible text. Do not add a legend, inset title, process verb, architecture submodule, or explanatory heading outside that whitelist. -- Render every mandatory visible label at least once; repeat only labels explicitly marked as evidence-supported shared components, and never replace an output label with an icon-only node. -- Keep separately named scientific components as separately editable nodes; do not collapse special tokens, embeddings, heads, inputs, or outputs into composite labels. -- Include explicit input-boundary and output-boundary connectors. +{overlay_requirements} """.strip() diff --git a/rfs/paper_to_image/preparation.py b/rfs/paper_to_image/preparation.py index 2179f9c..04132f1 100644 --- a/rfs/paper_to_image/preparation.py +++ b/rfs/paper_to_image/preparation.py @@ -3059,7 +3059,7 @@ def prepare_paper_figure_contract( write_json(cache_path, plan) overlay = build_overlay_spec(plan) write_json(root / "overlay_spec.json", overlay) - final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1) + final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1, deterministic_overlay=True) write_text(root / "image_prompt.md", final_prompt) write_text(root / "image_prompt.txt", final_prompt) referenced_ids = _collect_evidence_ids({"paper_review": paper_review, "figure_specification": plan["figure_specification"]}) diff --git a/rfs/paper_to_image/review_contract.py b/rfs/paper_to_image/review_contract.py index 387b0ce..2109a83 100644 --- a/rfs/paper_to_image/review_contract.py +++ b/rfs/paper_to_image/review_contract.py @@ -37,6 +37,34 @@ def _tokens(value: Any) -> set[str]: return {token for token in re.findall(r"[a-z0-9\u4e00-\u9fff]+", str(value or "").casefold()) if len(token) >= 3 and token not in stop} +def _repair_owner(issue_type: str, instruction: str) -> str: + structural_prefixes = ( + "ocr_", + "topology_", + "scientific_missing_relations", + "scientific_reversed_relations", + "scientific_containment_mismatches", + ) + if issue_type.startswith(structural_prefixes): + return "overlay_compiler" + normalized = instruction.casefold() + structural_terms = ( + "arrow", + "connector", + "edge direction", + "reversed relation", + "missing relation", + "missing label", + "misspelled", + "duplicate label", + "forbidden label", + "unexpected label", + "ocr", + "text spelling", + ) + return "overlay_compiler" if any(term in normalized for term in structural_terms) else "image2_visual_substrate" + + def compile_review_ground_truth(plan: dict, evidence_records: list[dict] | None = None) -> dict: """Compile a compact, evidence-bearing contract used only for image review. @@ -254,6 +282,7 @@ def add(issue_type: str, issue: Any, severity: str, expected_check: str, evidenc "region": region, "issue_type": issue_type, "severity": severity, + "owner": _repair_owner(issue_type, instruction), "evidence_ids": list(dict.fromkeys([*(evidence_ids or []), *ids])), "preserve": preserve, "instruction": instruction, diff --git a/rfs/paper_to_image/semantic_blueprint.py b/rfs/paper_to_image/semantic_blueprint.py index da952e8..a8a4f37 100644 --- a/rfs/paper_to_image/semantic_blueprint.py +++ b/rfs/paper_to_image/semantic_blueprint.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import defaultdict, deque +from math import hypot from typing import Any @@ -314,7 +315,7 @@ def _connector_path( [round(target_center[0], 6), route_y], [round(target_center[0], 6), round(target_anchor_y, 6)], ], "outer_feedback") - if source["rank"] == target["rank"]: + if source_rank == target_rank: if source_center[1] <= target_center[1]: start = [source_center[0], source_box["y"] + source_box["h"]] end = [target_center[0], target_box["y"]] @@ -352,6 +353,329 @@ def _connector_path( ], "orthogonal") +def _segments(path: list[list[float]]) -> list[tuple[tuple[float, float], tuple[float, float]]]: + return [ + ((float(start[0]), float(start[1])), (float(end[0]), float(end[1]))) + for start, end in zip(path[:-1], path[1:]) + if start != end + ] + + +def _segment_length(segment: tuple[tuple[float, float], tuple[float, float]]) -> float: + return hypot(segment[1][0] - segment[0][0], segment[1][1] - segment[0][1]) + + +def _path_length(path: list[list[float]]) -> float: + return sum(_segment_length(segment) for segment in _segments(path)) + + +def _point_on_segment( + point: tuple[float, float], + segment: tuple[tuple[float, float], tuple[float, float]], + *, + tolerance: float = 1e-7, +) -> bool: + (x, y), ((x1, y1), (x2, y2)) = point, segment + cross = (x - x1) * (y2 - y1) - (y - y1) * (x2 - x1) + if abs(cross) > tolerance: + return False + return min(x1, x2) - tolerance <= x <= max(x1, x2) + tolerance and min(y1, y2) - tolerance <= y <= max(y1, y2) + tolerance + + +def _segment_crosses( + first: tuple[tuple[float, float], tuple[float, float]], + second: tuple[tuple[float, float], tuple[float, float]], +) -> bool: + (a1, a2), (b1, b2) = first, second + first_vertical = abs(a1[0] - a2[0]) < 1e-7 + second_vertical = abs(b1[0] - b2[0]) < 1e-7 + if first_vertical == second_vertical: + return False + vertical, horizontal = (first, second) if first_vertical else (second, first) + intersection = (vertical[0][0], horizontal[0][1]) + if not _point_on_segment(intersection, vertical) or not _point_on_segment(intersection, horizontal): + return False + endpoints = {vertical[0], vertical[1], horizontal[0], horizontal[1]} + return intersection not in endpoints + + +def _shared_segment_overlap( + first: tuple[tuple[float, float], tuple[float, float]], + second: tuple[tuple[float, float], tuple[float, float]], + *, + lane_tolerance: float = 0.006, +) -> float: + first_vertical = abs(first[0][0] - first[1][0]) < 1e-7 + second_vertical = abs(second[0][0] - second[1][0]) < 1e-7 + if first_vertical != second_vertical: + return 0.0 + if first_vertical: + if abs(first[0][0] - second[0][0]) > lane_tolerance: + return 0.0 + first_range = sorted((first[0][1], first[1][1])) + second_range = sorted((second[0][1], second[1][1])) + else: + if abs(first[0][1] - second[0][1]) > lane_tolerance: + return 0.0 + first_range = sorted((first[0][0], first[1][0])) + second_range = sorted((second[0][0], second[1][0])) + return max(0.0, min(first_range[1], second_range[1]) - max(first_range[0], second_range[0])) + + +def _segment_intersects_box( + segment: tuple[tuple[float, float], tuple[float, float]], + box: dict[str, Any], + *, + padding: float = 0.006, +) -> bool: + left = float(box["x"]) - padding + right = float(box["x"]) + float(box["w"]) + padding + top = float(box["y"]) - padding + bottom = float(box["y"]) + float(box["h"]) + padding + (x1, y1), (x2, y2) = segment + if abs(x1 - x2) < 1e-7: + return left < x1 < right and max(min(y1, y2), top) < min(max(y1, y2), bottom) + if abs(y1 - y2) < 1e-7: + return top < y1 < bottom and max(min(x1, x2), left) < min(max(x1, x2), right) + return False + + +def _compact_path(path: list[list[float]]) -> list[list[float]]: + compact: list[list[float]] = [] + for point in path: + normalized = [round(float(point[0]), 6), round(float(point[1]), 6)] + if compact and normalized == compact[-1]: + continue + compact.append(normalized) + while len(compact) >= 3: + a, b, c = compact[-3:] + if (abs(a[0] - b[0]) < 1e-7 and abs(b[0] - c[0]) < 1e-7) or (abs(a[1] - b[1]) < 1e-7 and abs(b[1] - c[1]) < 1e-7): + compact.pop(-2) + else: + break + return compact + + +def _node_center_y(node: dict[str, Any]) -> float: + box = node["bbox_percent"] + return float(box["y"]) + float(box["h"]) / 2 + + +def _relation_ports( + relations: list[dict[str, str]], + by_id: dict[str, dict[str, Any]], +) -> dict[int, tuple[float, float]]: + incoming: dict[str, list[dict[str, str]]] = defaultdict(list) + outgoing: dict[str, list[dict[str, str]]] = defaultdict(list) + for relation in relations: + incoming[relation["target"]].append(relation) + outgoing[relation["source"]].append(relation) + ports: dict[int, tuple[float, float]] = {} + for relation in relations: + target_items = sorted( + incoming[relation["target"]], + key=lambda item: (_node_center_y(by_id[item["source"]]), relations.index(item)), + ) + source_items = sorted( + outgoing[relation["source"]], + key=lambda item: (_node_center_y(by_id[item["target"]]), relations.index(item)), + ) + ports[id(relation)] = ( + (source_items.index(relation) + 1) / (len(source_items) + 1), + (target_items.index(relation) + 1) / (len(target_items) + 1), + ) + return ports + + +def _forward_endpoints( + source: dict[str, Any], + target: dict[str, Any], + source_fraction: float, + target_fraction: float, +) -> tuple[list[float], list[float]]: + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_x = source_box["x"] + source_box["w"] / 2 + target_center_x = target_box["x"] + target_box["w"] / 2 + if target_center_x >= source_center_x: + return ( + [source_box["x"] + source_box["w"], source_box["y"] + source_fraction * source_box["h"]], + [target_box["x"], target_box["y"] + target_fraction * target_box["h"]], + ) + return ( + [source_box["x"], source_box["y"] + source_fraction * source_box["h"]], + [target_box["x"] + target_box["w"], target_box["y"] + target_fraction * target_box["h"]], + ) + + +def _route_candidates( + source: dict[str, Any], + target: dict[str, Any], + relation_type: str, + source_fraction: float, + target_fraction: float, +) -> list[dict[str, Any]]: + base_path, base_style = _connector_path( + source, + target, + relation_type, + source_port_fraction=source_fraction, + target_port_fraction=target_fraction, + ) + candidates = [{"path": base_path, "route_style": base_style, "lane_assignment": "local_default"}] + source_rank = int(source.get("global_rank", source.get("rank", 0))) + target_rank = int(target.get("global_rank", target.get("rank", 0))) + is_return = relation_type in _FEEDBACK_TYPES or target_rank <= source_rank + if is_return: + source_box, target_box = source["bbox_percent"], target["bbox_percent"] + source_center_x = source_box["x"] + source_box["w"] / 2 + target_center_x = target_box["x"] + target_box["w"] / 2 + for lane_name, lane_y in (("outer_top", 0.035), ("outer_bottom", 0.965)): + source_anchor_y = source_box["y"] if lane_y < 0.5 else source_box["y"] + source_box["h"] + target_anchor_y = target_box["y"] if lane_y < 0.5 else target_box["y"] + target_box["h"] + candidates.append({ + "path": [[source_center_x, source_anchor_y], [source_center_x, lane_y], [target_center_x, lane_y], [target_center_x, target_anchor_y]], + "route_style": "outer_feedback", + "lane_assignment": lane_name, + }) + return candidates + if source_rank == target_rank: + return candidates + start, end = _forward_endpoints(source, target, source_fraction, target_fraction) + direction = 1.0 if end[0] >= start[0] else -1.0 + if target_rank - source_rank > 1: + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_x = source_box["x"] + source_box["w"] / 2 + target_center_x = target_box["x"] + target_box["w"] / 2 + for lane_index, lane_y in enumerate((0.035, 0.055, 0.945, 0.965), start=1): + source_x = start[0] + direction * (0.020 + 0.008 * (lane_index % 2)) + target_x = end[0] - direction * (0.020 + 0.008 * ((lane_index + 1) % 2)) + candidates.append({ + "path": [start, [source_x, start[1]], [source_x, lane_y], [target_x, lane_y], [target_x, end[1]], end], + "route_style": "bypass_orthogonal", + "lane_assignment": ("outer_top" if lane_y < 0.5 else "outer_bottom") + f"_{lane_index}", + }) + source_edge_y = source_box["y"] if lane_y < 0.5 else source_box["y"] + source_box["h"] + target_edge_y = target_box["y"] if lane_y < 0.5 else target_box["y"] + target_box["h"] + candidates.append({ + "path": [[source_center_x, source_edge_y], [source_center_x, lane_y], [target_center_x, lane_y], [target_center_x, target_edge_y]], + "route_style": "bypass_orthogonal", + "lane_assignment": ("outer_top_edge" if lane_y < 0.5 else "outer_bottom_edge") + f"_{lane_index}", + }) + return candidates + if abs(start[1] - end[1]) >= 0.025: + for lane_index, fraction in enumerate((0.24, 0.38, 0.50, 0.62, 0.76), start=1): + lane_x = start[0] + (end[0] - start[0]) * fraction + candidates.append({ + "path": [start, [lane_x, start[1]], [lane_x, end[1]], end], + "route_style": "orthogonal", + "lane_assignment": f"inter_rank_{lane_index}", + }) + return candidates + + +def _candidate_metrics( + candidate_path: list[list[float]], + relation: dict[str, str], + nodes: list[dict[str, Any]], + accepted_paths: list[list[list[float]]], +) -> dict[str, float | int]: + candidate_segments = _segments(candidate_path) + obstacle_intersections = 0 + for node in nodes: + if node["id"] in {relation["source"], relation["target"]}: + continue + if any(_segment_intersects_box(segment, node["bbox_percent"]) for segment in candidate_segments): + obstacle_intersections += 1 + crossings = 0 + shared_overlap = 0.0 + for accepted_path in accepted_paths: + accepted_segments = _segments(accepted_path) + crossings += sum(1 for first in candidate_segments for second in accepted_segments if _segment_crosses(first, second)) + shared_overlap += sum(_shared_segment_overlap(first, second) for first in candidate_segments for second in accepted_segments) + return { + "node_obstacle_intersections": obstacle_intersections, + "crossing_count": crossings, + "shared_segment_overlap": round(shared_overlap, 6), + "bend_count": max(0, len(candidate_path) - 2), + "path_length": round(_path_length(candidate_path), 6), + } + + +def _route_connectors( + positioned_nodes: list[dict[str, Any]], + relations: list[dict[str, str]], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + by_id = {item["id"]: item for item in positioned_nodes} + ports = _relation_ports(relations, by_id) + indexed_relations = list(enumerate(relations)) + + def priority(item: tuple[int, dict[str, str]]) -> tuple[int, int, int]: + index, relation = item + source = by_id[relation["source"]] + target = by_id[relation["target"]] + source_rank = int(source.get("global_rank", source.get("rank", 0))) + target_rank = int(target.get("global_rank", target.get("rank", 0))) + is_return = relation["type"] in _FEEDBACK_TYPES or target_rank <= source_rank + return (2 if is_return else 1 if target_rank - source_rank > 1 else 0, abs(target_rank - source_rank), index) + + accepted_paths: list[list[list[float]]] = [] + routed: dict[int, dict[str, Any]] = {} + for original_index, relation in sorted(indexed_relations, key=priority): + source_fraction, target_fraction = ports[id(relation)] + candidates = _route_candidates( + by_id[relation["source"]], + by_id[relation["target"]], + relation["type"], + source_fraction, + target_fraction, + ) + scored = [] + for candidate_index, candidate in enumerate(candidates): + path = _compact_path(candidate["path"]) + metrics = _candidate_metrics(path, relation, positioned_nodes, accepted_paths) + score = ( + int(metrics["node_obstacle_intersections"]) * 10000 + + float(metrics["shared_segment_overlap"]) * 4000 + + int(metrics["crossing_count"]) * 250 + + int(metrics["bend_count"]) * 0.7 + + float(metrics["path_length"]) + + candidate_index * 1e-5 + ) + scored.append((score, path, candidate, metrics)) + _, path, selected, metrics = min(scored, key=lambda item: item[0]) + accepted_paths.append(path) + routed[original_index] = { + "id": f"semantic_connector_{original_index + 1:02d}", + **relation, + "path_percent": path, + "route_style": selected["route_style"], + "lane_assignment": selected["lane_assignment"], + "routing_metrics": metrics, + } + + connectors = [routed[index] for index in range(len(relations))] + crossing_count = 0 + shared_overlap = 0.0 + for first_index, first in enumerate(connectors): + for second in connectors[first_index + 1:]: + first_segments = _segments(first["path_percent"]) + second_segments = _segments(second["path_percent"]) + crossing_count += sum(1 for a in first_segments for b in second_segments if _segment_crosses(a, b)) + shared_overlap += sum(_shared_segment_overlap(a, b) for a in first_segments for b in second_segments) + obstacle_intersections = sum(int(item["routing_metrics"]["node_obstacle_intersections"]) for item in connectors) + quality = { + "routing_algorithm": "global_candidate_lane_router_v1", + "crossing_count": crossing_count, + "shared_segment_overlap": round(shared_overlap, 6), + "node_obstacle_intersections": obstacle_intersections, + "lane_assignments": {item["id"]: item["lane_assignment"] for item in connectors}, + "passed": obstacle_intersections == 0 and crossing_count == 0 and shared_overlap < 0.01, + } + return connectors, quality + + def _compile_panel_graph_blueprint( specification: dict[str, Any], *, @@ -457,6 +781,7 @@ def _compile_panel_graph_blueprint( connectors: list[dict[str, Any]] = [] isolated_node_ids: list[str] = [] connector_index = 1 + panel_route_quality: list[dict[str, Any]] = [] for panel in positioned_panels: panel_id = panel["id"] nodes = panel_nodes.get(panel_id, []) @@ -483,34 +808,25 @@ def _compile_panel_graph_blueprint( }, }) positioned_nodes.extend(local_positioned) - local_by_id = {item["id"]: item for item in local_positioned} connected_ids = {value for relation in relations for value in (relation["source"], relation["target"])} isolated_node_ids.extend(item["id"] for item in nodes if item["id"] not in connected_ids) - incoming_groups: dict[str, list[dict[str, str]]] = defaultdict(list) - outgoing_groups: dict[str, list[dict[str, str]]] = defaultdict(list) - for relation in relations: - incoming_groups[relation["target"]].append(relation) - outgoing_groups[relation["source"]].append(relation) - for relation in relations: - incoming = incoming_groups[relation["target"]] - outgoing = outgoing_groups[relation["source"]] - target_fraction = (incoming.index(relation) + 1) / (len(incoming) + 1) - source_fraction = (outgoing.index(relation) + 1) / (len(outgoing) + 1) - path, route_style = _connector_path( - local_by_id[relation["source"]], - local_by_id[relation["target"]], - relation["type"], - source_port_fraction=source_fraction, - target_port_fraction=target_fraction, - ) + local_connectors, local_quality = _route_connectors(local_positioned, relations) + panel_route_quality.append({"panel_id": panel_id, **local_quality}) + for connector in local_connectors: connectors.append({ + **connector, "id": f"semantic_connector_{connector_index:02d}", "panel_id": panel_id, - **relation, - "path_percent": path, - "route_style": route_style, }) connector_index += 1 + route_quality = { + "routing_algorithm": "panel_local_global_candidate_lane_router_v1", + "crossing_count": sum(int(item["crossing_count"]) for item in panel_route_quality), + "shared_segment_overlap": round(sum(float(item["shared_segment_overlap"]) for item in panel_route_quality), 6), + "node_obstacle_intersections": sum(int(item["node_obstacle_intersections"]) for item in panel_route_quality), + "panel_reports": panel_route_quality, + "passed": all(bool(item["passed"]) for item in panel_route_quality), + } return { "summary": "Panel-local paper semantic graphs compiled into normalized node and connector geometry.", "applied": True, @@ -521,6 +837,7 @@ def _compile_panel_graph_blueprint( "panels": positioned_panels, "nodes": positioned_nodes, "connectors": connectors, + "route_quality": route_quality, "source": "panel_graphs", } @@ -561,33 +878,7 @@ def compile_semantic_blueprint( positioned = _layout_nodes_in_dense_panels(nodes, relations, dense_panels, global_ranks) else: positioned = _layout_nodes(nodes, global_ranks) - by_id = {item["id"]: item for item in positioned} - incoming_groups: dict[str, list[dict[str, str]]] = defaultdict(list) - outgoing_groups: dict[str, list[dict[str, str]]] = defaultdict(list) - for relation in relations: - incoming_groups[relation["target"]].append(relation) - outgoing_groups[relation["source"]].append(relation) - connectors = [] - for index, relation in enumerate(relations, start=1): - incoming = incoming_groups[relation["target"]] - outgoing = outgoing_groups[relation["source"]] - target_index = incoming.index(relation) - source_index = outgoing.index(relation) - target_fraction = (target_index + 1) / (len(incoming) + 1) - source_fraction = (source_index + 1) / (len(outgoing) + 1) - path, route_style = _connector_path( - by_id[relation["source"]], - by_id[relation["target"]], - relation["type"], - source_port_fraction=source_fraction, - target_port_fraction=target_fraction, - ) - connectors.append({ - "id": f"semantic_connector_{index:02d}", - **relation, - "path_percent": path, - "route_style": route_style, - }) + connectors, route_quality = _route_connectors(positioned, relations) return { "summary": "Paper-grounded semantic graph compiled into normalized node and connector geometry.", "applied": True, @@ -598,5 +889,6 @@ def compile_semantic_blueprint( "panels": dense_panels, "nodes": positioned, "connectors": connectors, + "route_quality": route_quality, "source": "figure_specification", } diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 8ac221a..6f00080 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -5,8 +5,9 @@ from pathlib import Path from typing import Any -from ..utils import write_json, write_text +from ..utils import read_json, write_json, write_text from .generator import generate_and_select, native_image2_aspect_ratio +from .overlay_renderer import render_visual_substrate_blueprint from .planner import compile_image_prompt from .preparation import prepare_paper_figure_contract from .review import validate_review_coverage @@ -40,6 +41,7 @@ def run_paper_to_image( topology_adapter=None, resume_candidates: bool = False, require_visual_enrichment: bool = True, + deterministic_overlay: bool = True, ) -> dict: started = time.time() effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" @@ -123,6 +125,18 @@ def run_paper_to_image( write_json(root / "layout_blueprint.json", blueprint_report) if isinstance(blueprint_report.get("semantic_plan"), dict) and blueprint_report["semantic_plan"].get("applied"): selected_template["semantic_plan"] = blueprint_report["semantic_plan"] + overlay_spec = read_json(root / "overlay_spec.json") + overlay_enabled = bool(deterministic_overlay and selected_template.get("semantic_plan", {}).get("applied") and not repair_source) + generation_blueprint = root / "layout_blueprint.png" + if overlay_enabled: + generation_blueprint = root / "visual_substrate_blueprint.png" + visual_blueprint_report = render_visual_substrate_blueprint( + selected_template["semantic_plan"], + generation_blueprint, + width=int(blueprint_report.get("width") or 1536), + height=int(blueprint_report.get("height") or 1024), + ) + write_json(root / "visual_substrate_blueprint.json", visual_blueprint_report) plan["style_plan"]["selected_template_id"] = selected_template.get("profile_id") plan["style_plan"]["template_style"] = selected_template.get("style", {}) @@ -132,7 +146,7 @@ def run_paper_to_image( if not planning_validation["ok"]: raise ValueError(f"Paper-to-image planning failed scientific grounding validation: {planning_validation['errors']}") - final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1, selected_template=selected_template) + final_prompt = compile_image_prompt(plan, preferences, candidate_variant=1, selected_template=selected_template, deterministic_overlay=overlay_enabled) write_text(root / "image_prompt.txt", final_prompt) generation_parameters = { "summary": "Image generation parameters.", @@ -151,6 +165,8 @@ def run_paper_to_image( "resume_candidates": bool(resume_candidates), "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, + "deterministic_overlay": overlay_enabled, + "generation_blueprint": str(generation_blueprint), } write_json(root / "generation_parameters.json", generation_parameters) @@ -175,6 +191,9 @@ def run_paper_to_image( topology_adapter=topology_adapter, resume_candidates=resume_candidates, require_visual_enrichment=require_visual_enrichment, + generation_blueprint_path=generation_blueprint, + overlay_spec=overlay_spec, + deterministic_overlay=overlay_enabled, ) elapsed = round(time.time() - started, 3) run_summary: dict[str, Any] = { @@ -199,6 +218,7 @@ def run_paper_to_image( "repair_stop_reason": generation.get("repair_stop_reason"), "repair_source": str(Path(repair_source).resolve()) if repair_source else None, "resume_candidates": bool(resume_candidates), + "deterministic_overlay": overlay_enabled, "elapsed_seconds": elapsed, "pptx_generated": False, "artifacts": [ @@ -214,6 +234,7 @@ def run_paper_to_image( "template_profiles/", "selected_template.json", "layout_blueprint.png", + *( ["visual_substrate_blueprint.png", "visual_substrate_blueprint.json", "overlays/"] if overlay_enabled else [] ), "paper_summary.json", "figure_specification.json", "design_plan.json", diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index ed75bde..e45ae2e 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -10,12 +10,13 @@ import fitz import requests -from PIL import Image, ImageDraw +from PIL import Image, ImageChops, ImageDraw from rfs.cli import _doctor, _probe_executable, build_parser from rfs.paper_to_image import run_fast_framework_prompt, run_paper_to_image from rfs.paper_to_image.critics import required_labels, review_candidate from rfs.paper_to_image.generator import generate_and_select, native_image2_aspect_ratio +from rfs.paper_to_image.overlay_renderer import render_visual_substrate_blueprint from rfs.paper_to_image.planner import collect_visible_labels, collect_visual_relations, collect_visual_roles, compile_image_prompt from rfs.paper_to_image.review import detect_domain_profile, validate_review_coverage from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint @@ -323,6 +324,59 @@ def test_visual_enrichment_is_measured_inside_the_blueprint_active_region(self): self.assertGreater(review["basic"]["blueprint_active_enrichment_ratio"], 0.08) self.assertTrue(review["basic"]["visual_enrichment_passed"]) + def test_candidate_generation_reviews_deterministically_composed_overlay(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + plan = _plan() + specification = plan["figure_specification"] + specification["topology"] = "linear" + specification["required_labels"] = ["Document Parser", "Evidence Graph Builder"] + semantic = compile_semantic_blueprint(specification) + template = {"template_id": "linear", "profile_id": "builtin_linear", "panels": [], "connectors": [], "visual_density": "high", "style": {}, "forbidden_copy_terms": [], "semantic_plan": semantic} + review_blueprint = root / "layout_blueprint.png" + visual_blueprint = root / "visual_substrate_blueprint.png" + render_layout_blueprint(template, review_blueprint, "3:2", figure_specification=specification) + render_visual_substrate_blueprint(semantic, visual_blueprint, width=1536, height=1024) + overlay = { + "summary": "Exact overlay.", + "labels": [ + {"target_id": "parser", "text": "Document Parser", "role": "module"}, + {"target_id": "graph", "text": "Evidence Graph Builder", "role": "module"}, + ], + "connectors": [{"source": "parser", "target": "graph", "type": "data_flow", "label": ""}], + "groups": [], "reading_order": ["parser", "graph"], + } + + result = generate_and_select( + plan, + {"aspect_ratio": "3:2", "language": "English"}, + template, + review_blueprint, + root / "run", + asset_mode="placeholder", + candidates=1, + review_mode="vlm", + repair_rounds=0, + ocr_engine="off", + critic_adapter=_passing_critic, + require_visual_enrichment=False, + generation_blueprint_path=visual_blueprint, + overlay_spec=overlay, + deterministic_overlay=True, + ) + + candidate = result["candidates"][0] + self.assertTrue(result["deterministic_overlay"]) + self.assertTrue(Path(candidate["raw_path"]).exists()) + self.assertTrue(Path(candidate["path"]).exists()) + self.assertTrue(candidate["overlay_report"]["passed"]) + self.assertEqual(candidate["overlay_report"]["label_count"], 2) + self.assertEqual(candidate["overlay_report"]["connector_count"], 1) + prompt = Path(candidate["prompt_path"]).read_text(encoding="utf-8") + self.assertIn("do not render any arrow, line, connector, arrowhead", prompt) + with Image.open(candidate["raw_path"]) as raw, Image.open(candidate["path"]) as composed: + self.assertIsNotNone(ImageChops.difference(raw.convert("RGB"), composed.convert("RGB")).getbbox()) + def test_candidate_review_allows_evidence_supported_shared_label_reuse(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/tests/unit/test_overlay_renderer.py b/tests/unit/test_overlay_renderer.py new file mode 100644 index 0000000..18c2090 --- /dev/null +++ b/tests/unit/test_overlay_renderer.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from PIL import Image, ImageChops + +from rfs.paper_to_image.generator import _repair_prompt +from rfs.paper_to_image.overlay_renderer import compose_semantic_overlay, render_visual_substrate_blueprint +from rfs.paper_to_image.planner import compile_image_prompt +from rfs.paper_to_image.preparation import build_overlay_spec +from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint + + +def _specification() -> dict: + return { + "summary": "Overlay test contract.", + "figure_goal": "Show a grounded linear pipeline.", + "topology": "linear", + "inputs": [{"id": "input", "name": "Input Image", "role": "input"}], + "modules": [ + {"id": "patches", "name": "Image Patches", "role": "intermediate"}, + {"id": "projection", "name": "Linear Projection", "role": "module"}, + ], + "outputs": [{"id": "output", "name": "Class", "role": "output"}], + "relations": [ + {"source": "input", "target": "patches", "type": "data_flow"}, + {"source": "patches", "target": "projection", "type": "data_flow"}, + {"source": "projection", "target": "output", "type": "prediction"}, + ], + "required_labels": ["Input Image", "Image Patches", "Linear Projection", "Class"], + } + + +class OverlayRendererTests(unittest.TestCase): + def test_visual_substrate_contains_geometry_but_no_text_or_connectors(self): + semantic = compile_semantic_blueprint(_specification()) + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "visual_substrate.png" + + report = render_visual_substrate_blueprint(semantic, path, width=900, height=600) + + self.assertTrue(path.exists()) + self.assertFalse(report["contains_semantic_text"]) + self.assertFalse(report["contains_connectors"]) + self.assertEqual(report["node_count"], 4) + + def test_overlay_composes_every_exact_label_and_semantic_connector_path(self): + specification = _specification() + semantic = compile_semantic_blueprint(specification) + plan = { + "figure_specification": specification, + "paper_summary": {"summary": "Paper."}, + "design_plan": {"summary": "Design."}, + "layout_intent": {"summary": "Layout."}, + "visual_metaphors": {"summary": "Metaphors."}, + "style_plan": {"summary": "Style."}, + } + overlay = build_overlay_spec(plan) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + substrate = root / "substrate.png" + output = root / "composed.png" + report_path = root / "overlay.json" + render_visual_substrate_blueprint(semantic, substrate, width=900, height=600) + + report = compose_semantic_overlay(substrate, semantic, overlay, output, report_path=report_path) + + self.assertTrue(report["passed"]) + self.assertEqual(report["label_count"], 4) + self.assertEqual(report["connector_count"], 3) + patches_to_projection = next(item for item in report["connectors"] if item["source"] == "patches" and item["target"] == "projection") + semantic_relation = next(item for item in semantic["connectors"] if item["source"] == "patches" and item["target"] == "projection") + self.assertEqual(patches_to_projection["path_percent"], semantic_relation["path_percent"]) + saved = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual([item["text"] for item in saved["labels"]], specification["required_labels"]) + with Image.open(substrate) as before, Image.open(output) as after: + self.assertIsNotNone(ImageChops.difference(before.convert("RGB"), after.convert("RGB")).getbbox()) + + def test_overlay_prompt_forbids_image_model_text_and_arrows(self): + specification = _specification() + plan = { + "figure_specification": specification, + "design_plan": {}, + "layout_intent": {}, + "visual_metaphors": {}, + "style_plan": {}, + } + semantic = compile_semantic_blueprint(specification) + + prompt = compile_image_prompt( + plan, + {"aspect_ratio": "3:2", "language": "English"}, + selected_template={"template_id": "linear", "semantic_plan": semantic}, + deterministic_overlay=True, + ) + + self.assertIn("do not render any of them as visible text", prompt) + self.assertIn("do not render any arrow, line, connector, arrowhead", prompt) + self.assertIn("owned by overlay_spec.json", prompt) + self.assertNotIn("Every checklist item must appear at least once as clearly readable text", prompt) + + def test_overlay_repair_prompt_keeps_text_and_connectors_compiler_owned(self): + prompt = _repair_prompt( + {"preserve": ["correct visual substrate"], "remove": [], "repair_regions": [], "hard_errors": []}, + {"items": [{"issue_type": "aesthetic_quality", "instruction": "Improve visual hierarchy."}]}, + "Generate a visual-only substrate.", + {"figure_specification": _specification()}, + deterministic_overlay=True, + ) + + self.assertIn("Do not add, remove, rewrite, move, or repair any text", prompt) + self.assertIn("Repair only the image-rich visual substrate", prompt) + self.assertNotIn("Mandatory directed connector checklist after repair", prompt) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_review_contract.py b/tests/unit/test_review_contract.py index f4429df..e9c3ed6 100644 --- a/tests/unit/test_review_contract.py +++ b/tests/unit/test_review_contract.py @@ -84,6 +84,9 @@ def test_repair_plan_prioritizes_paper_conflicts_and_carries_evidence_ids(self): self.assertEqual(repair["items"][0]["severity"], "critical") self.assertEqual(repair["items"][0]["evidence_ids"], ["E2"]) self.assertEqual(repair["items"][0]["preserve"], ["correct left panel"]) + ocr_item = next(item for item in repair["items"] if item["issue_type"] == "ocr_missing_labels") + self.assertEqual(ocr_item["owner"], "overlay_compiler") + self.assertEqual(repair["items"][0]["owner"], "image2_visual_substrate") if __name__ == "__main__": diff --git a/tests/unit/test_semantic_blueprint.py b/tests/unit/test_semantic_blueprint.py index c72f4ff..efd4fc4 100644 --- a/tests/unit/test_semantic_blueprint.py +++ b/tests/unit/test_semantic_blueprint.py @@ -167,7 +167,66 @@ def test_skip_layer_relation_uses_bypass_lane_instead_of_crossing_middle_node(se bypass = next(item for item in report["connectors"] if item["source"] == "input" and item["target"] == "merge") self.assertEqual(bypass["route_style"], "bypass_orthogonal") - self.assertGreaterEqual(len(bypass["path_percent"]), 6) + self.assertGreaterEqual(len(bypass["path_percent"]), 4) + self.assertEqual(bypass["routing_metrics"]["node_obstacle_intersections"], 0) + + def test_global_router_separates_nearby_long_vertical_lanes(self): + specification = { + "topology": "linear", + "required_labels": ["Primary Input", "Auxiliary Input", "Stage One", "Stage Two", "Merger", "Output"], + "inputs": [ + {"id": "primary", "name": "Primary Input"}, + {"id": "auxiliary", "name": "Auxiliary Input"}, + ], + "modules": [ + {"id": "stage_one", "name": "Stage One"}, + {"id": "stage_two", "name": "Stage Two"}, + {"id": "merger", "name": "Merger"}, + ], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "primary", "target": "stage_one", "type": "data_flow"}, + {"source": "stage_one", "target": "stage_two", "type": "data_flow"}, + {"source": "stage_two", "target": "merger", "type": "data_flow"}, + {"source": "auxiliary", "target": "merger", "type": "data_flow"}, + {"source": "merger", "target": "output", "type": "prediction"}, + ], + } + + report = compile_semantic_blueprint(specification) + primary = next(item for item in report["connectors"] if item["source"] == "primary") + auxiliary = next(item for item in report["connectors"] if item["source"] == "auxiliary") + + self.assertTrue(report["route_quality"]["passed"]) + self.assertEqual(report["route_quality"]["shared_segment_overlap"], 0.0) + self.assertNotEqual(primary["lane_assignment"], auxiliary["lane_assignment"]) + self.assertIn("outer_", auxiliary["lane_assignment"]) + + def test_fan_in_relations_use_distinct_target_ports(self): + specification = { + "topology": "multimodal", + "required_labels": ["Input A", "Input B", "Input C", "Fusion", "Output"], + "inputs": [ + {"id": "a", "name": "Input A"}, + {"id": "b", "name": "Input B"}, + {"id": "c", "name": "Input C"}, + ], + "modules": [{"id": "fusion", "name": "Fusion"}], + "outputs": [{"id": "output", "name": "Output"}], + "relations": [ + {"source": "a", "target": "fusion", "type": "data_flow"}, + {"source": "b", "target": "fusion", "type": "data_flow"}, + {"source": "c", "target": "fusion", "type": "data_flow"}, + {"source": "fusion", "target": "output", "type": "prediction"}, + ], + } + + report = compile_semantic_blueprint(specification) + incoming = [item for item in report["connectors"] if item["target"] == "fusion"] + + self.assertEqual(len({tuple(item["path_percent"][-1]) for item in incoming}), 3) + self.assertEqual(report["route_quality"]["node_obstacle_intersections"], 0) + self.assertEqual(report["route_quality"]["crossing_count"], 0) def test_contract_larger_than_blueprint_limit_is_not_forced_into_one_canvas(self): modules = [{"id": f"node_{index}", "name": f"Node {index}"} for index in range(18)] From ed49f268b30649683684bd65280d9222d0199bf1 Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Sat, 25 Jul 2026 09:31:15 +0800 Subject: [PATCH 73/74] Normalize Image2 substrates for editable overlays --- README.md | 15 +- docs/architecture.md | 3 +- docs/paper-to-image.md | 37 ++- rfs/cli.py | 4 +- rfs/composition/pptx.py | 66 ++++- rfs/evaluation/benchmarking.py | 24 +- rfs/paper_to_image/__init__.py | 5 +- rfs/paper_to_image/editable_ppt.py | 143 ++++++++++- rfs/paper_to_image/generator.py | 118 +++++++-- rfs/paper_to_image/overlay_renderer.py | 311 +++++++++++++++++++++++ rfs/paper_to_image/workflow.py | 41 ++- tests/integration/test_paper_to_image.py | 54 +++- tests/unit/test_overlay_renderer.py | 88 ++++++- tests/unit/test_semantic_ppt.py | 54 +++- 14 files changed, 916 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 56dcbd2..6bfacd0 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ rfs benchmark pdf-suite --out output\benchmarks\pdf-extraction --ocr-engine auto It generates 10 deterministic parser cases covering columns, typography, rotation, repeated margins, CJK and Spanish text, hyphenation, formulas, and structured tables. A real OCR engine expands the suite to 16 cases with mixed, skewed, degraded, formula/table, and Spanish scans. Each case writes its fixture, rendered preview, document model, extraction report, assertions, and timing so reading-order or OCR regressions are reproducible without committing third-party PDFs. Use `paper-to-image` when the required endpoint is a generated raster framework -figure rather than an editable PowerPoint file. The production route performs a +figure, with an optional native editable PowerPoint structure layer. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts positive references into content-free architecture templates, selects a template, renders `layout_blueprint.png`, and uses Image2 edit to create and review three @@ -136,11 +136,15 @@ candidates. For evidence-grounded contracts with 2-16 visible nodes, a generic semantic compiler now writes ranked node boxes, distinct input/output ports, global obstacle-aware orthogonal paths, branch lanes, and outer feedback loops into `layout_blueprint.json.semantic_plan`. Image2 receives a separate -`visual_substrate_blueprint.png` with no text or arrows. The compiler then adds -exact paper labels and directed connectors to `candidate_XX_raw.png`, producing -the reviewed `candidate_XX.png` plus an auditable overlay report. Self-Refine feedback and SAM-style dense +`visual_substrate_blueprint.png` with no text or arrows. Because Image2 can move +or resize those cards, the compiler detects every generated card, requires an +exact one-to-one match with semantic nodes, and recomposes its header-free visual +content into `candidate_XX_substrate.png` at the exact semantic boxes without +cropping. It then adds exact paper labels and directed connectors, producing the +reviewed `candidate_XX.png` plus auditable geometry and overlay reports. Self-Refine feedback and SAM-style dense multi-frame contracts retain stricter specialized layouts. It never invokes the -PPTX compiler. +PPTX compiler unless `--editable-overlay-ppt` is requested; that option places +the normalized substrate below native editable node labels and connectors. The contract normalizer preserves semantic roles instead of checking labels alone. Explicit provenance clauses such as `CFP and OCT from MEH-MIDAS and public datasets` compile to `data source -> modality -> method`; multi-source phrases collapse into one evidence-backed source group; explicit internal/external evaluation outputs are restored; generic labels such as `foundation model` are replaced by an explicitly named paper model when the evidence provides the alias; and named techniques such as `masked autoencoder` become required innovations. Paper-contract caching is independent of canvas ratio and display-language preferences, so changing 16:9 to 3:2 cannot select a different scientific contract. @@ -284,6 +288,7 @@ rfs make-framework ` --text-extractor-mode ocr ` --ocr-engine paddle ` --ocr-lang en_ch ` + --editable-overlay-ppt ` --json rfs validate --out "output\demo_placeholder" --json diff --git a/docs/architecture.md b/docs/architecture.md index da6acdf..0ab1925 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -77,7 +77,7 @@ blueprint region and records both active-region and whole-canvas ratios. ## Stable workflow boundaries - `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. -- `paper-to-image`: produces reviewed visual candidates and paper-grounding artifacts; it never creates PPTX. +- `paper-to-image`: produces reviewed raw, geometry-normalized substrate, and composed visual candidates plus paper-grounding artifacts; `--editable-overlay-ppt` also creates a PPTX whose labels and connectors are native editable objects over the normalized visual substrate. - `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. - `paper-to-editable`: runs both stages and requires a production-approved image unless engineering preview use is explicitly enabled. - Plugin skill: selects and invokes workflows; it does not contain the engine or hardcoded repository paths. @@ -86,6 +86,7 @@ blueprint region and records both active-region and whole-canvas ratios. ## Production rules - Failed candidates never become `selected_image.png`. +- Whole-image Image2 candidates must pass exact card/node cardinality, unique geometry matching, deterministic re-slotting, restored connector corridors, and a no-semantic-crop gate before VLM approval can count. - Engineering previews are opt-in and never production-approved. - Critical text and relations remain editable PPT objects. - Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index e752ce3..ce5ca36 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -192,13 +192,19 @@ New semantic-plan runs enable deterministic overlay mode by default: 1. Image2 generates `candidates/candidate_XX_raw.png`, containing only the image-rich visual substrate. -2. `overlay_spec.json` supplies exact paper terminology and declared directed +2. The geometry normalizer detects the visual cards, requires the detected card + count to equal the semantic node count, rejects ambiguous matching, removes + only the non-semantic generated header band, and contain-fits each visual + region into the exact semantic node box. It writes + `candidates/candidate_XX_substrate.png` and + `geometry/candidate_XX_geometry.json`. +3. `overlay_spec.json` supplies exact paper terminology and declared directed relations. -3. `layout_blueprint.json.semantic_plan` supplies normalized node boxes and +4. `layout_blueprint.json.semantic_plan` supplies normalized node boxes and connector `path_percent` geometry. -4. The compiler writes `candidates/candidate_XX.png` and +5. The compiler writes `candidates/candidate_XX.png` and `overlays/candidate_XX_overlay.json`. -5. OCR, scientific, topology, template, and aesthetic critics review the +6. OCR, scientific, topology, template, aesthetic, and substrate-geometry gates review the composed candidate, not the raw substrate. Image2 prompts explicitly forbid words, letters, numbers, formulas, arrows, @@ -219,7 +225,13 @@ segments, bend count, and path length. `semantic_plan.route_quality` records: - per-connector `lane_assignments` - the routing algorithm and pass state -Overlay compilation is a hard contract step. Missing label targets, missing +Geometry normalization is a hard contract step before VLM review. Production +requires exact card/node cardinality, unique matching, exact post-normalization +semantic boxes, restored connector corridors, and `semantic_crop_used=false`. +Failures use `failure_stage=substrate_geometry`, never provider failure. A VLM +critic cannot override this gate. + +Overlay compilation is also a hard contract step. Missing label targets, missing connector geometry, or incomplete paths fail as `failure_stage=overlay_compilation` and are not misreported as Image2 provider failures. @@ -231,11 +243,16 @@ text or connectors in deterministic overlay mode. If only compiler-owned issues remain, the repair loop stops with `overlay_recompile_required` instead of waiting on an unnecessary image-provider edit. -`--resume-candidates` prefers the raw/composed pair. A raw candidate is -re-overlaid and re-reviewed using the current semantic routes. A legacy run that +`--resume-candidates` prefers the raw/normalized/composed chain. A raw candidate is +re-normalized, re-overlaid, and re-reviewed using the current semantic boxes and routes. A legacy run that contains only the composed PNG is reviewed without applying the overlay twice; -if it later needs an Image2 substrate repair, the run stops with -`overlay_raw_source_missing` and must regenerate the missing raw candidate. +it cannot pass the new substrate-geometry gate or produce an editable substrate +PPT until the missing raw candidate is regenerated. + +With `--editable-overlay-ppt`, the selected normalized substrate becomes the +bottom PPT image layer. Node outlines, exact labels, relation labels, halo paths, +and arrowheads are native editable PowerPoint objects. The PPT never uses the +geometrically drifting raw Image2 canvas as its structural source. For supported normalized topologies (`linear`, `branch`, `multimodal`, `feedback`, and `dense_multiframe`), contracts with 2-16 visible entities are @@ -424,6 +441,8 @@ geometry while the generation canvas uses the nearest native Image2 ratio Every Image2 candidate must pass: +- exact one-to-one visual-card detection and semantic geometry normalization, + with no ambiguous mapping and no semantic crop; - exact-label OCR with no missing, misspelled, duplicate, or copied reference labels; - scientific module and relation checks with no invention or reversal; - for every declared visible relation, a second focused topology review that diff --git a/rfs/cli.py b/rfs/cli.py index 9958034..f3ab372 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -202,7 +202,7 @@ def build_parser() -> argparse.ArgumentParser: make.add_argument("--no-export", action="store_true", help="Skip PDF/PNG export and only create PPTX/artifacts.") make.add_argument("--json", action="store_true", help="Emit JSON.") - paper_image = sub.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate raster image candidates without PPTX.") + paper_image = sub.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate reviewed raster candidates with an optional editable PPT overlay.") paper_image.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") paper_image.add_argument("--out", required=True, help="Output directory for review, templates, prompts, candidates, and production-only selected_image.png.") paper_image.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") @@ -223,6 +223,7 @@ def build_parser() -> argparse.ArgumentParser: paper_image.add_argument("--repair-rounds", type=int, default=3, help="Localized Image2 edit repair rounds, clamped to 0-4 with no-regression acceptance. Default: 3.") paper_image.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") paper_image.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") + paper_image.add_argument("--editable-overlay-ppt", action="store_true", help="Place the selected geometry-normalized Image2 visual substrate beneath native editable PPT labels and connectors. Requires deterministic overlay mode and a raw candidate that passes substrate geometry normalization.") paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") paper_image.add_argument("--json", action="store_true", help="Emit JSON.") @@ -484,6 +485,7 @@ def main(argv: list[str] | None = None) -> int: repair_rounds=args.repair_rounds, repair_source=args.repair_source, resume_candidates=args.resume_candidates, + editable_overlay_ppt=args.editable_overlay_ppt, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, ) diff --git a/rfs/composition/pptx.py b/rfs/composition/pptx.py index 77a1d94..f14c435 100644 --- a/rfs/composition/pptx.py +++ b/rfs/composition/pptx.py @@ -371,6 +371,33 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: prs.slide_height = Inches(height_in) slide = prs.slides.add_slide(prs.slide_layouts[6]) + background_visual = None + background = program.get("background_image") + if isinstance(background, dict) and str(background.get("path") or "").strip(): + background_path = Path(str(background["path"])) + if not background_path.is_absolute(): + background_path = out / background_path + if not background_path.exists(): + raise FileNotFoundError(f"PPT background visual substrate is missing: {background_path}") + left, top, fit_w, fit_h, fill_percent = _picture_contain_box(background_path, 0.0, 0.0, width_in, height_in) + picture = slide.shapes.add_picture( + str(background_path), + Inches(left), + Inches(top), + width=Inches(fit_w), + height=Inches(fit_h), + ) + picture.name = str(background.get("name") or "RFS Image2 Visual Substrate") + background_visual = { + "path": str(background_path), + "role": str(background.get("role") or "image2_visual_substrate"), + "contains_scientific_text": bool(background.get("contains_scientific_text", False)), + "contains_connectors": bool(background.get("contains_connectors", False)), + "canvas_area_fill_percent": round(fill_percent, 2), + "fit_policy": "contain_no_crop", + "rendered_as": "ppt_picture_background", + } + _add_title_block(slide, program, width_in, height_in) panels_by_id = _panel_map(program) @@ -434,19 +461,46 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: shape_type = MSO_SHAPE.RECTANGLE if shape_kind == "rect" else MSO_SHAPE.ROUNDED_RECTANGLE shape = slide.shapes.add_shape(shape_type, Inches(x), Inches(y), Inches(w), Inches(h)) shape.name = f"RFS Semantic Node {node.get('id') or 'unnamed'}" - shape.fill.solid() - shape.fill.fore_color.rgb = _rgb(str(node.get("fill_color") or "#F5FAFC")) + if float(node.get("fill_transparency", 0.0)) >= 1.0: + shape.fill.background() + else: + shape.fill.solid() + shape.fill.fore_color.rgb = _rgb(str(node.get("fill_color") or "#F5FAFC")) shape.line.color.rgb = _rgb(str(node.get("stroke_color") or "#1F6F8B")) shape.line.width = Pt(float(node.get("stroke_width_pt") or 1.6)) shape.shadow.inherit = False + label_bbox = node.get("label_bbox_percent") _set_text( shape, - str(node.get("label") or node.get("name") or node.get("id") or ""), + "" if isinstance(label_bbox, dict) else str(node.get("label") or node.get("name") or node.get("id") or ""), font_size=float(node.get("font_size_pt") or 16), bold=bool(node.get("bold", True)), color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), font_family=str(node.get("font_family") or "Arial"), ) + label_rendered_as = "node_shape_text" + if isinstance(label_bbox, dict): + lx, ly, lw, lh = pct_to_inches(label_bbox, width_in, height_in) + label_shape = _add_round_rect( + slide, + lx, + ly, + lw, + lh, + str(node.get("label_fill_color") or node.get("fill_color") or "#F5FAFC"), + str(node.get("label_stroke_color") or "#FFFFFF"), + width_pt=float(node.get("label_stroke_width_pt") or 0.5), + ) + label_shape.name = f"RFS Semantic Label {node.get('id') or 'unnamed'}" + _set_text( + label_shape, + str(node.get("label") or node.get("name") or node.get("id") or ""), + font_size=float(node.get("font_size_pt") or 16), + bold=bool(node.get("bold", True)), + color=str(node.get("text_color") or node.get("stroke_color") or "#163B4D"), + font_family=str(node.get("font_family") or "Arial"), + ) + label_rendered_as = "separate_editable_header_shape" rendered_semantic_nodes.append({ "node_id": node.get("id"), "label": node.get("label") or node.get("name"), @@ -454,8 +508,11 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: "role": node.get("role"), "bbox_percent": node.get("bbox_percent"), "shape_kind": shape_kind, + "fill_transparency": float(node.get("fill_transparency", 0.0)), + "label_bbox_percent": label_bbox, + "label_rendered_as": label_rendered_as, "editable_in": "pptx", - "render_policy": "ppt_shape_with_editable_text", + "render_policy": "ppt_transparent_node_plus_editable_header" if isinstance(label_bbox, dict) else "ppt_shape_with_editable_text", "status": "ok", }) @@ -625,5 +682,6 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: "arrows": rendered_arrows, "semantic_nodes": rendered_semantic_nodes, "text": rendered_text_items, + "background_visual": background_visual, }) return pptx_path diff --git a/rfs/evaluation/benchmarking.py b/rfs/evaluation/benchmarking.py index 4aaca5e..ece8ee7 100644 --- a/rfs/evaluation/benchmarking.py +++ b/rfs/evaluation/benchmarking.py @@ -410,6 +410,7 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, selected_candidate_record = max(candidate_records, key=lambda item: float(item.get("score") or 0.0)) deterministic_overlay = bool(candidate_report.get("deterministic_overlay")) overlay_report = selected_candidate_record.get("overlay_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("overlay_report"), dict) else {} + geometry_report = selected_candidate_record.get("geometry_alignment_report", {}) if isinstance(selected_candidate_record, dict) and isinstance(selected_candidate_record.get("geometry_alignment_report"), dict) else {} layout_blueprint = _load(run / "layout_blueprint.json") semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} @@ -522,6 +523,15 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, hard_failures.append("paper planning introduced forbidden benchmark labels") if deterministic_overlay and not overlay_report: hard_failures.append("deterministic overlay report is missing for the selected candidate") + if deterministic_overlay and not geometry_report: + hard_failures.append("visual substrate geometry report is missing for the selected candidate") + if deterministic_overlay and geometry_report and not ( + geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ): + hard_failures.append("visual substrate geometry normalization gate failed") if deterministic_overlay and overlay_report and not overlay_report.get("passed", False): hard_failures.append("deterministic overlay did not compile every required label and connector") if deterministic_overlay and route_quality and not route_quality.get("passed", False): @@ -549,6 +559,7 @@ def score_paper_to_image(case_dir: str | Path, run_dir: str | Path) -> dict[str, "hallucination_count": len(invented), "forbidden_content_count": len(forbidden_found), "unexpected_label_count": len(unexpected_labels), + "substrate_geometry_pass": bool(geometry_report.get("passed")) if deterministic_overlay else None, "plan_entity_recall": planning_contract.get("entity_recall"), "plan_relation_recall": planning_contract.get("relation_recall"), "plan_forbidden_content_count": len(planning_contract.get("forbidden_labels_found", [])), @@ -660,6 +671,7 @@ def audit_paper_image_run( semantic_plan = layout_blueprint.get("semantic_plan", {}) if isinstance(layout_blueprint.get("semantic_plan"), dict) else {} route_quality = semantic_plan.get("route_quality", {}) if isinstance(semantic_plan.get("route_quality"), dict) else {} overlay_report = selected_record.get("overlay_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("overlay_report"), dict) else {} + geometry_report = selected_record.get("geometry_alignment_report", {}) if isinstance(selected_record, dict) and isinstance(selected_record.get("geometry_alignment_report"), dict) else {} review = review_candidate( candidate, blueprint, @@ -691,12 +703,22 @@ def audit_paper_image_run( "review_ground_truth": str(run / "review_ground_truth.json"), "required_priority_count": ground_truth.get("required_priority_count"), "grounded_required_priority_count": ground_truth.get("grounded_required_priority_count"), - "production_pass": bool(review.get("production_pass")), + "production_pass": bool( + review.get("production_pass") + and ( + not candidate_report.get("deterministic_overlay") + or geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ) + ), "overall_score": review.get("overall_score"), "paper_alignment_score": review.get("paper_alignment", {}).get("score"), "topology_score": review.get("topology", {}).get("score"), "deterministic_overlay": bool(candidate_report.get("deterministic_overlay")), "overlay_report": overlay_report, + "geometry_alignment_report": geometry_report, "route_quality": route_quality, "repair_item_count": repair_plan.get("item_count"), "benchmark": benchmark, diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index ec3bc28..09cb4c7 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,8 +1,9 @@ """Paper analysis and paper-to-image workflows.""" from .inspection import inspect_paper -from .editable_ppt import build_semantic_figure_program, compile_semantic_ppt +from .editable_ppt import build_semantic_figure_program, build_visual_substrate_figure_program, compile_semantic_ppt, compile_visual_substrate_ppt from .preparation import prepare_paper_figure_contract, run_fast_framework_prompt +from .overlay_renderer import normalize_visual_substrate_geometry from .workflow import run_paper_to_image -__all__ = ["build_semantic_figure_program", "compile_semantic_ppt", "inspect_paper", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] +__all__ = ["build_semantic_figure_program", "build_visual_substrate_figure_program", "compile_semantic_ppt", "compile_visual_substrate_ppt", "inspect_paper", "normalize_visual_substrate_geometry", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] diff --git a/rfs/paper_to_image/editable_ppt.py b/rfs/paper_to_image/editable_ppt.py index ea73697..75a5f98 100644 --- a/rfs/paper_to_image/editable_ppt.py +++ b/rfs/paper_to_image/editable_ppt.py @@ -89,8 +89,9 @@ def build_semantic_figure_program( aspect_ratio: str | float = "16:9", title: str | None = None, show_title: bool = False, + semantic_plan: dict[str, Any] | None = None, ) -> dict[str, Any]: - semantic_plan = compile_semantic_blueprint(specification) + semantic_plan = semantic_plan if isinstance(semantic_plan, dict) and semantic_plan.get("applied") else compile_semantic_blueprint(specification) if not semantic_plan.get("applied"): raise ValueError(f"Cannot compile editable PPT semantic blueprint: {semantic_plan.get('reason') or 'not_applicable'}") @@ -189,6 +190,100 @@ def build_semantic_figure_program( return program +def _overlay_header_bbox(bbox: dict[str, Any]) -> dict[str, float]: + x = float(bbox.get("x") or 0.0) + y = float(bbox.get("y") or 0.0) + w = float(bbox.get("w") or 0.0) + h = float(bbox.get("h") or 0.0) + inset = min(0.004, w * 0.025, h * 0.025) + header_h = min(h * 0.42, max(0.04, h * 0.30)) + return { + "x": round(x + inset, 6), + "y": round(y + inset, 6), + "w": round(max(0.01, w - inset * 2), 6), + "h": round(max(0.025, header_h - inset), 6), + } + + +def build_visual_substrate_figure_program( + specification: dict[str, Any], + substrate_image: str | Path, + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, + semantic_plan: dict[str, Any] | None = None, + overlay_spec: dict[str, Any] | None = None, +) -> dict[str, Any]: + substrate = Path(substrate_image).resolve() + if not substrate.exists(): + raise FileNotFoundError(f"Visual substrate image does not exist: {substrate}") + program = build_semantic_figure_program( + specification, + aspect_ratio=aspect_ratio, + title=title, + show_title=show_title, + semantic_plan=semantic_plan, + ) + exact_labels = { + str(item.get("target_id")): str(item.get("text") or "").strip() + for item in (overlay_spec or {}).get("labels", []) + if isinstance(item, dict) and item.get("target_id") and str(item.get("text") or "").strip() + } + for node in program["semantic_nodes"]: + if exact_labels: + target_id = str(node.get("id") or "") + if target_id not in exact_labels: + raise ValueError(f"Editable overlay label is missing for semantic node: {target_id}") + node["label"] = exact_labels[target_id] + node["fill_transparency"] = 1.0 + node["label_bbox_percent"] = _overlay_header_bbox(node["bbox_percent"]) + node["label_fill_color"] = node.get("fill_color") + node["label_stroke_color"] = "#FFFFFF" + node["label_stroke_width_pt"] = 0.5 + node["render_policy"] = "transparent_editable_node_outline_plus_editable_header" + requested_connectors = { + (str(item.get("source") or ""), str(item.get("target") or ""), str(item.get("type") or "data_flow")) + for item in (overlay_spec or {}).get("connectors", []) + if isinstance(item, dict) and item.get("source") and item.get("target") + } + compiled_connectors = { + (str(item.get("source") or ""), str(item.get("target") or ""), str(item.get("type") or "data_flow")) + for item in program["arrows"] + } + if requested_connectors and requested_connectors != compiled_connectors: + missing = sorted(requested_connectors - compiled_connectors) + extra = sorted(compiled_connectors - requested_connectors) + raise ValueError(f"Editable overlay connector contract mismatch; missing={missing}, extra={extra}") + for arrow in program["arrows"]: + arrow["halo_width_pt"] = max(3.2, float(arrow.get("stroke_width_pt") or 1.7) + 1.3) + arrow["halo_color"] = "#FFFFFF" + arrow["routing_algorithm"] = program["semantic_plan"].get("route_quality", {}).get("routing_algorithm") + arrow["route_generation_status"] = "semantic_plan_exact_path" + program.update({ + "summary": "Image2 visual substrate combined with native editable PowerPoint labels, node outlines, and directed connectors.", + "visual_substrate_mode": True, + "background_image": { + "path": str(substrate), + "name": "RFS Image2 Visual Substrate", + "role": "image2_visual_substrate", + "contains_scientific_text": False, + "contains_connectors": False, + "fit_policy": "contain_no_crop", + }, + "assets": [{ + "id": "image2_visual_substrate", + "path": str(substrate), + "role": "visual_substrate", + "editable_in": "pptx_picture", + "contains_scientific_text": False, + "contains_connectors": False, + "fit_policy": "contain_no_crop", + }], + }) + return program + + def compile_semantic_ppt( specification: dict[str, Any], out: str | Path, @@ -215,3 +310,49 @@ def compile_semantic_ppt( } write_json(root / "semantic_ppt_report.json", report) return report + + +def compile_visual_substrate_ppt( + specification: dict[str, Any], + substrate_image: str | Path, + out: str | Path, + *, + aspect_ratio: str | float = "16:9", + title: str | None = None, + show_title: bool = False, + semantic_plan: dict[str, Any] | None = None, + overlay_spec: dict[str, Any] | None = None, +) -> dict[str, Any]: + root = Path(out) + root.mkdir(parents=True, exist_ok=True) + program = build_visual_substrate_figure_program( + specification, + substrate_image, + aspect_ratio=aspect_ratio, + title=title, + show_title=show_title, + semantic_plan=semantic_plan, + overlay_spec=overlay_spec, + ) + write_json(root / "figure_program.json", program) + pptx_path = compile_ppt(program, root) + composition = root / "composition_quality_report.json" + report = { + "summary": "Image2 visual substrate compiled beneath native editable PowerPoint terminology and connector geometry.", + "ok": pptx_path.exists(), + "pptx": str(pptx_path), + "figure_program": str(root / "figure_program.json"), + "composition_quality_report": str(composition), + "visual_substrate": str(Path(substrate_image).resolve()), + "node_count": len(program["semantic_nodes"]), + "connector_count": len(program["arrows"]), + "relation_label_count": len(program["labels"]), + "topology": program["semantic_plan"].get("topology"), + "route_quality": program["semantic_plan"].get("route_quality", {}), + "editable_layers": ["native_node_outlines", "native_header_labels", "native_connectors", "native_relation_labels"], + "visual_layer": "single_image2_substrate_picture", + "scientific_text_baked_into_visual_layer": False, + "connectors_baked_into_visual_layer": False, + } + write_json(root / "editable_overlay_report.json", report) + return report diff --git a/rfs/paper_to_image/generator.py b/rfs/paper_to_image/generator.py index a677e4d..04bd712 100644 --- a/rfs/paper_to_image/generator.py +++ b/rfs/paper_to_image/generator.py @@ -17,7 +17,7 @@ from ..asset_generator import _call_gemini from ..utils import ensure_dir, write_json, write_text from .critics import aggregate_reviews, review_candidate -from .overlay_renderer import compose_semantic_overlay +from .overlay_renderer import compose_semantic_overlay, normalize_visual_substrate_geometry from .planner import collect_visual_relations, compile_image_prompt from .review_contract import build_repair_plan @@ -114,6 +114,9 @@ def _candidate_failure_modes(record: dict) -> list[str]: topology = review.get("topology", {}) if isinstance(review.get("topology"), dict) else {} template = review.get("template", {}) if isinstance(review.get("template"), dict) else {} aesthetic = review.get("aesthetic", {}) if isinstance(review.get("aesthetic"), dict) else {} + substrate_geometry = review.get("substrate_geometry", {}) if isinstance(review.get("substrate_geometry"), dict) else {} + if substrate_geometry and not substrate_geometry.get("passed", False): + modes.append("substrate_geometry") unexpected_labels = ocr.get("unsupported_unexpected_labels") if "unsupported_unexpected_labels" in ocr else ocr.get("unexpected_labels") if ocr.get("missing_labels") or ocr.get("misspelled_labels") or ocr.get("duplicate_labels") or unexpected_labels: modes.append("label_accuracy") @@ -173,12 +176,13 @@ def _build_stability_report(records: list[dict], failures: list[dict], requested "failure_modes": modes, }) for item in independent_failures: - failure_modes["generation_provider"] = failure_modes.get("generation_provider", 0) + 1 + failure_mode = "substrate_geometry" if item.get("failure_stage") == "substrate_geometry" else "generation_provider" if item.get("provider_failure") else "generation_or_review" + failure_modes[failure_mode] = failure_modes.get(failure_mode, 0) + 1 candidates.append({ "candidate_id": item.get("candidate_id"), "score": 0.0, "production_pass": False, - "failure_modes": ["generation_provider"], + "failure_modes": [failure_mode], "error": str(item.get("error") or "candidate generation failed"), }) stable = bool(seed_count >= 2 and pass_rate >= 0.8 and worst_score >= 0.8) @@ -309,13 +313,14 @@ def _failure_metadata(error: Exception) -> dict[str, Any]: error_text = str(error) normalized = error_text.casefold() overlay_failure = "deterministic overlay compilation failed" in normalized - provider_timeout = not overlay_failure and any(token in normalized for token in ("timed out", "read timeout", "connect timeout")) - provider_failure = not overlay_failure and ( + geometry_failure = "visual substrate geometry normalization failed" in normalized + provider_timeout = not overlay_failure and not geometry_failure and any(token in normalized for token in ("timed out", "read timeout", "connect timeout")) + provider_failure = not overlay_failure and not geometry_failure and ( provider_timeout or any(token in normalized for token in ("http 429", "http 5", "provider", "connection")) ) return { "error": error_text, - "failure_stage": "overlay_compilation" if overlay_failure else "image_provider" if provider_failure else "generation_or_review", + "failure_stage": "substrate_geometry" if geometry_failure else "overlay_compilation" if overlay_failure else "image_provider" if provider_failure else "generation_or_review", "provider_failure": provider_failure, "provider_timeout": provider_timeout, } @@ -384,6 +389,7 @@ def generate_and_select( prompt_dir = ensure_dir(root / "prompts") review_dir = ensure_dir(root / "reviews") overlay_dir = ensure_dir(root / "overlays") + geometry_dir = ensure_dir(root / "geometry") blueprint = Path(blueprint_path) generation_blueprint = Path(generation_blueprint_path) if generation_blueprint_path else blueprint semantic_plan = selected_template.get("semantic_plan", {}) if isinstance(selected_template.get("semantic_plan"), dict) else {} @@ -453,20 +459,72 @@ def run_review(path: Path) -> dict[str, Any]: final_review["review_attempt_history"] = attempts return final_review + def apply_geometry_gate(review: dict[str, Any], geometry_report: dict[str, Any] | None) -> dict[str, Any]: + if not overlay_enabled: + return review + passed = bool( + geometry_report + and geometry_report.get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("passed") + and geometry_report.get("post_normalization_alignment", {}).get("unique_node_card_assignment") + and not geometry_report.get("post_normalization_alignment", {}).get("semantic_crop_used") + ) + review["substrate_geometry"] = { + "summary": "Hard gate for exact one-to-one visual-card normalization before semantic overlay.", + "passed": passed, + "expected_node_count": geometry_report.get("expected_node_count") if geometry_report else None, + "detected_card_count": geometry_report.get("detected_card_count") if geometry_report else None, + "matched_node_count": geometry_report.get("matched_node_count") if geometry_report else None, + "post_normalization_alignment": geometry_report.get("post_normalization_alignment", {}) if geometry_report else {}, + } + if not passed: + hard_errors = list(review.get("hard_errors", [])) + message = "Visual substrate geometry did not pass exact one-to-one normalization" + if message not in hard_errors: + hard_errors.append(message) + review["hard_errors"] = hard_errors + review["production_pass"] = False + return review + repair_source_path = Path(repair_source).resolve() if repair_source else None if repair_source_path and not repair_source_path.exists(): raise FileNotFoundError(f"Repair source image does not exist: {repair_source_path}") - def apply_overlay(raw_path: Path, composed_path: Path, report_path: Path) -> dict[str, Any]: + def compile_visual_candidate( + raw_path: Path, + substrate_path: Path, + composed_path: Path, + geometry_report_path: Path, + overlay_report_path: Path, + ) -> tuple[dict[str, Any], dict[str, Any]]: + try: + geometry_report = normalize_visual_substrate_geometry( + raw_path, + generation_blueprint, + semantic_plan, + substrate_path, + report_path=geometry_report_path, + ) + except Exception as exc: + raise RuntimeError(f"Visual substrate geometry normalization failed: {exc}") from exc try: - return compose_semantic_overlay(raw_path, semantic_plan, overlay_spec or {}, composed_path, report_path=report_path) + overlay_report = compose_semantic_overlay( + substrate_path, + semantic_plan, + overlay_spec or {}, + composed_path, + report_path=overlay_report_path, + ) except Exception as exc: raise RuntimeError(f"Deterministic overlay compilation failed: {exc}") from exc + return geometry_report, overlay_report def generate_candidate(index: int) -> dict[str, Any]: candidate_id = f"candidate_{index:02d}" path = candidate_dir / f"{candidate_id}.png" raw_path = candidate_dir / f"{candidate_id}_raw.png" if overlay_enabled else path + substrate_path = candidate_dir / f"{candidate_id}_substrate.png" if overlay_enabled else path + geometry_report_path = geometry_dir / f"{candidate_id}_geometry.json" overlay_report_path = overlay_dir / f"{candidate_id}_overlay.json" prompt = compile_image_prompt(plan, preferences, candidate_variant=index, selected_template=selected_template, deterministic_overlay=overlay_enabled) prompt_path = prompt_dir / f"{candidate_id}_prompt.txt" @@ -474,7 +532,9 @@ def generate_candidate(index: int) -> dict[str, Any]: started = time.monotonic() try: overlay_report = None + geometry_report = None record_raw_path: Path | None = raw_path if overlay_enabled else None + record_substrate_path: Path | None = substrate_path if overlay_enabled else None if resume_candidates and raw_path.exists(): generation = {"mode": "existing_candidate_resume", "source": str(raw_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} generation_seconds = 0.0 @@ -482,6 +542,13 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = {"mode": "existing_candidate_resume_legacy_composed", "source": str(path), "overlay_assumed_precomposed": overlay_enabled, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} raw_path = path record_raw_path = None + record_substrate_path = None + geometry_report = { + "summary": "Legacy composed-only candidate cannot prove visual-substrate geometry alignment.", + "passed": False, + "failure_stage": "substrate_geometry", + "reason": "raw_visual_substrate_missing", + } if overlay_report_path.exists(): overlay_report = json.loads(overlay_report_path.read_text(encoding="utf-8")) generation_seconds = 0.0 @@ -490,21 +557,32 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = _generate_one(prompt, plan, generation_blueprint, raw_path, aspect_ratio, asset_mode, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) if overlay_enabled and raw_path != path: - overlay_report = apply_overlay(raw_path, path, overlay_report_path) - generation = {**generation, "raw_path": str(raw_path), "composed_path": str(path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True} + geometry_report, overlay_report = compile_visual_candidate(raw_path, substrate_path, path, geometry_report_path, overlay_report_path) + generation = { + **generation, + "raw_path": str(raw_path), + "visual_substrate_path": str(substrate_path), + "composed_path": str(path), + "geometry_alignment_report": str(geometry_report_path), + "overlay_report": str(overlay_report_path), + "deterministic_overlay": True, + } review_started = time.monotonic() review = run_review(path) + review = apply_geometry_gate(review, geometry_report) review_seconds = round(time.monotonic() - review_started, 3) return { "record": { "candidate_id": candidate_id, "path": str(path), "raw_path": str(record_raw_path) if record_raw_path else None, + "visual_substrate_path": str(record_substrate_path) if record_substrate_path else None, + "geometry_alignment_report": geometry_report, "overlay_report": overlay_report, "prompt_path": str(prompt_path), "generation": generation, "review": review, - "production_pass": bool(asset_mode == "image2" and review["production_pass"]), + "production_pass": bool(asset_mode == "image2" and review["production_pass"] and (not overlay_enabled or geometry_report and geometry_report.get("passed"))), "score": review["overall_score"], "timings": { "generation_seconds": generation_seconds, @@ -608,6 +686,8 @@ def generate_candidate(index: int) -> dict[str, Any]: repaired_id = f"repair_{repair_index + 1:02d}" repaired_path = candidate_dir / f"{repaired_id}.png" repaired_raw_path = candidate_dir / f"{repaired_id}_raw.png" if overlay_enabled else repaired_path + repaired_substrate_path = candidate_dir / f"{repaired_id}_substrate.png" if overlay_enabled else repaired_path + geometry_report_path = geometry_dir / f"{repaired_id}_geometry.json" overlay_report_path = overlay_dir / f"{repaired_id}_overlay.json" prompt_path = prompt_dir / f"{repaired_id}_prompt.txt" repair_plan_path = prompt_dir / f"{repaired_id}_plan.json" @@ -641,12 +721,14 @@ def generate_candidate(index: int) -> dict[str, Any]: repair_started = time.monotonic() try: overlay_report = None + geometry_report = None if overlay_enabled and resume_candidates and repaired_raw_path.exists(): - overlay_report = apply_overlay(repaired_raw_path, repaired_path, overlay_report_path) - generation = {"mode": "existing_repair_reoverlay", "source": str(repaired_raw_path), "raw_path": str(repaired_raw_path), "composed_path": str(repaired_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} + geometry_report, overlay_report = compile_visual_candidate(repaired_raw_path, repaired_substrate_path, repaired_path, geometry_report_path, overlay_report_path) + generation = {"mode": "existing_repair_reoverlay", "source": str(repaired_raw_path), "raw_path": str(repaired_raw_path), "visual_substrate_path": str(repaired_substrate_path), "composed_path": str(repaired_path), "geometry_alignment_report": str(geometry_report_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True, "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} generation_seconds = 0.0 review_started = time.monotonic() review = run_review(repaired_path) + review = apply_geometry_gate(review, geometry_report) review_seconds = round(time.monotonic() - review_started, 3) write_json(review_path, review) elif resume_candidates and repaired_path.exists() and review_path.exists() and not overlay_enabled: @@ -656,6 +738,7 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = {"mode": "existing_repair_reaudit", "source": str(repaired_path), "api_key_present": bool(os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY"))} review_started = time.monotonic() review = run_review(repaired_path) + review = apply_geometry_gate(review, geometry_report) review_seconds = round(time.monotonic() - review_started, 3) write_json(review_path, review) else: @@ -668,10 +751,11 @@ def generate_candidate(index: int) -> dict[str, Any]: generation = _call_image2_edit(source_visual, prompt, repaired_raw_path, aspect_ratio, image_model, image_retries) generation_seconds = round(time.monotonic() - generation_started, 3) if overlay_enabled and repaired_raw_path != repaired_path: - overlay_report = apply_overlay(repaired_raw_path, repaired_path, overlay_report_path) - generation = {**generation, "raw_path": str(repaired_raw_path), "composed_path": str(repaired_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True} + geometry_report, overlay_report = compile_visual_candidate(repaired_raw_path, repaired_substrate_path, repaired_path, geometry_report_path, overlay_report_path) + generation = {**generation, "raw_path": str(repaired_raw_path), "visual_substrate_path": str(repaired_substrate_path), "composed_path": str(repaired_path), "geometry_alignment_report": str(geometry_report_path), "overlay_report": str(overlay_report_path), "deterministic_overlay": True} review_started = time.monotonic() review = run_review(repaired_path) + review = apply_geometry_gate(review, geometry_report) review_seconds = round(time.monotonic() - review_started, 3) write_json(review_path, review) requests_manifest.append({"candidate_id": repaired_id, "repair_source": source["candidate_id"], "generation_seconds": generation_seconds, **generation}) @@ -680,13 +764,15 @@ def generate_candidate(index: int) -> dict[str, Any]: "candidate_id": repaired_id, "path": str(repaired_path), "raw_path": str(repaired_raw_path) if overlay_enabled else None, + "visual_substrate_path": str(repaired_substrate_path) if overlay_enabled else None, + "geometry_alignment_report": geometry_report, "overlay_report": overlay_report, "prompt_path": str(prompt_path), "repair_plan_path": str(repair_plan_path), "review_path": str(review_path), "generation": generation, "review": review, - "production_pass": bool(review["production_pass"] and accepted), + "production_pass": bool(review["production_pass"] and accepted and (not overlay_enabled or geometry_report and geometry_report.get("passed"))), "raw_production_pass": bool(review["production_pass"]), "score": review["overall_score"], "repair_source": source["candidate_id"], diff --git a/rfs/paper_to_image/overlay_renderer.py b/rfs/paper_to_image/overlay_renderer.py index 415f0bc..f1b7e30 100644 --- a/rfs/paper_to_image/overlay_renderer.py +++ b/rfs/paper_to_image/overlay_renderer.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Any +import cv2 +import numpy as np from PIL import Image, ImageDraw, ImageFont from ..utils import write_json @@ -192,6 +194,315 @@ def render_visual_substrate_blueprint( } +def _normalized_bbox(box: tuple[int, int, int, int], width: int, height: int) -> dict[str, float]: + x, y, w, h = box + return {"x": x / width, "y": y / height, "w": w / width, "h": h / height} + + +def _bbox_center(box: dict[str, Any]) -> tuple[float, float]: + return (float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2) + + +def _bbox_iou(first: dict[str, Any], second: dict[str, Any]) -> float: + left = max(float(first["x"]), float(second["x"])) + top = max(float(first["y"]), float(second["y"])) + right = min(float(first["x"]) + float(first["w"]), float(second["x"]) + float(second["w"])) + bottom = min(float(first["y"]) + float(first["h"]), float(second["y"]) + float(second["h"])) + intersection = max(0.0, right - left) * max(0.0, bottom - top) + union = float(first["w"]) * float(first["h"]) + float(second["w"]) * float(second["h"]) - intersection + return intersection / union if union > 0 else 0.0 + + +def _detect_visual_card_boxes(image: Image.Image) -> list[tuple[int, int, int, int]]: + rgb = np.asarray(image.convert("RGB")) + bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) + hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV) + mask = (((hsv[:, :, 1] > 35) & (hsv[:, :, 2] < 250)) | (hsv[:, :, 2] < 190)).astype(np.uint8) * 255 + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7)) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel, iterations=2) + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + height, width = rgb.shape[:2] + boxes: list[tuple[int, int, int, int]] = [] + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + normalized_w, normalized_h = w / width, h / height + area = normalized_w * normalized_h + if 0.06 <= normalized_w <= 0.30 and 0.08 <= normalized_h <= 0.45 and area < 0.12: + boxes.append((x, y, w, h)) + return sorted(boxes, key=lambda item: (item[1], item[0])) + + +def _match_visual_cards( + nodes: list[dict[str, Any]], + boxes: list[tuple[int, int, int, int]], + width: int, + height: int, +) -> list[dict[str, Any]]: + pairs = [] + node_costs: dict[int, list[tuple[float, int]]] = {} + box_costs: dict[int, list[tuple[float, int]]] = {} + for node_index, node in enumerate(nodes): + expected = node["bbox_percent"] + expected_center = _bbox_center(expected) + for box_index, box in enumerate(boxes): + detected = _normalized_bbox(box, width, height) + detected_center = _bbox_center(detected) + center_drift = math.dist(expected_center, detected_center) + width_error = abs(math.log(max(float(detected["w"]), 1e-6) / max(float(expected["w"]), 1e-6))) + height_error = abs(math.log(max(float(detected["h"]), 1e-6) / max(float(expected["h"]), 1e-6))) + cost = center_drift * 4.0 + (width_error + height_error) * 0.25 + pairs.append((cost, node_index, box_index)) + node_costs.setdefault(node_index, []).append((cost, box_index)) + box_costs.setdefault(box_index, []).append((cost, node_index)) + assignments: dict[int, tuple[int, float]] = {} + used_boxes: set[int] = set() + for cost, node_index, box_index in sorted(pairs): + if node_index in assignments or box_index in used_boxes: + continue + assignments[node_index] = (box_index, cost) + used_boxes.add(box_index) + matches = [] + for node_index, node in enumerate(nodes): + if node_index not in assignments: + continue + box_index, cost = assignments[node_index] + detected = _normalized_bbox(boxes[box_index], width, height) + expected = node["bbox_percent"] + center_drift = math.dist(_bbox_center(expected), _bbox_center(detected)) + width_relative_error = abs(float(detected["w"]) - float(expected["w"])) / max(float(expected["w"]), 1e-6) + height_relative_error = abs(float(detected["h"]) - float(expected["h"])) / max(float(expected["h"]), 1e-6) + node_alternatives = sorted(node_costs.get(node_index, [])) + box_alternatives = sorted(box_costs.get(box_index, [])) + node_margin = node_alternatives[1][0] - cost if len(node_alternatives) > 1 else float("inf") + box_margin = box_alternatives[1][0] - cost if len(box_alternatives) > 1 else float("inf") + matches.append({ + "node_id": node["id"], + "node_index": node_index, + "detected_box_index": box_index, + "expected_bbox_percent": expected, + "detected_bbox_percent": {key: round(value, 6) for key, value in detected.items()}, + "detected_bbox_pixels": list(boxes[box_index]), + "assignment_cost": round(cost, 6), + "node_assignment_margin": round(node_margin, 6) if math.isfinite(node_margin) else None, + "box_assignment_margin": round(box_margin, 6) if math.isfinite(box_margin) else None, + "center_drift": round(center_drift, 6), + "width_relative_error": round(width_relative_error, 6), + "height_relative_error": round(height_relative_error, 6), + "iou": round(_bbox_iou(expected, detected), 6), + }) + return matches + + +def _header_separator(card_rgb: np.ndarray) -> int: + height, width = card_rgb.shape[:2] + hsv = cv2.cvtColor(cv2.cvtColor(card_rgb, cv2.COLOR_RGB2BGR), cv2.COLOR_BGR2HSV) + edge_mask = ((hsv[:, :, 1] > 35) & (hsv[:, :, 2] < 245)).astype(np.uint8) + left, right = max(2, int(width * 0.03)), min(width - 2, int(width * 0.97)) + for y in range(max(3, int(height * 0.07)), max(4, int(height * 0.36))): + if edge_mask[y, left:right].mean() > 0.58: + return y + return int(height * 0.20) + + +def _contain_dimensions(source_w: int, source_h: int, target_w: int, target_h: int) -> tuple[int, int]: + source_ratio = source_w / max(source_h, 1) + target_ratio = target_w / max(target_h, 1) + if source_ratio > target_ratio: + width = target_w + height = max(1, round(width / source_ratio)) + else: + height = target_h + width = max(1, round(height * source_ratio)) + return width, height + + +def _trim_background_only(image: Image.Image) -> tuple[Image.Image, dict[str, Any]]: + """Trim only border-connected near-background pixels; preserve all detected foreground.""" + + rgb = np.asarray(image.convert("RGB")) + height, width = rgb.shape[:2] + if width < 12 or height < 12: + return image, {"applied": False, "reason": "image_too_small", "bbox_pixels": [0, 0, width, height]} + border = max(2, round(min(width, height) * 0.04)) + border_pixels = np.concatenate( + [ + rgb[:border, :, :].reshape(-1, 3), + rgb[-border:, :, :].reshape(-1, 3), + rgb[:, :border, :].reshape(-1, 3), + rgb[:, -border:, :].reshape(-1, 3), + ], + axis=0, + ) + background = np.median(border_pixels.astype(np.float32), axis=0) + delta = np.linalg.norm(rgb.astype(np.float32) - background, axis=2) + gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY) + hsv = cv2.cvtColor(cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), cv2.COLOR_BGR2HSV) + foreground = ((delta > 22.0) | (gray < 205) | (hsv[:, :, 1] > 42)).astype(np.uint8) * 255 + foreground = cv2.morphologyEx(foreground, cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8), iterations=1) + count, labels, stats, _ = cv2.connectedComponentsWithStats(foreground, connectivity=8) + minimum_component = max(6, round(width * height * 0.0008)) + kept = [index for index in range(1, count) if int(stats[index, cv2.CC_STAT_AREA]) >= minimum_component] + if not kept: + return image, { + "applied": False, + "reason": "no_reliable_foreground", + "background_rgb": [round(float(value), 2) for value in background], + "bbox_pixels": [0, 0, width, height], + } + left = min(int(stats[index, cv2.CC_STAT_LEFT]) for index in kept) + top = min(int(stats[index, cv2.CC_STAT_TOP]) for index in kept) + right = max(int(stats[index, cv2.CC_STAT_LEFT] + stats[index, cv2.CC_STAT_WIDTH]) for index in kept) + bottom = max(int(stats[index, cv2.CC_STAT_TOP] + stats[index, cv2.CC_STAT_HEIGHT]) for index in kept) + padding = max(2, round(min(width, height) * 0.025)) + left, top = max(0, left - padding), max(0, top - padding) + right, bottom = min(width, right + padding), min(height, bottom + padding) + retained_area = (right - left) * (bottom - top) / max(1, width * height) + if retained_area < 0.04: + return image, { + "applied": False, + "reason": "foreground_bbox_too_small", + "background_rgb": [round(float(value), 2) for value in background], + "bbox_pixels": [0, 0, width, height], + } + applied = bool(left > 0 or top > 0 or right < width or bottom < height) + return image.crop((left, top, right, bottom)) if applied else image, { + "applied": applied, + "reason": "border_connected_background_removed" if applied else "foreground_reaches_canvas_edges", + "background_rgb": [round(float(value), 2) for value in background], + "bbox_pixels": [left, top, right, bottom], + "retained_area_percent": round(retained_area * 100, 2), + "semantic_crop_used": False, + } + + +def normalize_visual_substrate_geometry( + source_path: str | Path, + guide_path: str | Path, + semantic_plan: dict[str, Any], + out_path: str | Path, + *, + report_path: str | Path | None = None, +) -> dict[str, Any]: + """Recompose Image2 node visuals into exact semantic boxes before overlay.""" + + if not semantic_plan.get("applied"): + raise ValueError("Visual substrate normalization requires an applied semantic plan") + source = Path(source_path) + guide = Path(guide_path) + if not source.exists() or not guide.exists(): + raise FileNotFoundError(f"Visual substrate normalization inputs are missing: source={source}, guide={guide}") + with Image.open(source) as raw_image: + raw = raw_image.convert("RGB") + with Image.open(guide) as guide_image: + base = guide_image.convert("RGB").resize(raw.size, Image.Resampling.LANCZOS) + width, height = raw.size + nodes = [item for item in semantic_plan.get("nodes", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + boxes = _detect_visual_card_boxes(raw) + if len(boxes) != len(nodes): + raise ValueError(f"Detected {len(boxes)} visual cards for {len(nodes)} semantic nodes; exact cardinality is required") + matches = _match_visual_cards(nodes, boxes, width, height) + if len(matches) != len(nodes): + raise ValueError(f"Matched only {len(matches)} visual cards for {len(nodes)} semantic nodes") + if any(float(item["assignment_cost"]) > 1.0 for item in matches): + raise ValueError("Visual card matching is too ambiguous for deterministic geometry normalization") + if any( + item.get("node_assignment_margin") is not None and float(item["node_assignment_margin"]) < 0.015 + or item.get("box_assignment_margin") is not None and float(item["box_assignment_margin"]) < 0.015 + for item in matches + ): + raise ValueError("Visual card matching is ambiguous because multiple node-card assignments have nearly equal cost") + + raw_array = np.asarray(raw) + normalized_items = [] + for match in matches: + x, y, card_w, card_h = [int(value) for value in match["detected_bbox_pixels"]] + card = raw_array[y:y + card_h, x:x + card_w] + separator = _header_separator(card) + crop_left = max(2, int(card_w * 0.035)) + crop_right = min(card_w - 2, int(card_w * 0.965)) + crop_top = min(card_h - 5, separator + 3) + crop_bottom = max(crop_top + 2, int(card_h * 0.96)) + visual_crop = raw.crop((x + crop_left, y + crop_top, x + crop_right, y + crop_bottom)) + visual_crop, background_trim = _trim_background_only(visual_crop) + + expected = match["expected_bbox_percent"] + node_left = round(float(expected["x"]) * width) + node_top = round(float(expected["y"]) * height) + node_right = round((float(expected["x"]) + float(expected["w"])) * width) + node_bottom = round((float(expected["y"]) + float(expected["h"])) * height) + content_top = round(node_top + (node_bottom - node_top) * 0.32) + padding = max(3, round(min(node_right - node_left, node_bottom - content_top) * 0.04)) + target_left, target_top = node_left + padding, content_top + padding + target_right, target_bottom = node_right - padding, node_bottom - padding + resized_w, resized_h = _contain_dimensions( + visual_crop.width, + visual_crop.height, + max(1, target_right - target_left), + max(1, target_bottom - target_top), + ) + resized = visual_crop.resize((resized_w, resized_h), Image.Resampling.LANCZOS) + paste_left = target_left + (target_right - target_left - resized_w) // 2 + paste_top = target_top + (target_bottom - target_top - resized_h) // 2 + base.paste(resized, (paste_left, paste_top)) + fill_percent = resized_w * resized_h / max(1, (target_right - target_left) * (target_bottom - target_top)) * 100 + normalized_items.append({ + **match, + "header_separator_fraction": round(separator / max(card_h, 1), 6), + "visual_crop_pixels": [x + crop_left, y + crop_top, x + crop_right, y + crop_bottom], + "background_trim": background_trim, + "target_content_pixels": [target_left, target_top, target_right, target_bottom], + "pasted_visual_pixels": [paste_left, paste_top, paste_left + resized_w, paste_top + resized_h], + "content_area_fill_percent": round(fill_percent, 2), + "fit_policy": "contain_no_crop", + "semantic_crop_used": False, + }) + + output = Path(out_path) + output.parent.mkdir(parents=True, exist_ok=True) + base.save(output) + pre_passed = all( + float(item["center_drift"]) <= 0.035 + and float(item["width_relative_error"]) <= 0.30 + and float(item["height_relative_error"]) <= 0.30 + and float(item["iou"]) >= 0.55 + for item in matches + ) + report = { + "summary": "Image2 visual cards detected, matched to semantic nodes, and recomposed into exact deterministic geometry without semantic cropping.", + "source": str(source), + "guide": str(guide), + "output": str(output), + "expected_node_count": len(nodes), + "detected_card_count": len(boxes), + "matched_node_count": len(matches), + "pre_normalization_alignment": { + "passed": pre_passed, + "max_center_drift": round(max(float(item["center_drift"]) for item in matches), 6), + "min_iou": round(min(float(item["iou"]) for item in matches), 6), + "max_width_relative_error": round(max(float(item["width_relative_error"]) for item in matches), 6), + "max_height_relative_error": round(max(float(item["height_relative_error"]) for item in matches), 6), + }, + "post_normalization_alignment": { + "passed": True, + "node_geometry_source": "semantic_plan_exact_bbox", + "unique_node_card_assignment": len({item["node_id"] for item in matches}) == len(nodes) + and len({item["detected_box_index"] for item in matches}) == len(nodes), + "connector_corridors_restored": True, + "semantic_crop_used": False, + }, + "nodes": normalized_items, + "passed": bool( + len(boxes) == len(nodes) + and len(normalized_items) == len(nodes) + and len({item["node_id"] for item in matches}) == len(nodes) + and len({item["detected_box_index"] for item in matches}) == len(nodes) + ), + } + if report_path: + write_json(report_path, report) + return report + + def compose_semantic_overlay( source_path: str | Path, semantic_plan: dict[str, Any], diff --git a/rfs/paper_to_image/workflow.py b/rfs/paper_to_image/workflow.py index 6f00080..cf01473 100644 --- a/rfs/paper_to_image/workflow.py +++ b/rfs/paper_to_image/workflow.py @@ -42,6 +42,7 @@ def run_paper_to_image( resume_candidates: bool = False, require_visual_enrichment: bool = True, deterministic_overlay: bool = True, + editable_overlay_ppt: bool = False, ) -> dict: started = time.time() effective_planner_model = planner_model or os.getenv("RFS_FAST_FRAMEWORK_MODEL", "").strip() or "gemini-2.5-flash" @@ -127,6 +128,8 @@ def run_paper_to_image( selected_template["semantic_plan"] = blueprint_report["semantic_plan"] overlay_spec = read_json(root / "overlay_spec.json") overlay_enabled = bool(deterministic_overlay and selected_template.get("semantic_plan", {}).get("applied") and not repair_source) + if editable_overlay_ppt and not overlay_enabled: + raise RuntimeError("Editable visual-substrate PPT requires an applied deterministic semantic overlay and cannot use an external baked repair source") generation_blueprint = root / "layout_blueprint.png" if overlay_enabled: generation_blueprint = root / "visual_substrate_blueprint.png" @@ -166,6 +169,8 @@ def run_paper_to_image( "ocr_engine": ocr_engine, "ocr_lang": ocr_lang, "deterministic_overlay": overlay_enabled, + "visual_substrate_geometry_normalization": overlay_enabled, + "editable_overlay_ppt": bool(editable_overlay_ppt), "generation_blueprint": str(generation_blueprint), } write_json(root / "generation_parameters.json", generation_parameters) @@ -195,9 +200,34 @@ def run_paper_to_image( overlay_spec=overlay_spec, deterministic_overlay=overlay_enabled, ) + editable_overlay = None + if editable_overlay_ppt: + selected_id = generation.get("selected_candidate_id") + candidates_by_id = { + str(item.get("candidate_id")): item + for item in generation.get("candidates", []) + if isinstance(item, dict) and item.get("candidate_id") + } + selected_record = candidates_by_id.get(str(selected_id)) if selected_id else None + if selected_record is None and candidates_by_id: + selected_record = max(candidates_by_id.values(), key=lambda item: float(item.get("score") or 0.0)) + substrate_path = Path(str(selected_record.get("visual_substrate_path") or "")).resolve() if isinstance(selected_record, dict) and selected_record.get("visual_substrate_path") else Path() + if not substrate_path.is_file(): + raise RuntimeError("Editable visual-substrate PPT requires the selected candidate normalized visual substrate") + from .editable_ppt import compile_visual_substrate_ppt + + editable_overlay = compile_visual_substrate_ppt( + plan["figure_specification"], + substrate_path, + root, + aspect_ratio=preferences["aspect_ratio"], + title=str(plan.get("paper_summary", {}).get("title") or "").strip() or None, + semantic_plan=selected_template["semantic_plan"], + overlay_spec=overlay_spec, + ) elapsed = round(time.time() - started, 3) run_summary: dict[str, Any] = { - "summary": "Paper-to-image workflow completed without PPTX generation.", + "summary": "Paper-to-image workflow completed with a geometry-normalized visual substrate and optional native editable PPT structure layer.", "ok": True, "out_dir": str(root), "paper": archived_paper, @@ -219,8 +249,10 @@ def run_paper_to_image( "repair_source": str(Path(repair_source).resolve()) if repair_source else None, "resume_candidates": bool(resume_candidates), "deterministic_overlay": overlay_enabled, + "editable_overlay_ppt": bool(editable_overlay_ppt), + "editable_composition": editable_overlay.get("pptx") if isinstance(editable_overlay, dict) else None, "elapsed_seconds": elapsed, - "pptx_generated": False, + "pptx_generated": bool(editable_overlay and editable_overlay.get("ok")), "artifacts": [ "input_manifest.json", "preferences.json", @@ -234,7 +266,8 @@ def run_paper_to_image( "template_profiles/", "selected_template.json", "layout_blueprint.png", - *( ["visual_substrate_blueprint.png", "visual_substrate_blueprint.json", "overlays/"] if overlay_enabled else [] ), + *( ["visual_substrate_blueprint.png", "visual_substrate_blueprint.json", "geometry/", "overlays/"] if overlay_enabled else [] ), + *( ["figure_program.json", "editable_composition.pptx", "editable_overlay_report.json", "composition_quality_report.json"] if editable_overlay else [] ), "paper_summary.json", "figure_specification.json", "design_plan.json", @@ -259,4 +292,6 @@ def run_paper_to_image( ], } write_json(root / "run_summary.json", run_summary) + if editable_overlay: + run_summary["editable_overlay"] = editable_overlay return run_summary diff --git a/tests/integration/test_paper_to_image.py b/tests/integration/test_paper_to_image.py index e45ae2e..09191fd 100644 --- a/tests/integration/test_paper_to_image.py +++ b/tests/integration/test_paper_to_image.py @@ -5,6 +5,7 @@ import json import tempfile import unittest +import zipfile from pathlib import Path from unittest.mock import patch @@ -124,6 +125,37 @@ def test_cli_production_defaults(self): self.assertEqual(args.domain_profile, "auto") self.assertEqual(args.template, "auto") self.assertEqual(args.repair_rounds, 3) + self.assertFalse(args.editable_overlay_ppt) + + def test_offline_workflow_can_compile_raw_substrate_to_editable_ppt_overlay(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paper = root / "paper.md" + paper.write_text(PAPER_TEXT, encoding="utf-8") + out = root / "run" + + result = run_paper_to_image( + paper=paper, + out=out, + planner_mode="heuristic", + asset_mode="placeholder", + candidates=1, + aspect_ratio="3:2", + review_mode="heuristic", + repair_rounds=0, + ocr_engine="off", + editable_overlay_ppt=True, + ) + + self.assertTrue(result["pptx_generated"]) + self.assertTrue(result["editable_overlay_ppt"]) + self.assertTrue((out / "editable_composition.pptx").exists()) + self.assertTrue((out / "editable_overlay_report.json").exists()) + with zipfile.ZipFile(out / "editable_composition.pptx") as archive: + slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") + self.assertIn("RFS Image2 Visual Substrate", slide_xml) + self.assertIn("RFS Semantic Label", slide_xml) + self.assertIn("RFS Connector", slide_xml) def test_image2_native_canvas_ratio_normalization(self): self.assertEqual(native_image2_aspect_ratio("16:9"), "3:2") @@ -368,7 +400,10 @@ def test_candidate_generation_reviews_deterministically_composed_overlay(self): candidate = result["candidates"][0] self.assertTrue(result["deterministic_overlay"]) self.assertTrue(Path(candidate["raw_path"]).exists()) + self.assertTrue(Path(candidate["visual_substrate_path"]).exists()) self.assertTrue(Path(candidate["path"]).exists()) + self.assertTrue(candidate["geometry_alignment_report"]["passed"]) + self.assertTrue(candidate["review"]["substrate_geometry"]["passed"]) self.assertTrue(candidate["overlay_report"]["passed"]) self.assertEqual(candidate["overlay_report"]["label_count"], 2) self.assertEqual(candidate["overlay_report"]["connector_count"], 1) @@ -1366,10 +1401,27 @@ def critic(path, blueprint, prompt): self.assertIn("generation_seconds", repair["timings"]) self.assertIn("review_seconds", repair["timings"]) - @patch("rfs.paper_to_image.generator.requests.post", return_value=FakeResponse()) + @patch("rfs.paper_to_image.generator.requests.post") @patch("rfs.paper_to_image.workflow.validate_review_coverage", return_value={"ok": True, "errors": [], "warnings": []}) @patch("rfs.paper_to_image.planner.call_vlm_json") def test_mock_full_production_workflow(self, planner_call, _coverage, _post): + def echo_generation_blueprint(_url, **kwargs): + payload = base64.b64encode(kwargs["files"]["image"][1].read()).decode("ascii") + + class BlueprintResponse: + status_code = 200 + + @staticmethod + def raise_for_status(): + return None + + @staticmethod + def json(): + return {"data": [{"b64_json": payload}]} + + return BlueprintResponse() + + _post.side_effect = echo_generation_blueprint fact = {"id": "fact_01", "statement": "Evidence-grounded reasoning", "status": "required", "importance": "critical", "confidence": 1.0, "evidence_ids": ["E0001"], "must_appear_in_figure": True, "visual_role": "module"} planner_call.return_value = {"summary": "Planning result.", **_plan()} with tempfile.TemporaryDirectory() as temp, patch.dict("os.environ", {"API_BASE": "https://example.test/v1", "API_KEY": "known-test-secret", "RFS_IMAGE_MODEL": "image-2", "RFS_CACHE_DIR": temp}, clear=False): diff --git a/tests/unit/test_overlay_renderer.py b/tests/unit/test_overlay_renderer.py index 18c2090..51e0733 100644 --- a/tests/unit/test_overlay_renderer.py +++ b/tests/unit/test_overlay_renderer.py @@ -5,10 +5,10 @@ import unittest from pathlib import Path -from PIL import Image, ImageChops +from PIL import Image, ImageChops, ImageDraw from rfs.paper_to_image.generator import _repair_prompt -from rfs.paper_to_image.overlay_renderer import compose_semantic_overlay, render_visual_substrate_blueprint +from rfs.paper_to_image.overlay_renderer import compose_semantic_overlay, normalize_visual_substrate_geometry, render_visual_substrate_blueprint from rfs.paper_to_image.planner import compile_image_prompt from rfs.paper_to_image.preparation import build_overlay_spec from rfs.paper_to_image.semantic_blueprint import compile_semantic_blueprint @@ -35,6 +35,38 @@ def _specification() -> dict: class OverlayRendererTests(unittest.TestCase): + @staticmethod + def _drifted_visual_cards(semantic: dict, path: Path, *, omit_last: bool = False, duplicate_first: bool = False) -> None: + width, height = 900, 600 + image = Image.new("RGB", (width, height), "white") + draw = ImageDraw.Draw(image) + nodes = [item for item in semantic["nodes"] if isinstance(item, dict)] + if omit_last: + nodes = nodes[:-1] + for index, node in enumerate(nodes): + bbox = node["bbox_percent"] + center_x = (float(bbox["x"]) + float(bbox["w"]) / 2) * width + center_y = (float(bbox["y"]) + float(bbox["h"]) / 2) * height + card_w = float(bbox["w"]) * width * 1.22 + card_h = float(bbox["h"]) * height * 1.45 + shift_x = (-1 if index % 2 else 1) * width * 0.018 + shift_y = (-1 if index % 3 else 1) * height * 0.012 + box = ( + round(center_x - card_w / 2 + shift_x), + round(center_y - card_h / 2 + shift_y), + round(center_x + card_w / 2 + shift_x), + round(center_y + card_h / 2 + shift_y), + ) + draw.rounded_rectangle(box, radius=12, fill=(105 + index * 20, 155, 205 - index * 15), outline=(31, 80, 115), width=4) + separator = round(box[1] + (box[3] - box[1]) * 0.22) + draw.line((box[0] + 4, separator, box[2] - 4, separator), fill=(25, 70, 95), width=4) + draw.ellipse((box[0] + 18, separator + 12, box[0] + 55, separator + 49), fill=(235, 185, 72), outline=(40, 60, 70), width=3) + if duplicate_first and nodes: + left = round(width * 0.43) + top = round(height * 0.10) + draw.rounded_rectangle((left, top, left + 105, top + 85), radius=10, fill=(160, 130, 200), outline=(40, 40, 80), width=4) + image.save(path) + def test_visual_substrate_contains_geometry_but_no_text_or_connectors(self): semantic = compile_semantic_blueprint(_specification()) with tempfile.TemporaryDirectory() as temp: @@ -79,6 +111,58 @@ def test_overlay_composes_every_exact_label_and_semantic_connector_path(self): with Image.open(substrate) as before, Image.open(output) as after: self.assertIsNotNone(ImageChops.difference(before.convert("RGB"), after.convert("RGB")).getbbox()) + def test_visual_substrate_normalization_restores_exact_semantic_geometry_without_crop(self): + semantic = compile_semantic_blueprint(_specification()) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + guide = root / "guide.png" + raw = root / "raw.png" + normalized = root / "normalized.png" + report_path = root / "geometry.json" + render_visual_substrate_blueprint(semantic, guide, width=900, height=600) + self._drifted_visual_cards(semantic, raw) + + report = normalize_visual_substrate_geometry(raw, guide, semantic, normalized, report_path=report_path) + + self.assertTrue(normalized.exists()) + self.assertFalse(report["pre_normalization_alignment"]["passed"]) + self.assertTrue(report["post_normalization_alignment"]["passed"]) + self.assertTrue(report["post_normalization_alignment"]["unique_node_card_assignment"]) + self.assertFalse(report["post_normalization_alignment"]["semantic_crop_used"]) + self.assertEqual(report["matched_node_count"], len(semantic["nodes"])) + self.assertTrue(all(item["fit_policy"] == "contain_no_crop" for item in report["nodes"])) + self.assertTrue(all(not item["semantic_crop_used"] for item in report["nodes"])) + self.assertTrue(all(not item["background_trim"].get("semantic_crop_used", False) for item in report["nodes"])) + saved = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual(saved["post_normalization_alignment"]["node_geometry_source"], "semantic_plan_exact_bbox") + + def test_visual_substrate_normalization_rejects_missing_or_extra_cards(self): + semantic = compile_semantic_blueprint(_specification()) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + guide = root / "guide.png" + render_visual_substrate_blueprint(semantic, guide, width=900, height=600) + for name, options in (("missing", {"omit_last": True}), ("extra", {"duplicate_first": True})): + with self.subTest(name=name): + raw = root / f"{name}.png" + self._drifted_visual_cards(semantic, raw, **options) + with self.assertRaisesRegex(ValueError, "exact cardinality is required"): + normalize_visual_substrate_geometry(raw, guide, semantic, root / f"{name}_normalized.png") + + def test_visual_substrate_normalization_rejects_ambiguous_node_card_matching(self): + semantic = compile_semantic_blueprint(_specification()) + ambiguous = json.loads(json.dumps(semantic)) + ambiguous["nodes"][1]["bbox_percent"] = dict(ambiguous["nodes"][0]["bbox_percent"]) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + guide = root / "guide.png" + raw = root / "raw.png" + render_visual_substrate_blueprint(semantic, guide, width=900, height=600) + self._drifted_visual_cards(semantic, raw) + + with self.assertRaisesRegex(ValueError, "ambiguous"): + normalize_visual_substrate_geometry(raw, guide, ambiguous, root / "normalized.png") + def test_overlay_prompt_forbids_image_model_text_and_arrows(self): specification = _specification() plan = { diff --git a/tests/unit/test_semantic_ppt.py b/tests/unit/test_semantic_ppt.py index 70bf75a..2e68ee9 100644 --- a/tests/unit/test_semantic_ppt.py +++ b/tests/unit/test_semantic_ppt.py @@ -4,7 +4,9 @@ import zipfile from pathlib import Path -from rfs.paper_to_image.editable_ppt import build_semantic_figure_program, compile_semantic_ppt +from PIL import Image + +from rfs.paper_to_image.editable_ppt import build_semantic_figure_program, build_visual_substrate_figure_program, compile_semantic_ppt, compile_visual_substrate_ppt class SemanticPptTests(unittest.TestCase): @@ -87,6 +89,56 @@ def test_long_words_and_relation_labels_are_compiled_for_readable_rendering(self self.assertLess(relation_label["bbox_percent"]["w"], 0.13) self.assertLess(relation_label["bbox_percent"]["y"], 0.5) + def test_image2_substrate_is_below_native_editable_labels_and_connectors(self): + specification = { + "topology": "linear", + "required_labels": ["Input Image", "Patch Encoder", "Prediction"], + "inputs": [{"id": "image", "name": "Input Image"}], + "modules": [{"id": "patch", "name": "Patch Encoder"}], + "outputs": [{"id": "prediction", "name": "Prediction"}], + "relations": [ + {"source": "image", "target": "patch", "type": "data_flow"}, + {"source": "patch", "target": "prediction", "type": "prediction"}, + ], + } + overlay = { + "labels": [ + {"target_id": "image", "text": "Input Image"}, + {"target_id": "patch", "text": "Patch Encoder"}, + {"target_id": "prediction", "text": "Prediction"}, + ], + "connectors": [ + {"source": "image", "target": "patch", "type": "data_flow"}, + {"source": "patch", "target": "prediction", "type": "prediction"}, + ], + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + substrate = root / "candidate_raw.png" + Image.new("RGB", (1600, 900), "#F7F8F5").save(substrate) + + program = build_visual_substrate_figure_program(specification, substrate, aspect_ratio="16:9", overlay_spec=overlay) + self.assertTrue(program["visual_substrate_mode"]) + self.assertTrue(all(node["fill_transparency"] == 1.0 for node in program["semantic_nodes"])) + self.assertTrue(all(node.get("label_bbox_percent") for node in program["semantic_nodes"])) + + report = compile_visual_substrate_ppt(specification, substrate, root, aspect_ratio="16:9", overlay_spec=overlay) + + self.assertTrue(report["ok"]) + self.assertFalse(report["scientific_text_baked_into_visual_layer"]) + self.assertFalse(report["connectors_baked_into_visual_layer"]) + with zipfile.ZipFile(report["pptx"]) as archive: + slide_xml = archive.read("ppt/slides/slide1.xml").decode("utf-8") + self.assertGreaterEqual(slide_xml.count(""), 1) + self.assertEqual(slide_xml.count("RFS Semantic Label"), 3) + self.assertGreaterEqual(slide_xml.count(""), 2) + for label in specification["required_labels"]: + self.assertIn(label, slide_xml) + + quality = json.loads((root / "composition_quality_report.json").read_text(encoding="utf-8")) + self.assertGreaterEqual(quality["background_visual"]["canvas_area_fill_percent"], 99.0) + self.assertTrue(all(item["label_rendered_as"] == "separate_editable_header_shape" for item in quality["semantic_nodes"])) + if __name__ == "__main__": unittest.main() From d8a7f5ef51e758a507fd5eed72a5f7c47dc2969c Mon Sep 17 00:00:00 2001 From: yiweiqin <3011291271@qq.com> Date: Tue, 25 Aug 2026 20:50:22 +0800 Subject: [PATCH 74/74] Integrate teammate rebuild audit architecture --- .gitignore | 11 + AGENTS.md | 326 +++ README.md | 146 +- docs/architecture.md | 36 +- docs/code-structure.md | 68 + docs/gpt-reproduction-workflow.md | 3 +- docs/judge-shadow-deployment.md | 42 + docs/judge_model_training_plan.md | 456 ++++ docs/module-map-and-migration.md | 198 ++ .../paper-to-image-reference-learning-plan.md | 241 +++ docs/paper-to-image.md | 203 +- docs/rebuild-editable.md | 8 +- docs/rebuild-process-judge.md | 68 + docs/run-log-events.md | 67 + docs/target_evolution_pipeline_plan.md | 429 ++++ docs/workflow.md | 67 +- examples/banana_game.md | 22 +- .../legacy_reference_rebuilds/README.md | 4 +- ...build_autofigure_architecture_ai_assets.py | 1894 ----------------- .../rebuild_c60_editable.py | 551 ----- ...ild_virtual_interview_pipeline_editable.py | 550 ----- experiments/literature_review/README.md | 6 +- .../raw_text/diagrammergpt.txt | 1644 -------------- .../literature_review/raw_text/scidoc.txt | 1563 -------------- .../literature_review/raw_text/scirex.txt | 591 ----- .../raw_text/structured_ie.txt | 615 ------ .../literature_review/reviewer/README.md | 109 + .../literature_review/reviewer/organize.py | 168 ++ .../reviewer/push_to_github.py | 284 +++ .../literature_review/reviewer/server.py | 346 +++ .../literature_review/reviewer/static/app.js | 355 +++ .../reviewer/static/index.html | 52 + .../reviewer/static/style.css | 266 +++ .../reviewer/sync_classified.py | 148 ++ .../scripts/extract_layout_text.py | 55 - .../scripts/translate_papers.py | 191 -- pyproject.toml | 3 +- reference-figures/README.md | 50 + reference-figures/excellent/.gitkeep | 1 + reference-figures/negative/.gitkeep | 1 + reference-figures/user/.gitkeep | 1 + rfs/analysis/__init__.py | 12 +- rfs/analysis/rebuild/__init__.py | 16 + rfs/analysis/rebuild/controls.py | 832 ++++++++ rfs/analysis/rebuild/layout.py | 1017 +++++++++ rfs/analysis/rebuild/reference_text.py | 452 ++++ rfs/arrow_router.py | 708 +----- rfs/asset_generator.py | 975 +-------- rfs/asset_reviewer.py | 74 +- rfs/cli.py | 477 ++--- rfs/cli_commands/__init__.py | 5 + rfs/cli_commands/categories.py | 36 + rfs/cli_commands/experimental.py | 65 + rfs/cli_commands/maintenance.py | 80 + rfs/cli_commands/product.py | 139 ++ rfs/cli_commands/rebuild.py | 144 ++ rfs/composition/__init__.py | 3 +- rfs/composition/pptx.py | 47 +- rfs/composition/preview.py | 17 +- rfs/composition/rebuild/__init__.py | 7 + rfs/composition/rebuild/arrows.py | 707 ++++++ rfs/composition/rebuild/export.py | 112 + rfs/composition/rebuild/program.py | 231 ++ rfs/composition/rebuild/text.py | 469 ++++ rfs/contracts/semantic.py | 753 ++++++- rfs/control_localizer.py | 620 +++++- rfs/editable_rebuild.py | 1152 +--------- rfs/evaluation/__init__.py | 8 +- rfs/evaluation/outputs.py | 5 + rfs/evaluation/rebuild_coverage.py | 94 + rfs/evaluation/rebuild_visual.py | 335 +++ rfs/evaluation/validator.py | 721 +++++++ rfs/evolution_contracts.py | 162 ++ rfs/exporter.py | 156 +- rfs/generation/__init__.py | 15 +- rfs/generation/assets.py | 974 +++++++++ rfs/generation/rebuild_assets.py | 167 ++ rfs/generation/rebuild_policy.py | 138 ++ rfs/input_archive.py | 8 +- rfs/judge_training/__init__.py | 8 + rfs/judge_training/dataset_builder.py | 132 ++ rfs/judge_training/evaluator.py | 56 + rfs/judge_training/evidence_builder.py | 174 ++ rfs/judge_training/registry.py | 75 + rfs/judge_training/schemas.py | 54 + rfs/judge_training/select_frameworks.py | 84 + rfs/layout_planner.py | 250 +-- rfs/paper_to_image/__init__.py | 20 +- rfs/paper_to_image/analyzer.py | 287 ++- rfs/paper_to_image/critics.py | 980 ++++++++- rfs/paper_to_image/document_cache.py | 2 +- rfs/paper_to_image/evolution.py | 97 + rfs/paper_to_image/exemplar_selector.py | 221 ++ rfs/paper_to_image/generator.py | 941 +++++++- rfs/paper_to_image/overlay_renderer.py | 19 +- rfs/paper_to_image/planner.py | 1031 ++++++++- rfs/paper_to_image/preference_feedback.py | 225 ++ rfs/paper_to_image/preparation.py | 1806 +++++++++++++++- rfs/paper_to_image/reference_learning.py | 316 +++ rfs/paper_to_image/review_contract.py | 47 +- rfs/paper_to_image/semantic_blueprint.py | 291 ++- rfs/paper_to_image/templates.py | 665 +++++- rfs/paper_to_image/user_style_learning.py | 180 ++ rfs/paper_to_image/workflow.py | 324 ++- rfs/planning/__init__.py | 4 +- rfs/planning/rebuild/__init__.py | 8 + rfs/planning/rebuild/design.py | 373 ++++ rfs/planning/rebuild/segmentation.py | 225 ++ rfs/planning/rebuild/semantics.py | 194 ++ rfs/professional/__init__.py | 28 + rfs/professional/compiler.py | 171 ++ rfs/professional/dsl.py | 393 ++++ rfs/professional/gap.py | 93 + rfs/professional/planner.py | 154 ++ rfs/professional/rebuild.py | 521 +++++ rfs/professional/repair.py | 114 + rfs/professional_compiler.py | 159 +- rfs/professional_dsl.py | 238 +-- rfs/professional_gap.py | 94 +- rfs/professional_rebuild.py | 366 +--- rfs/professional_repair.py | 104 +- rfs/professional_script_planner.py | 147 +- rfs/program_builder.py | 194 +- rfs/prompt_planner.py | 216 +- rfs/providers/__init__.py | 4 +- rfs/providers/client.py | 121 ++ rfs/providers/rebuild.py | 13 + rfs/rebuild_design_planner.py | 361 +--- rfs/rebuild_eval.py | 4 +- rfs/rebuild_evolution.py | 205 ++ rfs/rebuild_evolution_contracts.py | 81 + rfs/rebuild_judge.py | 136 ++ rfs/rebuild_process_judge.py | 218 ++ rfs/rebuild_vlm_adapters.py | 464 +++- rfs/rebuild_vlm_validation.py | 2 +- rfs/reference_analyzer.py | 236 +- rfs/reference_text_extractor.py | 366 +--- rfs/run_log.py | 150 ++ rfs/run_projection.py | 115 + rfs/stylist.py | 27 +- rfs/text_grouping.py | 2 +- rfs/text_layer.py | 470 +--- rfs/validator.py | 720 +------ rfs/visual_critic.py | 93 +- rfs/workflow.py | 12 +- rfs/workflows/__init__.py | 28 +- rfs/workflows/paper_to_editable.py | 76 +- rfs/workflows/professional_rebuild.py | 8 + rfs/workflows/rebuild_editable.py | 1563 ++++++++++++++ rfs/workflows/runtime.py | 45 + scripts/install-local.ps1 | 2 +- scripts/run_tests_with_project_temp.ps1 | 19 + skills/research-figure-studio/SKILL.md | 30 +- .../references/critic_stage.md | 188 +- .../references/journal_quality_checklist.md | 8 +- .../references/paper_to_figure.md | 4 +- .../references/stylist_stage.md | 101 +- .../references/visual_framework_figures.md | 331 +-- tests/integration/test_judge_data_cli.py | 24 + tests/integration/test_paper_to_editable.py | 162 ++ tests/integration/test_paper_to_image.py | 1273 ++++++++++- tests/integration/test_rebuild_editable.py | 88 +- tests/unit/test_asset_reviewer.py | 67 + tests/unit/test_card_color_status.py | 63 + tests/unit/test_control_localizer.py | 180 ++ tests/unit/test_evolution_contracts.py | 46 + tests/unit/test_exporter.py | 112 + tests/unit/test_full_image_route.py | 512 +++++ tests/unit/test_judge_training_dataset.py | 44 + tests/unit/test_judge_training_registry.py | 50 + tests/unit/test_overlay_renderer.py | 43 +- tests/unit/test_panelwise_layout.py | 82 + tests/unit/test_paper_analyzer.py | 161 +- tests/unit/test_paper_contract.py | 775 ++++++- tests/unit/test_paper_native_framework.py | 134 ++ tests/unit/test_paper_to_editable_route.py | 94 +- tests/unit/test_ppt_compiler.py | 47 + tests/unit/test_rebuild_evolution.py | 90 + .../unit/test_rebuild_evolution_contracts.py | 32 + tests/unit/test_rebuild_failure_log.py | 49 + tests/unit/test_rebuild_from_stage.py | 76 + tests/unit/test_rebuild_judge.py | 85 + tests/unit/test_rebuild_process_judge.py | 90 + tests/unit/test_rebuild_visual_critic.py | 32 + tests/unit/test_run_log.py | 130 ++ tests/unit/test_run_projection.py | 90 + tests/unit/test_semantic_blueprint.py | 116 + tests/unit/test_semantic_contract.py | 274 +++ tests/unit/test_text_layer_ocr.py | 52 + tests/unit/test_visual_critic.py | 51 + tests/unit/test_vlm_client.py | 33 + tests/unit/test_whole_image_evolution.py | 43 + webapp/README.md | 115 + webapp/__init__.py | 9 + webapp/app.py | 397 ++++ webapp/nodes.py | 440 ++++ webapp/requirements.txt | 3 + webapp/run_server.py | 100 + webapp/runner.py | 660 ++++++ webapp/smoke_test.py | 154 ++ webapp/static/app.js | 690 ++++++ webapp/static/editor.css | 177 ++ webapp/static/editor.js | 702 ++++++ webapp/static/index.html | 231 ++ webapp/static/review.css | 1 + webapp/static/review.html | 1 + webapp/static/review.js | 1 + webapp/static/style.css | 546 +++++ webapp/utils.py | 13 + 209 files changed, 36014 insertions(+), 15482 deletions(-) create mode 100644 AGENTS.md create mode 100644 docs/code-structure.md create mode 100644 docs/judge-shadow-deployment.md create mode 100644 docs/judge_model_training_plan.md create mode 100644 docs/module-map-and-migration.md create mode 100644 docs/paper-to-image-reference-learning-plan.md create mode 100644 docs/rebuild-process-judge.md create mode 100644 docs/run-log-events.md create mode 100644 docs/target_evolution_pipeline_plan.md delete mode 100644 experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py delete mode 100644 experiments/legacy_reference_rebuilds/rebuild_c60_editable.py delete mode 100644 experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py delete mode 100644 experiments/literature_review/raw_text/diagrammergpt.txt delete mode 100644 experiments/literature_review/raw_text/scidoc.txt delete mode 100644 experiments/literature_review/raw_text/scirex.txt delete mode 100644 experiments/literature_review/raw_text/structured_ie.txt create mode 100644 experiments/literature_review/reviewer/README.md create mode 100644 experiments/literature_review/reviewer/organize.py create mode 100644 experiments/literature_review/reviewer/push_to_github.py create mode 100644 experiments/literature_review/reviewer/server.py create mode 100644 experiments/literature_review/reviewer/static/app.js create mode 100644 experiments/literature_review/reviewer/static/index.html create mode 100644 experiments/literature_review/reviewer/static/style.css create mode 100644 experiments/literature_review/reviewer/sync_classified.py delete mode 100644 experiments/literature_review/scripts/extract_layout_text.py delete mode 100644 experiments/literature_review/scripts/translate_papers.py create mode 100644 reference-figures/README.md create mode 100644 reference-figures/excellent/.gitkeep create mode 100644 reference-figures/negative/.gitkeep create mode 100644 reference-figures/user/.gitkeep create mode 100644 rfs/analysis/rebuild/__init__.py create mode 100644 rfs/analysis/rebuild/controls.py create mode 100644 rfs/analysis/rebuild/layout.py create mode 100644 rfs/analysis/rebuild/reference_text.py create mode 100644 rfs/cli_commands/__init__.py create mode 100644 rfs/cli_commands/categories.py create mode 100644 rfs/cli_commands/experimental.py create mode 100644 rfs/cli_commands/maintenance.py create mode 100644 rfs/cli_commands/product.py create mode 100644 rfs/cli_commands/rebuild.py create mode 100644 rfs/composition/rebuild/__init__.py create mode 100644 rfs/composition/rebuild/arrows.py create mode 100644 rfs/composition/rebuild/export.py create mode 100644 rfs/composition/rebuild/program.py create mode 100644 rfs/composition/rebuild/text.py create mode 100644 rfs/evaluation/outputs.py create mode 100644 rfs/evaluation/rebuild_coverage.py create mode 100644 rfs/evaluation/validator.py create mode 100644 rfs/evolution_contracts.py create mode 100644 rfs/generation/assets.py create mode 100644 rfs/generation/rebuild_assets.py create mode 100644 rfs/generation/rebuild_policy.py create mode 100644 rfs/judge_training/__init__.py create mode 100644 rfs/judge_training/dataset_builder.py create mode 100644 rfs/judge_training/evaluator.py create mode 100644 rfs/judge_training/evidence_builder.py create mode 100644 rfs/judge_training/registry.py create mode 100644 rfs/judge_training/schemas.py create mode 100644 rfs/judge_training/select_frameworks.py create mode 100644 rfs/paper_to_image/evolution.py create mode 100644 rfs/paper_to_image/exemplar_selector.py create mode 100644 rfs/paper_to_image/preference_feedback.py create mode 100644 rfs/paper_to_image/reference_learning.py create mode 100644 rfs/paper_to_image/user_style_learning.py create mode 100644 rfs/planning/rebuild/__init__.py create mode 100644 rfs/planning/rebuild/design.py create mode 100644 rfs/planning/rebuild/segmentation.py create mode 100644 rfs/planning/rebuild/semantics.py create mode 100644 rfs/professional/__init__.py create mode 100644 rfs/professional/compiler.py create mode 100644 rfs/professional/dsl.py create mode 100644 rfs/professional/gap.py create mode 100644 rfs/professional/planner.py create mode 100644 rfs/professional/rebuild.py create mode 100644 rfs/professional/repair.py create mode 100644 rfs/providers/client.py create mode 100644 rfs/providers/rebuild.py create mode 100644 rfs/rebuild_evolution.py create mode 100644 rfs/rebuild_evolution_contracts.py create mode 100644 rfs/rebuild_judge.py create mode 100644 rfs/rebuild_process_judge.py create mode 100644 rfs/run_log.py create mode 100644 rfs/run_projection.py create mode 100644 rfs/workflows/professional_rebuild.py create mode 100644 rfs/workflows/rebuild_editable.py create mode 100644 rfs/workflows/runtime.py create mode 100644 scripts/run_tests_with_project_temp.ps1 create mode 100644 tests/integration/test_judge_data_cli.py create mode 100644 tests/integration/test_paper_to_editable.py create mode 100644 tests/unit/test_asset_reviewer.py create mode 100644 tests/unit/test_card_color_status.py create mode 100644 tests/unit/test_evolution_contracts.py create mode 100644 tests/unit/test_exporter.py create mode 100644 tests/unit/test_full_image_route.py create mode 100644 tests/unit/test_judge_training_dataset.py create mode 100644 tests/unit/test_judge_training_registry.py create mode 100644 tests/unit/test_panelwise_layout.py create mode 100644 tests/unit/test_paper_native_framework.py create mode 100644 tests/unit/test_rebuild_evolution.py create mode 100644 tests/unit/test_rebuild_evolution_contracts.py create mode 100644 tests/unit/test_rebuild_failure_log.py create mode 100644 tests/unit/test_rebuild_from_stage.py create mode 100644 tests/unit/test_rebuild_judge.py create mode 100644 tests/unit/test_rebuild_process_judge.py create mode 100644 tests/unit/test_run_log.py create mode 100644 tests/unit/test_run_projection.py create mode 100644 tests/unit/test_visual_critic.py create mode 100644 tests/unit/test_whole_image_evolution.py create mode 100644 webapp/README.md create mode 100644 webapp/__init__.py create mode 100644 webapp/app.py create mode 100644 webapp/nodes.py create mode 100644 webapp/requirements.txt create mode 100644 webapp/run_server.py create mode 100644 webapp/runner.py create mode 100644 webapp/smoke_test.py create mode 100644 webapp/static/app.js create mode 100644 webapp/static/editor.css create mode 100644 webapp/static/editor.js create mode 100644 webapp/static/index.html create mode 100644 webapp/static/review.css create mode 100644 webapp/static/review.html create mode 100644 webapp/static/review.js create mode 100644 webapp/static/style.css create mode 100644 webapp/utils.py diff --git a/.gitignore b/.gitignore index 0045bb2..4a7edc3 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,17 @@ assets_raw/ *.tiff *.svg +# Local reference-learning figures; keep only repository-authored guides/placeholders +reference-figures/**/*.png +reference-figures/**/*.jpg +reference-figures/**/*.jpeg +reference-figures/**/*.webp +reference-figures/**/*.tif +reference-figures/**/*.tiff +reference-figures/**/*.pdf +!reference-figures/README.md +!reference-figures/**/.gitkeep + # Keep repository-authored docs/examples if explicitly tracked by extension overrides !docs/**/*.md !docs/assets/*.png diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d647158 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,326 @@ +# AGENTS.md + +## 必读:进入项目后的第一件事 + +任何代理进入本仓库后,先阅读项目核心思路文档 `docs/paperimage.docx`(「论文 → 文生框架图 → 可编辑框架图」整体思路与 Judge 模型训练目标的最新权威描述)。它是 Word 二进制文件,读取时请先抽取文本(解压 `word/document.xml` 并提取 `` 内容,或先转成纯文本)。该文档的优先级高于本文件,冲突时以用户最新意图为准。 + +## 协作身份 + +- 用户叫 Sen-illion。 +- 助手叫 Kira1。 +- 本机 git 可执行文件:`D:\Git\cmd\git.exe`(PATH 中可能没有 `git`,请始终用完整路径调用,例如 `& "D:\Git\cmd\git.exe" -C "D:\文件夹\Paperimage" status`)。 +- 本文件用于帮助后续进入 `C:\Users\User\Desktop\Paperimage` 的代理快速理解项目、遵守约定并安全修改代码。 + +## 项目一句话 + +Paperimage 是从 ResearchFigureStudio 分支 `codex/integrate-teammate-and-restructure` 拆出的公开仓库。它是一个 **PPTX-first 的论文到科研框架图生成与可编辑重建引擎**,同时按 Codex plugin 形态整理:论文语义合同负责科学内容,生成或参考图负责视觉布局,本地 deterministic Python 代码负责编译可编辑 PowerPoint。 + +项目重点不是生成一张扁平化大图,而是生成: + +- 可编辑的 `editable_composition.pptx` +- 槽级别 raster 图像资产或按策略跳过的可编辑对象 +- PPT 可编辑的标签、箭头、连接线、面板、公式和分组 +- 可验证的中间 JSON/Markdown 合同文件 +- 可由 Codex skill/插件继续复现的科研制图工作流 + +## 当前能力边界 + +- 当前是早期工程原型,强项是把多个小图像块稳定放入 PPTX 精确位置,并保留关键文本、箭头、面板和关系为可编辑对象。 +- 生成的图像块仍然是 raster asset,不是完全可编辑的科学矢量对象。 +- 不应把它宣传成端到端顶会级成图器;它更像一个方便人工继续修订的可编辑科研图生成管线。 +- 项目目前最重要的开放难题仍然是箭头和连接器定位,包括 source-target 绑定、多段路由、避让、虚线环和参考图逻辑保持。 +- Paperimage 额外保留了 `rebuild-editable-pro` 的 controlled professional DSL 路线,用于从参考图生成受控 DSL,再由本地代码编译为 PPT 合同;VLM 不应直接写任意 PowerPoint 代码。 +- 默认产品策略是 `smart-api`:reference crop 默认只作为 API 参考上下文,不作为最终 PPT 图像资产;只有显式启用 generation-policy / policy 实验模式时,才允许按槽保留 `reference_crop`。 + +## 技术栈 + +- Python 包名:`research-figure-studio` +- Python 要求:`>=3.10` +- CLI 入口:`rfs = rfs.cli:main` +- 主要依赖: + - `pillow` + - `python-pptx` + - `PyMuPDF` + - `requests` + - `opencv-python-headless` + - Windows 上可选/条件依赖 `pywin32` +- 可选 OCR 依赖:`paddleocr`、`easyocr`,通过 `python -m pip install -e ".[ocr]"` 安装。 +- Windows + 本机 PowerPoint 是最佳导出环境,因为 PDF/PNG 导出依赖 PowerPoint COM 和 PyMuPDF。 + +## 目录结构 + +- `.codex-plugin/plugin.json`:Codex plugin manifest。 +- `skills/research-figure-studio/`:随仓库提供的 Codex skill,描述科研制图和验证工作流。 +- `skills/research-figure-studio/scripts/validate_framework_outputs.py`:独立输出验证脚本。 +- `rfs/`:主 Python 包,仍保持兼容导入。 +- `rfs/cli.py`:命令行入口,提供 `doctor`、`make-framework`、`paper-to-image`、`paper-to-editable`、`rebuild-editable`、`rebuild-editable-eval`、`rebuild-editable-pro`、`rebuild-judge`、`rebuild-evolve`、`coevolve-image`、`coevolution-report`、`presentations-qa`、`validate`。 +- `rfs/workflows/`:产品级 workflow 入口;`paper_to_editable.py` 是 paper-to-image + editable rebuild 的组合路径。 +- `rfs/contracts/`:稳定合同与 semantic binding;`semantic.py` 将 paper review / figure specification 重新绑定到 PPT 文本和关系。 +- `rfs/composition/`:PowerPoint 编译与 preview renderer;`rfs/ppt_compiler.py` 是兼容 wrapper。 +- `rfs/evaluation/`:deterministic rebuild 质量检查。 +- `rfs/paper_to_image/`:论文解析、paper review、模板、blueprint、候选整图生成和候选审查。 +- `rfs/editable_rebuild.py`:reference-only 可编辑 PPT rebuild 流水线。 +- `rfs/rebuild_design_planner.py`:rebuild-editable 的全局参考图理解阶段,生成 narrative、layer plan、generation policy 和 flow graph。 +- `rfs/layout_planner.py`、`rfs/control_localizer.py`、`rfs/layout_semantic_planner.py`:rebuild-editable 使用的布局、控件和语义规划模块。 +- `rfs/rebuild_vlm_adapters.py`、`rfs/rebuild_vlm_validation.py`、`rfs/rebuild_eval.py`、`rfs/vlm_client.py`:rebuild 工作流的 VLM 调用、输出验证和 heuristic-vs-VLM 评估支持。 +- `rfs/professional_*.py`:`rebuild-editable-pro` controlled professional DSL 工作流。 +- `rfs/coevolution/`:whole-image Creator/Judge co-evolution 实验基线和指标汇总。 +- `rfs/run_log.py`:append-only 运行事件日志(`run_log.jsonl`),状态 = fold(事件);支持续写 seq 与 `last_event`。 +- `rfs/run_projection.py`:把运行日志投影成「给人看的 `run_report.md`」和「给 Judge 的过程轨迹」。 +- `rfs/rebuild_process_judge.py`:过程监督 Judge(prompt-as-judge),按阶段归因输出 `stage_feedback`。 +- `rfs/rebuild_judge.py`:把 Judge 的 fault 阶段映射成粗修复计划(reanalyze/regen_assets/recompile)。 +- `rfs/rebuild_evolution.py`:自动分块进化循环(重建 → 判 → 修 → 再判)。 +- `tests/unit/`、`tests/integration/`、`tests/e2e/`:按层级组织的测试。 +- `benchmarks/`:产品验收用例和指标说明。 +- `experiments/`:研究原型、legacy rebuild 脚本和文献综述材料。 +- `webapp/`:本地 Web Studio 前端(论文/参考图 → 可编辑 PPTX 的可视化运行页),`webapp/run_server.py` 启动并注入 `.env` API,运行节点按中间产物文件实时点亮,产物输出到 `output/webapp_runs/`(被 gitignore)。 +- `docs/architecture.md`、`docs/workflow.md`、`docs/rebuild-editable.md`、`docs/paper-to-image.md`、`docs/gpt-reproduction-workflow.md`:架构、流程、重建入口和复现说明。 +- `docs/run-log-events.md`、`docs/rebuild-process-judge.md`:运行日志事件 schema 与过程监督 Judge 用法。 + +## 核心工作流 + +Paperimage 的主要路线: + +```text +paper-to-image: +paper -> paper_review.json / figure_specification.json -> +template profile / layout_blueprint.png -> +image candidates -> selected_image.png 或 engineering_preview.png + +rebuild-editable: +reference image -> global design plan / layer plan / generation policy / flow graph -> +reference_geometry.json / reference_controls.json -> +semantic binding / editable text layer / arrow routing -> +asset_generation_specs.json -> slot assets -> +figure_program.json -> editable_composition.pptx -> preview/QA/report + +paper-to-editable: +paper-to-image -> load paper semantic contract -> +rebuild-editable under editable/ -> paper_to_editable_result.json + +rebuild-editable-pro: +reference image -> baseline contracts -> controlled professional DSL -> +local compiler -> editable PPTX + professional reports / repairs +``` + +运行日志 + 过程监督(分块进化): + +```text +rebuild-editable -> run_log.jsonl / run_report.md -> +rebuild_process_judge -> stage_feedback -> +rebuild-judge / rebuild-evolve 粗修复(reanalyze/regen_assets/recompile)-> 再判 +``` + +重要原则: + +- 论文合同负责科学术语、模块含义、实体标签、关系和证据链接。 +- 参考图或生成图负责布局、视觉节奏、颜色和局部视觉风格。 +- 箭头、连接线、虚线环、面板框、标签、公式和关键文本必须是 PPT 可编辑对象,不应作为图像资产生成。 +- 禁止生成一张完整架构大图后裁剪或截图作为交付。 +- 禁止 semantic crop、cover-crop、crop-to-ratio;图像插入应是 contain/no-crop 语义。 +- `smart-api` 是默认产品策略;generation-policy / policy 是显式实验能力,必须在报告里记录每个槽的策略来源和执行结果。 +- OCR/VLM 失败时可以 heuristic fallback,但必须在报告中明示,不能伪装成高置信 OCR/VLM 结果。 + +## CLI 常用命令 + +安装开发包: + +```powershell +python -m pip install --upgrade pip +python -m pip install -e . +``` + +健康检查: + +```powershell +rfs doctor --json +``` + +离线 paper-to-image 烟测: + +```powershell +rfs paper-to-image ` + --paper "C:\path\paper.pdf" ` + --out "output\paper_to_image_placeholder" ` + --planner-mode heuristic ` + --asset-mode placeholder ` + --candidates 2 ` + --review-mode heuristic ` + --ocr-engine off ` + --json +``` + +reference-only 可编辑重建: + +```powershell +rfs rebuild-editable ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_check" ` + --design-plan-mode vlm ` + --asset-mode api ` + --asset-policy smart-api ` + --layout-mode hybrid ` + --control-mode hybrid ` + --text-mode ocr ` + --export-preview ` + --json +``` + +显式 generation-policy 实验模式: + +```powershell +rfs rebuild-editable ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_policy" ` + --asset-mode policy ` + --asset-policy generation-policy ` + --layout-mode hybrid ` + --control-mode hybrid ` + --json +``` + +professional DSL 重建: + +```powershell +rfs rebuild-editable-pro ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_pro" ` + --asset-mode api ` + --asset-policy smart-api ` + --repair-rounds 2 ` + --export-preview ` + --json +``` + +过程监督判定(需 API): + +```powershell +rfs rebuild-judge ` + --out "output\rebuild_check" ` + --repair-plan ` + --json +``` + +自动分块进化闭环(需 API): + +```powershell +rfs rebuild-evolve ` + --reference "C:\path\reference.png" ` + --out "output\rebuild_evolve" ` + --max-rounds 3 ` + --asset-mode api ` + --asset-policy smart-api ` + --layout-mode hybrid ` + --control-mode hybrid ` + --text-mode ocr ` + --export-preview ` + --json +``` + +## 环境变量与密钥 + +不要把 API key 写入源码、文档示例的真实值、测试夹具或输出目录。只使用环境变量: + +- `API_BASE` +- `API_KEY` +- `GEMINI_API_KEY` +- `GEMINI_GEN_IMG_URL` +- `MODEL_VLM` +- `RFS_VLM_TIMEOUT` +- `RFS_VLM_RETRIES` +- `RFS_REBUILD_LAYOUT_MODEL` +- `RFS_REBUILD_DESIGN_MODEL` +- `RFS_REBUILD_CONTROL_MODEL` +- `RFS_REBUILD_SEMANTIC_MODEL` +- `RFS_TEXT_GROUPING_MODEL` +- `RFS_CRITIC_MODEL` +- `RFS_ONLINE_JUDGE_MODEL` +- `RFS_FROZEN_JUDGE_MODEL` +- `RFS_IMAGE_MODEL` +- `RFS_IMAGE_EDIT_URL` +- `IMAGE_MODEL` + +## 开发与测试 + +本地开发检查: + +```powershell +python -m compileall -q rfs +python -m py_compile skills\research-figure-studio\scripts\validate_framework_outputs.py +python -m py_compile skills\research-figure-studio\scripts\estimate_asset_fill.py +python -m unittest discover -s tests -q +``` + +CI 在 Windows 上跑 Python 3.10 和 3.11,步骤包括: + +- editable install +- compile `rfs` +- py_compile skill scripts +- unittest +- 检查 bundled skill frontmatter + +运行项目时优先使用虚拟环境;仓库根目录 `.env` 里的 API 变量需要手动注入当前 shell,不要依赖代码自动读取密钥文件。 + +## 仓库卫生 + +不要提交: + +- `output/`、`outputs/` +- 论文、手稿、私有数据、用户参考图 +- 生成的 PPTX/PDF/PNG/JPG/SVG/TIFF/WebP 等图像或成图资产 +- `.env`、API key、私钥 +- `__pycache__/`、`.pytest_cache/`、`*.egg-info/` +- 本地虚拟环境,如 `.venv/` + +当前 `.gitignore` 已经覆盖多数生成物。注意 `AGENTS.md` 是需要保留的仓库说明文件,修改时不要误删用户内容。 + +## 修改代码时的建议 + +- 优先保持 Paperimage 的插件化和稳定包边界,不要把 `rfs/workflows`、`rfs/contracts`、`rfs/composition` 回退成旧的扁平结构。 +- 对 JSON/Markdown 输出继续使用 `rfs.utils.write_json` 和 `write_text`,保持 UTF-8 和 `ensure_ascii=False`。 +- 新增 artifact 时同步考虑 validator、docs、README 和 tests。 +- 修改输出合同相关代码后,优先补充 `tests/integration/test_validator.py` 或相应 workflow 测试。 +- VLM 只能产生结构化 JSON、布局建议、审查结果或 prompt 计划;不要让 VLM 直接写任意 PowerPoint 代码。 +- 真实 API 失败时不要静默切换到 placeholder 或 vector-only;应明确失败或让调用者选择 fallback。 +- 保持 labels、formulas、arrows、controls 在 PPT 层可编辑,图像资产只承载局部视觉块。 +- Presentations QA 是检查层,不是编译权威;不要让它反向改写 RFS 的 PPTX 合同。 + +## 长期结构门禁(防止代码结构退化) + +每次修改前必须先阅读 `docs/code-structure.md`,并按其中的 canonical +归属落代码。新业务逻辑只能放入以下生产包: + +- `rfs/contracts`:稳定语义合同; +- `rfs/analysis`:论文、参考图、OCR、布局和控件分析; +- `rfs/planning`:设计、布局和语义规划; +- `rfs/generation`:候选图、槽资产和资产策略; +- `rfs/composition`:figure program、PPTX、预览与导出; +- `rfs/evaluation`:validator、质量检查和 benchmark; +- `rfs/providers`:VLM/API、模型选择和重试; +- `rfs/workflows`:跨阶段产品编排; +- `rfs/professional`:controlled professional DSL。 + +强制规则: + +1. 根目录历史模块(如 `rfs.vlm_client`、`rfs.validator`、`rfs.editable_rebuild`、`rfs.professional_rebuild`)仅用于兼容导入;**禁止**新增业务逻辑、第三套实现或新的生产 import。 +2. `rfs/cli.py` 只负责总 parser、dispatch 和结果输出;新命令参数注册进入 `rfs/cli_commands/`,业务逻辑进入 workflow/领域包。 +3. 依赖方向必须保持为 `cli -> workflows -> contracts/analysis/planning/generation/composition/evaluation/providers`;禁止合同、编译或评估层反向依赖 CLI/workflow;评估层只能检查和报告,不能偷偷修写生产合同。 +4. 外部 API 调用进入 `rfs/providers`;实验代码仅进入 `rfs/coevolution`、`rfs/judge_training` 或 `experiments/`,不得被默认产品 workflow 隐式导入。 +5. 新文件必须有模块级 docstring、唯一职责和对应测试;同一职责已有 canonical 实现时,必须扩展它,不得复制一套实现。 +6. 单个改动不得混合“文件移动/结构迁移、算法重写、CLI/输出合同变更”。超过约 500 行的模块或约 100 行的函数,应优先拆分或在 commit 中说明必要性。 +7. 修改后至少运行: + +```powershell +python -m compileall -q rfs +python -m unittest discover -s tests -q +rfs doctor --json +``` + +涉及 CLI、兼容层、合同或 workflow 时,必须同时运行相应的 parser/compatibility/integration 测试。不得删除或绕过 `tests/unit/test_package_boundaries.py` 中的结构门禁;如确需调整门禁,必须同步更新 `docs/code-structure.md` 并在 commit 说明原因。 + +## 约定 +1. 每次进行修改后都要提交一次commit,提交简介简单概括本次修改内容,并且给出commit ID,目的是保证修改不满意可退回到上一次修改。 +2. 每次拉取新的代码先更新 `AGENTS.md` 的快速理解项目部分。 +3. 运行项目时在虚拟环境里运行,API在.env文件中,需要手动注入。 +4. 每次拉取或合并代码时,如果出现冲突,不要自行保守合并;先列出冲突文件、双方差异和推荐选项,询问 Sen-illion 后再决定保留哪个实现或如何组合。 +5. 如果有同时修改的内容,一次commit只要提交本次自己修改的内容,不要碰其他的修改。 diff --git a/README.md b/README.md index 6bfacd0..de75792 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,14 @@ ## Summary -ResearchFigureStudio is a paper-to-editable-PPT scientific figure engine and Codex plugin. Paper contracts provide exact labels and scientific relations; generated or user-provided images provide layout and visual style; deterministic code compiles the result into editable PowerPoint objects. +ResearchFigureStudio is a paper-to-editable-PPT scientific figure engine and Codex plugin. The primary route uses an evidence-grounded paper contract to ask Image2 for one complete scientific figure, including visible text, arrows, modules, and illustrations, then reconstructs that image into an editable PowerPoint composition. The current workflow is optimized for AI/ML/NLP system figures: - paper-grounded concept extraction -- reference-primary geometry, style, color, and flow alignment -- 25-50 non-arrow image slots -- `slot_visual_spec.json` for dense mini-scene/image-block planning +- complete whole-figure Image2 generation with exact visible terminology and directed relations +- VLM review of scientific faithfulness, text, topology, information density, and visual quality +- image-to-PPT reconstruction of panels, text, assets, and connectors - AutoFigure-inspired control candidates and overlays for arrow/source-target binding - reference-preserving arrow styling/routing reports for softer editable PPT connectors - reference-constrained orthogonal fallback routing for missing or explicitly fallback-allowed connectors @@ -17,22 +17,21 @@ The current workflow is optimized for AI/ML/NLP system figures: - multi-candidate image generation through placeholder, Gemini, or Yunwu image2-compatible APIs - deterministic PPTX composition with editable labels, panels, arrows, connectors, and formulas - optional Presentations-plugin QA for importing/rendering/inspecting the PPTX without mutating it -- strict validation for no single full diagram, no semantic crop, no vector-only fallback, low blank space, and non-trivial image-block complexity +- optional paper-native multi-asset experiments for comparison, not the default product route This repository does not include API keys, papers, reference images, generated outputs, or local run artifacts. ## Current Capability Boundary ResearchFigureStudio is an early engineering prototype. Its current practical -ability is limited to placing many small generated image blocks into precise -PowerPoint positions, then keeping surrounding labels, arrows, panels, formulas, -and grouping elements editable in PPTX. +route can generate a complete paper figure through Image2 and then reconstruct +the result into editable PowerPoint layers. The generated reference image remains +the visual source of truth for the reconstruction stage. -It does **not** yet solve the harder goal of generating truly publication-grade, -fully editable scientific figures end to end. The image blocks themselves are -still raster assets, not editable scientific vector objects. The system can -produce a PPTX composition that is easier to manually revise, but it should not -be treated as a finished top-tier-paper figure generator. +It does **not** yet guarantee perfect OCR, connector recovery, or fully editable +scientific vector objects for every figure. The system can produce a PPTX that +is easier to revise, but it should not yet be treated as a universally finished +top-tier-paper figure generator. ## Help Wanted @@ -79,17 +78,20 @@ Windows with PowerPoint installed gives the best PPTX/PDF/PNG export path. The P ## Paper To Editable PPTX -The main product route is: +The main product route is complete-image generation followed by precise reconstruction: ```powershell rfs paper-to-editable ` --paper "C:\path\paper.pdf" ` --out "output\paper_to_editable" ` - --positive-reference "C:\path\optional_style_reference.png" ` --json ``` -This writes paper-review and image-generation artifacts under `paper_to_image/`, then writes the semantic contract, editable figure program, PPTX, preview, and QA reports under `editable/`. If no image candidate passes the production gates, the workflow stops before creating an official editable deliverable. +This first prepares an evidence-grounded semantic contract under `paper_to_image/`. Image2 then generates the complete figure as one raster image, including visible terminology, arrows, modules, and visual explanations. The approved `selected_image.png` is passed to the image-to-PPT reconstruction stage under `editable/`, where OCR, layout parsing, connector localization, and the paper contract recover editable structure. + +`--editable-route rebuild` is the default and primary route. Use `--positive-reference` when supplied figures should influence style or layout. `--editable-route framework` remains an optional multi-asset experiment and is not the target product behavior. + +The first uncached Image2 generation and VLM review may take minutes. Contract, candidate, and reconstruction caches should be reused when their content fingerprints remain valid. For offline engineering validation only, add `--image-asset-mode placeholder --rebuild-asset-mode placeholder --allow-engineering-preview`. This does not create a production-approved result. @@ -105,7 +107,7 @@ rfs fast-framework-prompt ` --json ``` -This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and contracts are cached separately by paper hash. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. +This writes `paper.md`, `document_model.json`, `extraction_report.json`, `section_summary.md`, `key_evidence.json`, `figure_specification.json`, `contract_completion_report.json`, `image_prompt.md`, `overlay_spec.json`, and `run_report.json`. Fast mode gives the VLM a compact semantic-only task, compiles layout/style deterministically, and applies evidence-gated computer-paper primitives without checking paper names. The contract normalizer tolerates scalar/string entities and `source_id`/`target_id` relation variants, grounds declared terms only through exact paper evidence, repairs uniquely labeled endpoint aliases, and drops unresolved relations into explicit uncertainties instead of rendering dangling or guessed edges. When a VLM already supplies a rich graph, new deterministic entities must occur in an overview caption or connect directly to a VLM-declared entity through non-reference evidence. Credential-free heuristic mode seeds its graph from overview captions and expands only along evidence-supported connected method chains; if no overview seed exists, it selects the largest connected method component. This prevents Related Work examples and unrelated experimental branches from becoming visible framework nodes. Successful document extraction and raw semantic plans are cached separately by paper hash. Older compiled semantic caches are intentionally not reused because normalizing them again can duplicate or mutate relations; each affected paper needs one fresh VLM compilation, after which the versioned raw-plan cache is stable and repeatable. Every accepted cached string is passed through the current Unicode/mojibake repair before new artifacts are written. Set `RFS_FAST_FRAMEWORK_MODEL` to override the default `gemini-2.5-flash` model. Add `--editable-ppt` to compile the same evidence-backed semantic contract directly into `editable_composition.pptx` without image generation. Every visible module is a native PowerPoint shape with editable text, every relation is a native connector, and `figure_program.json`, `semantic_ppt_report.json`, and `composition_quality_report.json` remain available for auditing. This direct route is intended for fast structural drafts; generated icons or richer visual assets can be added later without changing the scientific graph. @@ -131,26 +133,27 @@ Use `paper-to-image` when the required endpoint is a generated raster framework figure, with an optional native editable PowerPoint structure layer. The production route performs a universal evidence-grounded paper review, loads a domain extension, converts positive references into content-free architecture templates, selects a template, -renders `layout_blueprint.png`, and uses Image2 edit to create and review three + renders `layout_blueprint.png`, and uses direct Image2 complete-figure generation to create and review three candidates. For evidence-grounded contracts with 2-16 visible nodes, a generic semantic compiler now writes ranked node boxes, distinct input/output ports, global obstacle-aware orthogonal paths, branch lanes, and outer feedback loops -into `layout_blueprint.json.semantic_plan`. Image2 receives a separate -`visual_substrate_blueprint.png` with no text or arrows. Because Image2 can move -or resize those cards, the compiler detects every generated card, requires an -exact one-to-one match with semantic nodes, and recomposes its header-free visual -content into `candidate_XX_substrate.png` at the exact semantic boxes without -cropping. It then adds exact paper labels and directed connectors, producing the -reviewed `candidate_XX.png` plus auditable geometry and overlay reports. Self-Refine feedback and SAM-style dense -multi-frame contracts retain stricter specialized layouts. It never invokes the -PPTX compiler unless `--editable-overlay-ppt` is requested; that option places -the normalized substrate below native editable node labels and connectors. + into `layout_blueprint.json.semantic_plan`. The semantic blueprint is embedded in + the generation Prompt, and Image2 is responsible for the final visible text, arrows, modules, +and visual content in `candidate_XX.png`. The critic checks the generated text +and connector topology against the paper contract. Self-Refine feedback and SAM-style dense +multi-frame contracts retain stricter specialized layouts. The command never +adds programmatic text or arrows after generation. `overlay_spec.json` is audit +metadata and a downstream image-to-PPT reconstruction contract only. The contract normalizer preserves semantic roles instead of checking labels alone. Explicit provenance clauses such as `CFP and OCT from MEH-MIDAS and public datasets` compile to `data source -> modality -> method`; multi-source phrases collapse into one evidence-backed source group; explicit internal/external evaluation outputs are restored; generic labels such as `foundation model` are replaced by an explicitly named paper model when the evidence provides the alias; and named techniques such as `masked autoencoder` become required innovations. Paper-contract caching is independent of canvas ratio and display-language preferences, so changing 16:9 to 3:2 cannot select a different scientific contract. -Production Image-2 review rejects a neat but empty text-only flowchart. The critic checks entity-role mismatches, focused provenance topology, mechanism visualization, visual information density, and publication polish. A deterministic blueprint-enrichment gate requires substantive visual content beyond gradients and shadows; configure its default `0.08` changed-pixel threshold with `RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO`. Generation prompts use a compact scientific contract so evidence, geometry, roles, and visual requirements remain explicit without repeating the full planning documents. +Production Image-2 review rejects a neat but empty text-only flowchart. The critic checks entity-role mismatches, focused provenance topology, mechanism visualization, visual information density, and publication polish. A deterministic blueprint-enrichment gate requires at least `0.10` active-region change beyond the bare blueprint. The detailed-tile gate defaults to `0.80`, with `0.75` for feedback figures whose functional return-loop corridors otherwise receive a false whitespace penalty. Configure both gates with `RFS_MIN_BLUEPRINT_ENRICHMENT_RATIO` and `RFS_MIN_ACTIVE_DETAIL_TILE_RATIO`. Density is saturated once the applicable gate passes: candidate selection and repair no longer reward packing an already adequate figure more tightly, and prompts forbid extra modules, charts, labels, or decorations added only to fill space. Large flat cards still do not count as useful content. + +Density failures now include a 24x16 active-region detail grid plus per-semantic-node 4x4 density measurements. Sparse named modules are copied into `repair_regions`, so localized Image-2 edits can enrich the actual empty cards instead of receiving only a generic whole-canvas instruction. Review cache versioning invalidates older reports whenever this diagnostic contract changes. -Branch contracts now promote evidence-backed classification, regression, mask, prediction, and similar parallel leaf heads to one consistent output role, remove ancestor shortcuts that bypass the shared branch point, and merge duplicate innovation labels into the existing node instead of drawing a second node. Feedback blueprints anchor nested feedback connectors at the actual inner artifact rather than the outer container. Visual enrichment is measured inside the blueprint's active figure region while retaining the same `0.08` threshold, so large white margins do not reject a genuinely illustrated figure and sparse box-only diagrams still fail. +Branch contracts now promote evidence-backed classification, regression, mask, prediction, and similar parallel leaf heads to one consistent output role, remove ancestor shortcuts that bypass the shared branch point, and merge duplicate innovation labels into the existing node instead of drawing a second node. Feedback blueprints anchor nested feedback connectors at the actual inner artifact rather than the outer container. Self-Refine candidates automatically run a focused topology judge in parallel; that judge sees only the generated candidate, avoiding answer leakage from the layout blueprint, and explicitly distinguishes `Self-Feedback -> REFINE` from the common wrong edges `FEEDBACK -> REFINE` and `Self-Feedback -> Refined output`. Visual enrichment and detailed-tile density are measured inside the blueprint's active figure region, so sparse box-only diagrams and flat color-filled panels fail even when their outer geometry is correct. + +Learned class/query tokens and positional embeddings that feed an internal model are normalized as conditioning modules rather than external inputs. A generic leaf such as `Head` is collapsed when it merely duplicates its upstream `MLP Head` and a concrete sibling output exists; the target overview figure's exact visible output label is preferred. Topology is inferred again after late repairs so removing a false duplicate branch can correctly restore a linear contract. Placeholder runs explicitly return `production_ready: false`, `engineering_only: true`, and `render_owner: engineering_placeholder`; they never claim that Image-2 produced the preview. Dense multi-frame overview normalization tolerates descriptive paper wording such as `heavyweight image encoder` and `lightweight mask decoder`. Those phrases still map to the canonical Figure 1 nodes `Image Encoder` and `Mask Decoder`, allowing the overview compiler to remove lower-level implementation details instead of misclassifying the whole contract as a generic branch graph. @@ -183,6 +186,8 @@ rfs paper-to-image ` --positive-reference "C:\path\reference2.png" ` --template auto ` --asset-mode image2 ` + --image-init-mode topology ` + --image-timeout 45 ` --candidates 3 ` --aspect-ratio auto ` --review-mode vlm ` @@ -193,11 +198,12 @@ rfs paper-to-image ` The main outputs are `paper_review.json`, `review_coverage_report.json`, `domain_profile.json`, `template_profiles/`, `selected_template.json`, -`layout_blueprint.png`, `layout_blueprint.json`, `visual_substrate_blueprint.png`, -`visual_substrate_blueprint.json`, `overlay_spec.json`, `overlays/`, +`layout_blueprint.png`, `layout_blueprint.json`, `overlay_spec.json`, `figure_specification.json`, `image_prompt.txt`, `image2_request_manifest.json`, the general and focused-topology critic reports, the `candidates/` -directory, and production-only `selected_image.png`. +directory, and production-only `selected_image.png`. `overlay_spec.json` is not +rendered onto the primary image; it records what Image2 must already have drawn +and supports later image-to-PPT reconstruction and benchmark auditing. To repair a reviewed failed candidate without paying for a new initial candidate batch, reuse it as the starting image: @@ -219,9 +225,15 @@ Only a failing source triggers one localized Image2 edit. This is useful after fixing a contract or Prompt bug because it preserves correct regions and avoids regenerating the initial image from scratch. -Reference-conditioned production generation requires an Image2 edit endpoint. -It defaults to `/images/edits` and may be overridden with -`RFS_IMAGE_EDIT_URL`. No API key value is written to output artifacts or logs. +Production generation defaults to topology-aware routing. Feedback-loop, +linear, branch, multimodal, and dense-multiframe figures use `--image-init-mode edit` internally +so gpt-image-2 fully redraws their benchmarked semantic blueprints. Linear, +branch, multimodal, and dense-multiframe blueprints include paper-grounded mechanism cues while +keeping all final text, arrows, and pixels owned by gpt-image-2. Other topologies continue to use +direct generation until their own multi-seed benchmarks justify blueprint editing. Use +`--image-init-mode generate` or `edit` to force either route, or `auto` to fall +back to direct generation after an edit timeout. Endpoints may be overridden with +`RFS_IMAGE_GENERATION_URL` and `RFS_IMAGE_EDIT_URL`. No API key value is written to output artifacts or logs. If a key was pasted into a chat, issue, or terminal transcript, revoke it and use a newly rotated key through environment variables before running production. See [docs/paper-to-image.md](docs/paper-to-image.md) for the review schema, @@ -240,16 +252,30 @@ modules. The judge understands explicit containment: an evidence-supported repeatable shared component inside a labeled operation container may implement that container's outgoing relation without being treated as an invented edge. -The general production critic and focused connector critic run concurrently. -Each uses a 90-second default timeout with no automatic full-call retry; override -them with `RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and -`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT`. Use `--repair-source` to re-review or -locally repair an existing candidate without regenerating correct regions. -Image-2 generation uses a 120-second default request timeout configured by -`RFS_IMAGE2_TIMEOUT`. A network timeout is not blindly retried; retryable HTTP +The default production critic uses one integrated VLM call for exact text, +paper alignment, aesthetics, and topology. Relations include structured source +side, target side, and arrowhead-contact fields. If a complex feedback or +cross-panel edge lacks explicit endpoint evidence, one short focused topology +fallback runs; set `RFS_PAPER_TO_IMAGE_FOCUSED_TOPOLOGY=1` to always run the +independent pass. Timeouts are controlled by +`RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and +`RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT`. OCR returns one record per physical text +occurrence, allowing deterministic duplicate-label checks. Use `--repair-source` +to re-review or locally repair an existing candidate without regenerating correct +regions. Every reviewed run also writes +`paper_image_comparison_report.json`, which consolidates text, paper evidence, +topology, density, aesthetics, blocking issues, and the next repair action. +The CLI uses a 45-second per-candidate Image-2 timeout through `--image-timeout`. +Low-level edit and generation defaults can be overridden with `RFS_IMAGE2_TIMEOUT` +and `RFS_IMAGE2_GENERATION_TIMEOUT`. A network timeout is not blindly retried; retryable HTTP or provider failures receive at most the configured `--image-retries` attempts, whose default is one. +Set `RFS_IMAGE_QUALITY` to `low`, `medium`, or `high` to request an explicit +Image2 quality tier; leaving it unset preserves the provider default. The value +is recorded in generation artifacts, and every tier must pass the same OCR, +paper-evidence, topology, information-density, and aesthetic gates. + Use `--candidates 3` for a minimum stability audit. Independent candidates are generated and reviewed concurrently, and `stability_report.json` records the production pass rate, mean score, worst-case score, standard deviation, and @@ -288,7 +314,6 @@ rfs make-framework ` --text-extractor-mode ocr ` --ocr-engine paddle ` --ocr-lang en_ch ` - --editable-overlay-ppt ` --json rfs validate --out "output\demo_placeholder" --json @@ -296,22 +321,22 @@ rfs validate --out "output\demo_placeholder" --json `output/` is intentionally ignored by Git. -## Reuse the Fast Paper Contract +## Optional Paper-Native Multi-Asset Experiment When `fast-framework-prompt` or `paper-to-image` has already produced a validated `figure_specification.json`, pass that run directory directly to -`make-framework`. This bypasses the legacy paper planner: panel-local node -instances, exact paper labels, evidence IDs, and panel-local relations become -the scientific authority for slots, editable text, and PPT connectors. +`make-framework`. In paper-native mode no complete reference image is required. +The semantic contract owns every visible module, relation, label, and visual +facet; the generated seed owns only geometry, density, and palette. ```powershell rfs make-framework ` --paper "C:\path\paper.pdf" ` - --reference "output\paper_to_image\run\selected_image.png" ` --paper-contract-dir "output\paper_to_image\run" ` --out "output\editable\contract_driven" ` --slot-count 25 ` - --slot-source reference-primary ` + --slot-source paper-native ` + --complexity-profile paper-native-dense ` --asset-mode placeholder ` --candidates-per-slot 1 ` --asset-workers 3 ` @@ -324,9 +349,9 @@ rfs make-framework ` --json ``` -For the complete paper-to-image-to-editable route, use -`paper-to-editable --editable-route framework`. The default `rebuild` route is -kept for compatibility with older image-reconstruction runs. +This `make-framework --slot-source paper-native` route is retained for research +comparison only. The product default is `paper-to-editable --editable-route +rebuild`, where Image2 generates the complete figure before reconstruction. ## Real VLM + Image Generation @@ -490,14 +515,10 @@ Use lower worker counts if your API provider rate-limits requests. ## Workflow ```text -input archive -> paper brief -> reference_geometry.json/reference_control_candidates.json -> -slot_overlay.png/reference_control_overlay.png -> reference_controls.json -> -arrow_style_profile.json/selected_arrow_routes.json/arrow_quality_report.json -> -reference_style_profile.json/style_sheet.md -> layout_plan.json -> figure_program.json -> -slot_visual_spec.json -> reference_slot_prompt_brief.json -> slot_prompt_plan.json -> -multi-candidate slot assets -> asset_quality_report.json -> asset_complexity_report.json -> -asset_visual_review.json/contact sheets -> editable_composition.pptx -> PDF/PNG export -> -visual_critic_iter_0.json -> critic_report.md -> validation +paper -> evidence-grounded semantic contract -> full layout blueprint and Prompt -> +Image2 complete figure (text + arrows + modules + illustrations) -> VLM review/repair -> +approved selected_image.png -> image-to-PPT reconstruction -> editable_composition.pptx -> +PDF/PNG export -> validation ``` Optional Presentations-plugin QA can run after validation: @@ -519,6 +540,9 @@ User content hierarchy, emphasis, aesthetic, reference-image, and A/B preference Key rules: +- Image2 owns the complete visible raster figure in the primary route, including text and arrows. +- The paper semantic contract constrains which labels, modules, relations, numbers, and claims Image2 may render. +- The reconstruction stage must follow the approved complete image rather than silently redesigning it. - The reference image is the source of truth for layout, local visual object choice, color, visual rhythm, and arrow logic when `--slot-source reference-primary` is used. - The paper provides scientific terminology and concept mapping; it should not override the reference image into a generic template. - Arrows, connector lines, dashed loops, panel frames, labels, formulas, and critical text are PPT editable objects, not image assets. diff --git a/docs/architecture.md b/docs/architecture.md index 0ab1925..c4a2a7b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,21 +1,20 @@ # Summary -ResearchFigureStudio has one product goal: turn a paper into a scientifically faithful, editable PowerPoint figure. Paper-derived contracts own semantics; generated or supplied images own layout and visual style; deterministic code owns PPTX composition and validation. +ResearchFigureStudio has one product goal: let Image2 generate a scientifically faithful complete paper figure, then reconstruct that approved image into editable PowerPoint structure. ## Product architecture ```mermaid flowchart LR P["Paper"] --> G["PaperGroundTruth"] - R["Optional visual references"] --> I["Reference image generation"] - G --> I - I --> V["Approved visual reference"] - V --> A["Layout, cards, slots, style analysis"] - G --> B["Semantic binding"] - A --> B - B --> C["Editable figure program"] - C --> PPT["Editable PPTX"] - PPT --> Q["Deterministic and optional VLM QA"] + R["Optional visual references"] --> B["Full layout blueprint and Prompt"] + G --> B + B --> I["Image2 complete raster figure"] + I --> Q["Scientific, text, topology, and aesthetic review"] + Q --> A["Approved selected_image.png"] + A --> D["Image-to-PPT reconstruction program"] + D --> PPT["Editable PPTX"] + PPT --> V["Deterministic and optional VLM QA"] ``` The semantic path never passes through OCR alone. OCR may help recover geometry from an image, but exact labels, entity IDs, relations, and evidence links are restored from `paper_review.json` and `figure_specification.json` before compilation. @@ -58,8 +57,8 @@ The current package remains under `rfs/` to avoid a disruptive all-at-once move. | Scientific relations and evidence | Paper semantic contract | | Entity roles, provenance, and evaluation boundaries | Normalized evidence-backed contract completion | | Paper-to-image draft node and connector geometry | `layout_blueprint.json.semantic_plan` | -| Direct paper-to-PPT node and connector geometry | `figure_program.json` compiled from the same semantic plan | -| Layout, visual rhythm, palette, object style | Approved generated/user reference image | +| Complete visible raster result | Approved `selected_image.png` generated by Image2 | +| Reconstructed layout, controls, and style | Approved complete image plus paper semantic contract | | Editable object geometry | `figure_program.json` | | PowerPoint rendering | `ppt_compiler.py` | | Production eligibility | Candidate, semantic, visual, and PPTX validation reports | @@ -77,18 +76,23 @@ blueprint region and records both active-region and whole-canvas ratios. ## Stable workflow boundaries - `fast-framework-prompt`: produces the cached semantic contract and prompt; `--editable-ppt` additionally emits a fast native-shape PPTX without image generation. -- `paper-to-image`: produces reviewed raw, geometry-normalized substrate, and composed visual candidates plus paper-grounding artifacts; `--editable-overlay-ppt` also creates a PPTX whose labels and connectors are native editable objects over the normalized visual substrate. +- `paper-to-image`: produces complete Image2 figures by default, including visible text and arrows, then selects only candidates that pass scientific and visual review. - `rebuild-editable`: reconstructs an image; it can optionally accept a paper semantic contract. -- `paper-to-editable`: runs both stages and requires a production-approved image unless engineering preview use is explicitly enabled. +- `paper-to-editable`: defaults to `rebuild`: generate the complete figure first, then reconstruct the approved image. +- `paper-to-image`: always produces a complete Image2-owned raster figure; post-generation text/connector overlays are not supported. +- Topology-aware initialization uses benchmarked blueprint-conditioned full redraw for feedback, linear, branch, multimodal, and dense-multiframe figures; unbenchmarked topologies remain on direct generation. +- `make-framework --slot-source paper-native`: consumes a validated semantic contract, generates a content-free seed, expands real entities into grounded facets, and compiles slot assets plus editable structure. - Plugin skill: selects and invokes workflows; it does not contain the engine or hardcoded repository paths. - Benchmark layer: independently scores paper-to-image scientific/visual quality and image-to-PPT fidelity/editability. ## Production rules - Failed candidates never become `selected_image.png`. -- Whole-image Image2 candidates must pass exact card/node cardinality, unique geometry matching, deterministic re-slotting, restored connector corridors, and a no-semantic-crop gate before VLM approval can count. +- Whole-image Image2 candidates must pass visible-label, scientific-entity, relation-direction, topology, information-density, and aesthetic review before VLM approval can count. - Engineering previews are opt-in and never production-approved. -- Critical text and relations remain editable PPT objects. +- The complete Image2 source visibly contains critical text and relations; the later reconstruction must recover them as editable PPT objects wherever possible. +- Paper-native geometry/style seeds and slot assets belong only to the optional framework experiment. +- PowerPoint PDF/600-DPI PNG export runs in a separate process with a hard timeout so COM cannot stall the whole workflow indefinitely. - Image/OCR guesses cannot override paper-grounded terminology or relation endpoints. - Deterministic completion is graph-scoped: rich VLM contracts only gain overview-local or directly connected evidence-backed entities, while heuristic contracts grow from overview seeds or the largest connected method chain. Late appendix-only diagram labels, ungrounded innovations, and input shortcuts that bypass an explicit intermediate node are removed before planning. - `semantic_blueprint.py` compiles 2-16 visible contract entities into normalized graph geometry. It ranks DAG layers, places source-only conditioning nodes immediately before their targets, assigns distinct fan-in/fan-out ports, routes skip-layer edges around intermediate nodes, and sends feedback loops through an outer lane. diff --git a/docs/code-structure.md b/docs/code-structure.md new file mode 100644 index 0000000..bb18e97 --- /dev/null +++ b/docs/code-structure.md @@ -0,0 +1,68 @@ +# Paperimage 代码结构 + +本文是仓库中代码归属的唯一导航页。它描述的是**应修改的位置**,而不是 +Python 历史导入路径;历史路径仍由兼容层保留。 + +## 产品主线 + +```text +论文 / 参考图 + -> contracts(科学语义) + -> analysis + planning(理解和规划) + -> generation(局部图像资产) + -> composition(figure program 与 PPTX) + -> evaluation(验证与报告) +``` + +`workflows` 只编排这些阶段。`providers` 只负责外部模型/API。论文到候选图 +的领域实现保留在 `paper_to_image`;它生产并消费上述稳定合同。 + +## 目录职责 + +| 状态 | 位置 | 负责内容 | 新代码规则 | +|---|---|---|---| +| Production | `rfs/contracts` | 语义合同、绑定与兼容转换 | 合同字段先在此定义和验证 | +| Production | `rfs/analysis` | 论文、布局、控件、OCR 与文本分析 | 不编译 PPTX | +| Production | `rfs/planning` | 设计、布局和语义规划 | 不调用资产 API | +| Production | `rfs/generation` | 候选图、槽资产和资产策略 | 不拥有 PPTX 几何 | +| Production | `rfs/composition` | figure program、PPTX、预览和导出 | 唯一编译权威 | +| Production | `rfs/evaluation` | validator、视觉检查和 benchmark | 只检查和报告,不修写合同 | +| Production | `rfs/providers` | VLM/API、模型选择、重试 | 不包含业务规划 | +| Production | `rfs/workflows` | 产品级跨阶段编排 | 不复制领域实现 | +| Production | `rfs/professional` | controlled professional DSL | 与普通 rebuild 分开 | +| Production | `rfs/paper_to_image` | 论文 review、grounding、候选图路线 | 保持领域内聚 | +| Experimental | `rfs/coevolution`、`rfs/judge_training`、`experiments/` | Creator/Judge 与研究原型 | 不被默认产品 workflow 隐式导入 | +| Compatibility | 根目录的已迁移 alias | 旧 import 路径 | 仅转发,禁止新增逻辑 | +| Transitional infrastructure | 根目录的 `cli` 等 | 尚未完全下沉的公共入口或实现 | 维护行为兼容;新功能进入对应 canonical 包 | + +## 根目录文件的阅读规则 + +根目录 `rfs/*.py` 不表示一个独立的业务层: + +- 薄 alias(例如 `editable_rebuild`、`ppt_compiler`、`asset_generator`、 + `program_builder`、`layout_planner`)只为外部兼容而保留。 +- `cli.py` 是唯一命令行入口;它不拥有算法或输出合同。 +- 仅 `cli.py` 与历史 `workflow.py` 属于 *transitional infrastructure*。 + 修改其行为时,必须同时在本页对应的 canonical 目录建立或使用明确入口,不能再创建第二套实现。 +- `workflow.py` 是历史 `make-framework` 路线;新增产品编排进入 + `rfs/workflows`。 + +## 命令分类 + +| 分类 | 命令 | +|---|---| +| Product | `paper-to-image`、`paper-to-editable`、`rebuild-editable`、`make-framework`、`fast-framework-prompt` | +| Experimental | `rebuild-editable-pro`、`rebuild-evolve`、`rebuild-editable-eval`、`coevolve-image`、`coevolution-report`、`rebuild-judge` | +| Maintenance / QA | `doctor`、`inspect-pdf`、`benchmark`、`validate`、`presentations-qa` 和参考图库辅助命令 | + +命令名、参数、输出 JSON 以及输出合同是公共兼容面;结构整理不能改变它们。 + +## 修改后的最低验证 + +```powershell +python -m compileall -q rfs +python -m unittest discover -s tests -q +rfs doctor --json +``` + +涉及 workflow、合同或 CLI 时,还必须运行相应离线 smoke 和兼容导入测试。 diff --git a/docs/gpt-reproduction-workflow.md b/docs/gpt-reproduction-workflow.md index c3811db..38c670f 100644 --- a/docs/gpt-reproduction-workflow.md +++ b/docs/gpt-reproduction-workflow.md @@ -80,7 +80,8 @@ VLM planning uses an OpenAI-compatible chat-completions endpoint: $env:API_BASE='https://your-openai-compatible-provider/v1' $env:API_KEY='' $env:MODEL_VLM='your-vision-language-model' -$env:RFS_REBUILD_LAYOUT_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_DESIGN_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_LAYOUT_MODEL='gemini-3.5-flash' $env:RFS_REBUILD_CONTROL_MODEL=$env:MODEL_VLM $env:RFS_REBUILD_SEMANTIC_MODEL=$env:MODEL_VLM $env:RFS_PROFESSIONAL_REBUILD_MODEL=$env:MODEL_VLM diff --git a/docs/judge-shadow-deployment.md b/docs/judge-shadow-deployment.md new file mode 100644 index 0000000..ca1ecd9 --- /dev/null +++ b/docs/judge-shadow-deployment.md @@ -0,0 +1,42 @@ +# Judge Shadow deployment + +This interface activates only after the training team has produced a checkpoint +and exported JSONL predictions. It does not execute arbitrary checkpoint code +or silently substitute a Bootstrap VLM. + +## Register a candidate + +Create a manifest outside source control. A Shadow candidate needs its model +identifier, dataset version, Git commit and allowed role: + +```json +{ + "model_id": "judge-v1-rc1", + "dataset_version": "judge-dataset-v001", + "roles": ["shadow"], + "git_commit": "" +} +``` + +```powershell +rfs judge register --checkpoint models/judge-v1-rc1 --manifest manifest.json --registry models/judge_registry.json +``` + +`online` and `frozen` roles require `promotion.approved`, +`promotion.hidden_test_passed`, and `promotion.adversarial_passed`. `frozen` +also requires `promotion.shadow_stable`. This prevents a newly trained model +from replacing both Judge roles at once. + +## Evaluate and Shadow compare + +The training/runtime system exports one JSONL row per `example_id`; either the +top-level row or its `prediction` object may contain `scientific_score`. + +```powershell +rfs judge evaluate --checkpoint models/judge-v1-rc1 --dataset datasets/judge-v001 --predictions outputs/judge-v1-hidden.jsonl --suite hidden_test +rfs judge shadow --checkpoint models/judge-v1-rc1 --dataset datasets/judge-v001 --predictions outputs/judge-v1-shadow.jsonl --bootstrap-predictions outputs/bootstrap-shadow.jsonl +``` + +Shadow comparison records all divergent decisions but does not control the +Creator or acceptance path. Promotion remains a human-and-metric decision; +the deterministic scientific Grounding Gate continues to be mandatory. diff --git a/docs/judge_model_training_plan.md b/docs/judge_model_training_plan.md new file mode 100644 index 0000000..6ab08a5 --- /dev/null +++ b/docs/judge_model_training_plan.md @@ -0,0 +1,456 @@ +# 论文框架图质量 Judge 模型训练计划(落地执行版) + +> 状态:v1.1(2026-08-23) +> 定位:本文是「现在、用现有数据、具体怎么做」的执行计划。与两份既有文档互补: +> - `docs/judge_model_training_roadmap.md` —— 总路线 + 决策记录(为什么这么做) +> - `docs/Final_target.md` —— 产品目标蓝图(最终系统长什么样) +> 本文只回答一件事:**先做什么、用什么、做到什么程度算完成。** + +--- + +## 0. 一句话目标 + +本阶段训练一个**论文条件化的共享底座多任务 Judge**,让模型通过同时阅读论文证据与候选框架图,学习判断图是否科学地表达论文方法,以及什么样的构图在信息组织、可读性和审美上更好。 + +本阶段的模型目标是: + +```text +Judge(paper, evidence, figure, preferences) + → scientific grounding + → method representation quality + → figure design quality + → blocking issues and evidence + → preserve / repair + → pairwise preference +``` + +Judge 只负责评价、诊断、排序和提出可执行的改进建议,**不负责生成框架图**。Creator Agent 的训练不属于本计划,只有 Judge 通过科学门禁后才进入后续独立计划。 + +训练数据仍可覆盖图像和可编辑 PPTX,但两者是先后阶段,不代表本阶段同时训练 Creator: + +```text +论文 Ground Truth + → 像素图(paper-to-image 生成) + → 可编辑图(rebuild-editable 生成 PPTX) +``` + +即同一个底座,支撑两个评价任务: + +| 任务 | 输入 | 输出 | +|---|---|---| +| **图像 Judge**(本计划) | 论文证据/GT + 候选 PNG | 科学方法表达 / 审美偏好 / 视觉质量 / 成对排序 / 阻塞问题 / 修改建议 | +| **可编辑图 Judge**(后续计划) | GT + 目标图 + PPT 对象树 + 渲染预览 | 视觉还原度 / 文字·箭头·分组可编辑率 / 科学保留度 | + +**首版明确不做什么**:不从零训练、不全参微调、不训练 Creator Agent、不把人工好坏标签当成唯一监督、不让模型自行决定总分权重。图像 Judge 未过科学门禁前不开 PPT Judge。 + +### 0.1 训练边界 + +本计划要回答的是“什么样的框架图科学、完整、清晰、好看,以及为什么”,而不是“如何从论文直接生成一张图”。 + +因此本阶段不以最终图片生成率作为目标,也不把 Judge 的高分直接当作 Creator 的训练奖励。后续 Creator 训练必须使用冻结、通过门禁的 Judge,并另行设计论文到语义合同、拓扑和布局的训练任务。 + +--- + +## 1. Judge 的最终形态(三层,先建立心智) + +```text +Trained VLM Judge(7B~10B 多模态,QLoRA 微调) + 只输出分项分数 + 证据引用 + 错误定位 + 建议;禁止自己算总分 + ↓ +Deterministic Grounding Gate(纯代码) + OCR + 实体/关系覆盖 + 禁止内容 + 术语;硬错误直接 gate fail + ↓ +Score Aggregator(代码) + 按 GT 权重算总分与通过状态;硬错误不允许被审美分抵消 +``` + +**为什么必须三层**:你们自己的 AutoFigure 实验已证明——纯 VLM Judge 会放行"美观但科学错误"的图(最终图仍虚构 Encoder/Prediction 等无关模块却判无阻塞)。**科学硬错误漏检是当前第一风险,Grounding Gate 是安全层。** + +--- + +## 2. 现状盘点(已有 vs 缺失) + +### 2.1 已有资产 + +| 资产 | 位置 | 用途 | +|---|---|---| +| 闭环代码 | `rfs/coevolution/`(orchestrator/creator/judges/groundtruth/storage) | 生成候选 → Online Judge → Frozen Judge → 轨迹 | +| Bootstrap Judge | `rfs/coevolution/judges.py::VLMJudge` | 当前 Prompt 版 Judge,只能当基线和数据生产 | +| GT schema | `rfs/coevolution/groundtruth.py` | 已定义 scientific_truth + aesthetic_preferences + weights + thresholds | +| 轨迹 jsonl | 每次 `rfs coevolve-image` 输出 `preference_pairs/repair_trajectories/critique_outcomes.jsonl` | 真实生成轨迹,转训练数据 | +| 已有偏好 | `output/paper_to_image_autofigure/preference_pairs.jsonl` | 2 条人工 A/B(Sen-illion 标注) | +| **papers-dataset** | github.com/Sen-illion/papers-dataset | 1000 元数据 + 996 PDF + **491 条人工粗标注** | +| 审美问卷模板 | `docs/aesthetic_ground_truth_questionnaire.md` | 收集更细的人工偏好 | +| 文档 | `judge_model_training_roadmap.md` / `Final_target.md` | 决策依据 | + +### 2.2 缺失(本计划要补的) + +1. **论文—框架图级训练数据集**(现在只有论文级粗标注) +2. `corruption_generator.py`(主动造"已知错误"科学负样本) +3. `grounding_gate.py`(确定性硬门禁) +4. `rfs/judge_training/` 训练包(schema/dataset_builder/trainer/evaluator/registry) +5. 云端 QLoRA 训练脚本与配置 + +--- + +## 3. 数据资产深度分析(决定第 1 步怎么做) + +### 3.0 数据落盘位置与目录约定(所有脚本据此读写) + +| 用途 | 绝对路径 | 说明 | +|---|---|---| +| 源数据(已 clone) | `D:\文件夹\papers-dataset` | 含 `classified/分类清单.csv`、`classified/{优,好,坏,未筛}/`、`pdf/`、`_raw/` | +| 图级数据集根目录 | `D:\文件夹\Paperimage\datasets\` | 抽图/转换/造错/切分的产物,**需 gitignore** | +| 图资产 | `D:\文件夹\Paperimage\datasets\figures\\fig_.png` | 抽图输出 | +| 训练记录 | `D:\文件夹\Paperimage\datasets\judge-v001\*.jsonl` | 统一 `JudgeExample` 格式 | +| 模型权重 | `D:\文件夹\Paperimage\models\` | 下载的底座 + QLoRA checkpoint,**需 gitignore** | + +> 约定:源数据只读,不写回 `papers-dataset`;一切中间产物进 `datasets/`;`datasets/` 与 `models/` 都不提交到 git(`pdf/` 与论文原图属私有数据)。 + +### 3.1 `分类清单.csv` 精确结构(491 条) + +表头:`分类,风格,优,arxiv_id,标题,备注,被引,arXiv` + +| 字段 | 取值 | 含义 | +|---|---|---| +| 分类 | 坏 / 好 / 未筛 | 论文级三档质量 | +| 风格 | 科研风 / 生动活泼 / 空 | 仅「好」有;两种审美风格 | +| 优 | 优 / 空 | 是否额外标记为优选 | +| 备注 | 自由文本 | 人工原因,含**图级线索** | +| 被引 | 数字 | 引用数 | + +备注里已出现的关键词(可直接转成缺陷标签): + +- 审美缺陷:`配色不好看 / 配色难看` +- 信息量缺陷:`信息密度低 / 信息密度太低 / 太简单 / 较简单 / 好看,但信息密度低` +- 可读性缺陷:`字体太小,信息密度低` +- 图级优选:`图1:优 / 图2:优 / 图3:优 / 图2更好 / 图3` + +### 3.2 关键差距(必须讲清楚) + +> **现有标注是“论文级审美/风格”标注,不是“论文条件化的图级科学表达”标注。** + +| 维度 | 现有数据能训 | 现有数据不能训 | +|---|---|---| +| 审美质量(优/好/坏) | ✅ 可做正/负例 | — | +| 风格偏好(科研风 vs 生动活泼) | ✅ 可做偏好对 | — | +| 缺陷分类(配色/密度/字体) | ✅ 备注可转标签 | — | +| **科学正确性(虚构模块/关系反转/术语错)** | ❌ 完全没有 | 需另一路数据 | + +仅有“好/不好”不能告诉模型失败原因。正式样本必须同时提供论文证据、结构化方法合同、图中实体/关系和人工原因;否则模型容易把视觉风格误学成科学质量。 + +**科学正确性数据只能用两路补**:① `coevolve-image` 真实轨迹(有 GT 对照);② `corruption_generator` 主动注入已知错误(标签 100% 确定)。 + +### 3.3 分布估计(以脚本统计为准,约数) + +```text +优 ≈ 50 篇(正例最强信号) +好·科研风 ≈ 90 篇 +好·生动活泼 ≈ 60 篇 +坏 ≈ 200 篇(负例) +未筛 ≈ 50 篇(暂不用于训练,留作待标注池) +``` + +--- + +## 4. 第 1 步:数据工程(最需人力,一半可脚本化) + +### 4.1 要产出的四个数据集 + +| 数据集 | 内容 | 训练用途 | +|---|---|---| +| `fig_quality` | 论文证据 + 图级 优/好/坏 + 科学/叙事/视觉缺陷 | 综合质量与缺陷识别 | +| `fig_style_pairs` | 同一主题「科研风 vs 生动活泼」偏好对 | 风格偏好排序(Pairwise) | +| `fig_scientific` | GT/论文证据 + 正确图 + 注入错误图 + 确定标签 | 科学方法表达与硬错误召回 | +| `fig_feedback` | 修改前图 + 建议 + 修改后图 + 是否有效 | 建议有效性预测 | + +### 4.2 具体动作(按顺序) + +**A. 抽图脚本 `rfs/judge_training/figure_extractor.py`** +- 用 PyMuPDF 遍历 `classified/{优,好/*,坏}/` 的 PDF,提取框架图(按渲染的图片对象 + 图注上下文裁出,过滤小图/图标)。 +- 输出:`datasets/figures//fig_.png` + `figures_manifest.json`(记录页码、bbox、所属 PDF)。 +- 验收:能稳定抽出 >80% 论文的主框架图。 + +**B. 转换脚本 `rfs/judge_training/classified_csv_to_dataset.py`** +- 读 `分类清单.csv`,把论文级标签投影到抽出的每张图: + - `分类=好/优` → positive;`坏` → negative + - `风格` → style 标签;`优` → excellent 标记 + - `备注` 按关键词表 → 缺陷标签(配色/信息密度/字体/太简单) + - `备注` 含 `图N` 的 → 打在图级(更高置信) +- 输出统一为 `JudgeExample` 格式(见 4.3)。 +- 验收:每条记录可回溯到 `arxiv_id + 页码 + bbox + 原始 CSV 行`。 + +**C. 造错误脚本 `rfs/judge_training/corruption_generator.py`(科学数据命根子)** +- 对每张"正确"框架图,用图像编辑 + 文字覆盖主动注入已知错误: + 1. 删除一个核心模块(遮挡/裁掉) + 2. 添加不存在的模块(贴一个假模块卡片) + 3. 反转关键箭头方向 + 4. 替换核心术语(改文字) + 5. 保持美观但篡改事实(只改语义不改构图)→ 这就是 `beautiful_but_wrong` 对抗集 +- 每个注入错误记录:注入位置 + 类型 + 预期标签(`invented_module/relation_reversal/...`)。 +- 验收:注入错误召回率 ≥95%(门禁能抓)。 + +**D. 切分 `rfs/judge_training/split.py`** +- **按 arxiv_id 切分**:train 70% / val 15% / hidden_test 15%,同一篇论文绝不跨集。 +- 另建 8 个对抗集:`beautiful_but_wrong / correct_but_ugly / relation_reversal / missing_core_stage / invented_module / terminology_corruption / style_preference_conflict / reward_hacking`。 + +### 4.3 统一样本格式 `JudgeExample`(`schemas.py`) + +```json +{ + "example_id": "fig-2505.20824-1", + "dataset_version": "judge-dataset-v001", + "ground_truth_id": "arxiv-2505.20824", + "task_type": "method_grounding | quality_classification | style_preference | pairwise_preference | scientific_error | feedback", + "paper_evidence": { + "title": "...", + "abstract": "...", + "method": "...", + "citations": [{"text": "...", "page": 3, "section": "Method"}] + }, + "ground_truth": { + "must_show": ["..."], + "must_not_invent": ["..."], + "entities": ["..."], + "relations": [["source", "target", "relation"]], + "terminology": ["..."] + }, + "candidate_a": "datasets/figures/2505.20824/fig_1.png", + "candidate_b": null, + "labels": { + "scientific_status": "correct|incomplete|incorrect|unverifiable", + "quality": "excellent|good|insufficient|bad", + "style": "research|vivid|null", + "defects": ["配色不好看"], + "scientific_score": null, + "method_coverage_score": null, + "relation_correctness_score": null, + "narrative_clarity_score": null, + "readability_score": null, + "aesthetic_score": null, + "blocking_issues": [], + "missing_required_entities": [], + "ungrounded_entities": [], + "missing_relations": [], + "incorrect_relations": [], + "terminology_errors": [], + "injected_error_type": null, + "preferred_candidate": "A|B|tie", + "preserve": [], + "repair": [], + "confidence": 0.0 + }, + "provenance": {"arxiv_id": "2505.20824", "csv_row": 123, "page": 2, "bbox": [..], "human_audited": true} +} +``` + +### 4.4 规模目标 + +| 版本 | 论文数 | 图数 | 记录数 | 用途 | +|---|---|---|---|---| +| D0 协议验证 | 30~50 | 150~400 | ≥500 单图 + ≥500 配对 | 验证流程,不上线 | +| D1 正式训练 | 300~500 | 3k~5k | 8k~15k | Judge v1 训练 | + +### 4.5 第 1 步里"必须人来"的部分(不可脚本化) + +1. **审核论文—图对应关系**:确认图中模块和关系确实能由论文证据支持。 +2. **审核注入错误**:确认每张错误图“错对了、错干净了”,且没有引入未标注副作用。 +3. **A/B 综合裁决**:分别说明科学表达、叙事、可读性和审美原因,不能只给总偏好。 +4. **硬错误人工复核**:所有高置信科学错误样本人工确认。 + +### 4.6 人工标注协议:不能只标“好/不好” + +每张单图至少分开标注以下维度;总质量标签只作为汇总结果,不能替代分项原因: + +| 维度 | 必填标签 | 作用 | +|---|---|---| +| 科学状态 | `correct / incomplete / incorrect / unverifiable` | 区分正确、不完整、错误和证据不足 | +| 方法覆盖 | 必须模块、缺失模块、允许简化项 | 判断是否完整展示论文方法 | +| 关系正确性 | 正确、缺失、反向或无依据的关系 | 监督输入输出、依赖和训练/推理流程 | +| 贡献突出度 | 核心贡献是否被突出、是否被次要内容淹没 | 学习框架图叙事重点 | +| 叙事清晰度 | 阶段划分、阅读顺序、分组是否清楚 | 学习科学内容如何组织成图 | +| 可读性 | 字体、拥挤、遮挡、箭头、信息密度 | 学习表达质量 | +| 审美质量 | 配色、风格、一致性、视觉层级 | 学习人工偏好 | +| 修复信息 | `preserve`、`repair`、依据和目标区域 | 训练可执行诊断 | + +Pairwise 标注必须使用同一论文或等价科学合同下的 A/B 图,并分别记录: + +```text +preferred_candidate +scientific_reason +method_representation_reason +narrative_reason +visual_reason +confidence +``` + +如果 A 更美观但 B 更科学,不能强行压成单一标签;应分别保存分项偏好,并由硬门禁保证科学错误不能被审美优势抵消。 + +--- + +## 5. 第 2 步:门禁 + 基线 + 底座选型(8G 本地可做) + +### 5.1 写 Grounding Gate v0(`rfs/judge_training/grounding_gate.py`) + +输入 GT(实体/关系/must_show/must_not_invent/terminology)+ 候选图 + OCR 文本 + VLM 抽取的实体关系,输出: + +```json +{ "grounded_entities":[], "ungrounded_entities":[], "missing_required_entities":[], + "incorrect_relations":[], "terminology_errors":[], "blocking_issues":[], "gate_pass": false } +``` + +**硬失败规则**(直接 gate fail,不参与加权):核心模块缺失 / 新增无依据模块 / 关键关系方向错 / 命中 must_not_invent / 术语被替换 / Judge 无法给出 GT 依据的高置信科学判断。 + +### 5.2 跑 Bootstrap 基线 + +用 `VLMJudge` 把 D0 全量评一遍,得到**未训练基线**分数(后续所有训练版本都对比它)。 + +### 5.3 底座 bake-off(云端小规模) + +候选(7B~10B 多模态,能读长 GT + 高清图 + 多图 Pairwise + JSON 稳定): +- Qwen2.5-VL-7B-Instruct(首选,中文/长上下文强) +- InternVL2-8B +- MiniCPM-V-2.6(8B) +- LLaVA-OneVision-7B + +各训一小轮,按「JSON 合法率 + 科学错误召回 + 多图能力」排序,**选定后全程锁定**(本数据版本消融结束前不许换)。 + +--- + +## 6. 第 3 步:云端 QLoRA 分阶段微调(租 24~48G) + +### 6.1 训练顺序(图像 Judge,5 阶段) + +| Stage | 训练任务 | 目的 | 数据 | +|---|---|---|---| +| A | 论文条件化结构化 SFT | 稳定 JSON + 论文证据引用 + 方法覆盖/关系核验 + 阻塞识别 | 全量单图样本 | +| B | Pairwise 排序 | 在同一论文/GT 下判断哪张图更科学、更清晰、更符合审美 | fig_style_pairs + 质量对 | +| C | 硬错误加权 | 提高错误样本损失权重,高召回优先 | fig_scientific 注入错误 | +| D | 建议有效性 | 预测"建议是否可执行/有效/有副作用" | fig_feedback | +| E | 分数校准 | 分项分 → 代码聚合总分;置信度校准 | 独立验证集 | + +### 6.2 可编辑图 Judge(不属于 Judge v1) + +Judge v1 只训练论文条件化的图像 Judge。可编辑图 Judge 是后续独立阶段:复用已经通过门禁的底座,再增加「PPT 对象树 + 渲染预览 vs 目标图」样本,评价视觉还原度、文字/箭头/分组可编辑率和科学保留度。 + +该阶段不得和 Judge v1 混合验收,也不得在图像 Judge 通过科学门禁前启动。 + +### 6.3 训练配置(起点,非定稿) + +- QLoRA:4-bit NF4,LoRA rank 32~64,alpha=2×rank,target 全线性层 +(可选)视觉投影 +- lr 1e-4~2e-4,cosine 衰减,epochs 1~3,warmup 3% +- batch 1~2 × gradient accumulation → 有效 batch 16~32 +- 图像 max pixel ≈ 1280×1280(高分辨率科研图必需) +- 每 epoch 存 checkpoint,只留验证最优为候选 + +### 6.4 上线门禁(训练完 ≠ 能用) + +```text +离线训练 → val 选候选 → hidden_test 评测 → 对抗集 → Shadow 影子运行 +→ 与当前 Judge 分歧分析 → 小流量 Online → Frozen 仍用旧稳定版 → 达标后升 Frozen +``` + +**Judge v1 硬指标(不达标不得上线,不许调低阈值)**: + +| 指标 | 门槛 | +|---|---| +| 科学硬错误召回率 | ≥95% | +| 科学硬错误漏检率 | ≤5% | +| 美观但错误对抗集拦截率 | ≥95% | +| 必须方法模块召回率 | ≥90% | +| 虚构科学模块召回率 | ≥95% | +| 关键关系方向准确率 | ≥90% | +| 方法表达完整度与人工评分相关性 | Spearman ≥0.70 | +| Pairwise 排序准确率 | ≥80% | +| 审美偏好排序准确率 | ≥75% | +| 修改建议有效率 | ≥70% | +| 重复评分标准差 | ≤0.03 | +| 新论文跨领域排序 | ≥70% | +| JSON 合法率 | ≥99% | + +隐藏测试必须以**未见论文**为单位,要求模型在没有见过该论文框架图的情况下完成:论文证据理解、图中模块与关系核验、方法表达质量判断和 A/B 排序。仅在训练论文或近重复图片上得分高,不算达到目标。 + +--- + +## 7. 算力与分工 + +### 7.1 算力(已实测你机器:RTX 5060 8GB,CUDA 13.1) + +| 阶段 | 硬件 | 说明 | +|---|---|---| +| 第 1 步数据 | 本机 CPU / 云 CPU | 几乎不用 GPU | +| 第 2 步门禁+推理 | 本机 8G(4-bit 推理)或云端 | 8G 够推理、不够训练 | +| 第 3 步训练 | 云 **24G 起,推荐 48G** | RTX4090/L4/A10(24G),L40S/A6000(48G) | + +### 7.2 分工 + +| 事项 | 谁 | +|---|---| +| 全部脚本/训练代码/配置/评测 | Kira1(本助手) | +| 审核注入错误 + 拍板 A/B 偏好 | Sen-illion(人工) | +| 租云卡 + 在云上执行命令 | Sen-illion(或给 Kira1 远程访问) | + +### 7.3 人力估算(粗) + +- 第 1 步:脚本 ~3~5 人天(助手),人工审核 ~2~4 人天(你)→ **总瓶颈在此** +- 第 2 步:~2~3 人天(助手) +- 第 3 步:代码 ~2~3 人天(助手)+ 云 GPU 每轮实验几小时~几十小时(算力,非人力) + +--- + +## 8. 风险与防护 + +| 风险 | 防护 | +|---|---| +| Reward Hacking(后续生成系统迎合 Judge 骗分) | 独立 Gate + 冻结对抗集 + Online/Frozen 分离 + 人工抽查 | +| Judge 与后续生成系统共谋漂移 | Judge 先独立训练和冻结;后续不得同周期互证 | +| 审美模板坍缩 | 多风格 Profile + 多样性指标 | +| 科学错误被总分掩盖 | 硬错误 gate fail,不参与加权抵消 | +| 数据泄漏 | 按 arxiv_id 切分 + 图像感知哈希去重 + 冻结 manifest | +| 私有论文泄漏 | 数据/模型目录 gitignore,本地训练优先 | + +--- + +## 9. 名词速查 + +- **Ground Truth(GT)**:人工确认的标准答案(论文事实 + 审美偏好),训练与打分的唯一权威。 +- **VLM / MLLM**:多模态大模型,能同时看图 + 读文字,Judge 底座必须是它。 +- **Backbone(底座)**:选来微调的开源基础模型;"7B"≈70 亿参数。 +- **LoRA / QLoRA**:只训练少量附加参数(LoRA),QLoRA 再把底座压成 4-bit 更省显存。 +- **SFT(监督微调)**:给"输入→标准答案"教模型模仿。 +- **Pairwise / 成对排序**:只判"A 和 B 谁更好",比绝对打分稳定。 +- **合成硬负例**:主动制造的错误图(删模块/反转箭头),用于监督错误召回。 +- **Grounding Gate(落地门禁)**:独立代码检查层,硬错误直接判失败。 +- **Bootstrap Judge**:当前靠 Prompt 调 API、未训练的 Judge,只做基线和数据生产。 +- **Reward Hacking(刷分作弊)**:生成模型迎合评分器而非满足真相。 +- **Checkpoint**:训练中保存的权重快照,可回滚。 +- **Shadow / Online / Frozen**:影子(只跑不决策)/ 在线(指导生成)/ 冻结(独立验收),三者不能同时换新模型。 +- **VRAM / 显存**:显卡训练专用内存;8G/24G/48G 均指此。 +- **数据泄漏**:同一论文同时进训练和测试集,导致虚高分数。 +- **bake-off**:多个候选模型"打擂选拔",选定后锁定。 +- **对象树**:PPTX 内部结构化描述(文本框/箭头/分组/坐标),可编辑 Judge 读它判断可编辑性。 +- **PDF 抽图(figure extraction)**:用 PyMuPDF 把 PDF 内嵌插图裁成独立 PNG。 + +--- + +## 10. 立即执行清单(下一步动作) + +1. 冻结论文证据包、科学 Ground Truth 和 `JudgeExample` schema,确保每个科学标签都能追溯到论文证据。 +2. 建 `rfs/judge_training/` 包骨架:`schemas.py / dataset_builder.py / figure_extractor.py / classified_csv_to_dataset.py / corruption_generator.py / grounding_gate.py / split.py`。 +3. 先写并跑通三个不依赖训练 GPU 的环节: + - 论文证据与方法合同构建; + - `classified_csv_to_dataset.py`(分类清单 → 待复核图级数据); + - `corruption_generator.py`(基于已确认 GT 生成科学负样本)。 +4. 统计 `分类清单.csv` 精确分布,抽图验证,并完成第一批论文—图对应关系人工复核。 +5. 建立 D0:同时覆盖正确图、不完整图、科学错误图、视觉不足图和同论文 A/B 图。 +6. 冻结 D0、manifest 和隐藏测试后,云端 bake-off 选底座,再进入 QLoRA。 + +--- + +## 11. 与既有文档的关系(避免重复) + +- 决策/为什么:`docs/judge_model_training_roadmap.md` +- 产品蓝图:`docs/Final_target.md` +- 闭环用法:`docs/coevolution.md` +- 人工审美收集:`docs/aesthetic_ground_truth_questionnaire.md` +- 本文件:落地执行(做什么、用什么、验收标准) diff --git a/docs/module-map-and-migration.md b/docs/module-map-and-migration.md new file mode 100644 index 0000000..d7482ee --- /dev/null +++ b/docs/module-map-and-migration.md @@ -0,0 +1,198 @@ +# Paperimage 现状模块地图与重构迁移清单 + +> 本文是代码导航和迁移计划,不改变任何运行行为。迁移期间以现有 CLI、输出合同和测试结果为准;旧模块保留兼容导入,直到对应阶段完成验收。 + +## 1. 先记住这一条主线 + +```text +论文/参考图 + -> 语义合同(paper_review / figure_specification) + -> 分析与规划(layout / design / controls / semantic binding) + -> 局部资产生成(按 asset policy) + -> figure_program.json + -> PPTX 编译与预览 + -> deterministic QA / benchmark +``` + +语义合同决定“画什么、文字是什么、关系是什么”;参考图决定“怎么排、什么风格”;本地编译器决定“如何生成可编辑 PPTX”。任何重构都不能改变这三个权责边界。 + +## 2. 当前仓库地图 + +### 2.1 用户可见入口 + +唯一 CLI 入口是 `rfs/cli.py`,对应以下产品/实验命令: + +| CLI 命令 | 当前实现 | 定位 | +|---|---|---| +| `paper-to-image` | `rfs.paper_to_image.workflow.run_paper_to_image` | 论文到候选图与论文 grounding 产物 | +| `fast-framework-prompt` | `rfs.paper_to_image.preparation.run_fast_framework_prompt` | 快速语义合同/提示词路线 | +| `paper-to-editable` | `rfs.workflows.paper_to_editable.run_paper_to_editable` | 产品级组合工作流 | +| `rebuild-editable` | `rfs.workflows.rebuild_editable.rebuild_editable`(旧 `rfs.editable_rebuild` 兼容) | 参考图到可编辑 PPTX 的基线路线 | +| `rebuild-editable-pro` | `rfs.professional.rebuild.rebuild_editable_pro`(`rfs.professional_rebuild` 兼容) | controlled professional DSL 实验路线 | +| `rebuild-editable-eval` | `rfs.rebuild_eval.evaluate_rebuild_vlm` | heuristic/hybrid rebuild 对比实验 | +| `coevolve-image` | `rfs.coevolution.orchestrator.run_image_coevolution` | whole-image Creator/Judge 实验 | +| `coevolution-report` | `rfs.coevolution.analysis.analyze_coevolution_run` | co-evolution 指标汇总 | +| `benchmark` / `validate` / `presentations-qa` | `rfs.evaluation.*`、`rfs.presentations_qa` | 验收、评分、QA,不拥有编译权 | + +### 2.2 目标子包(已存在,优先作为新代码落点) + +| 包 | 清晰职责 | 主要现有入口 | +|---|---|---| +| `rfs/contracts` | 稳定 JSON 合同、语义绑定、兼容转换 | `semantic.py` | +| `rfs/analysis` | 论文、参考图、文字/布局分析入口 | `parse_paper`、`extract_reference_text`、`plan_reference_layout` | +| `rfs/planning` | 论文图、重建设计与布局规划 | `plan_paper_image`、`plan_rebuild_design` | +| `rfs/generation` | 参考图候选和槽级 raster 资产生成 | `generate_and_select`、`generate_assets` | +| `rfs/composition` | figure program/PPTX 编译、预览渲染 | `compile_ppt`、`render_rebuild_preview` | +| `rfs/evaluation` | validator、视觉检查、benchmark、回归评分 | `run_rebuild_visual_quality_check`、`run_benchmark_case` | +| `rfs/workflows` | 面向产品的跨阶段编排 | `run_paper_to_editable` | +| `rfs/paper_to_image` | 论文路线的领域实现与中间合同生产 | `run_paper_to_image`、`run_fast_framework_prompt` | +| `rfs/coevolution` | Creator/Judge 研究实验 | `run_image_coevolution` | +| `rfs/providers` | VLM/API 适配、凭据解析、重试 | `call_vlm_json`、`build_rebuild_vlm_adapters` | + +### 2.3 `rfs/` 根目录的现状 + +根目录仍有约 46 个 Python 文件,主要是三类混合物: + +1. **产品核心但尚未完全归位**:`workflow.py` 与少量历史基础设施;`paper_to_editable.py`、`ppt_compiler.py`、`validator.py`、`editable_rebuild.py` 与 `program_builder.py` 均保留为兼容入口。 +2. **重建专用模块**:`layout_*`、`control_*`、`reference_*`、`rebuild_*`、`text_*`、`arrow_router.py`。 +3. **兼容/基础设施/历史接口**:`workflow.py`、`semantic_contract.py`、`asset_generator.py`、`vlm_client.py`、`presentations_qa.py` 等。 + +这些文件不是都应立即移动;当前最重要的是先建立“权威入口”,禁止新增同职责的第三套实现。 + +## 3. 修改代码时应该改哪里 + +| 需求 | 首选修改位置 | 不要直接改的地方 | +|---|---|---| +| 论文实体、标签、关系、证据合同 | `rfs/contracts/`、`rfs/paper_to_image/review_contract.py` | OCR 输出、PPTX XML、视觉 critic | +| 论文解析/grounding | `rfs/paper_to_image/analyzer.py`、`review.py`、`preparation.py` | `rfs/cli.py` | +| 论文到图的布局蓝图/提示词 | `rfs/paper_to_image/planner.py`、`semantic_blueprint.py` | 资产生成器 | +| 参考图全局设计、层、槽、控件 | `rfs/rebuild_design_planner.py`、`layout_planner.py`、`control_localizer.py` | 语义合同编译器 | +| OCR/文本分组/文本层 | `rfs/reference_text_extractor.py`、`text_grouping.py`、`text_layer*.py` | 资产 API client | +| 箭头、连接器和避让 | `rfs/arrow_router.py`,最终由 figure program 消费 | raster 资产生成 | +| 槽级图片策略和生成 | `rfs/asset_generator.py`、`rfs/generation/` | PPTX 编译器 | +| figure program 生成/校验 | `rfs/program_builder.py`、`rfs/paper_to_image/editable_ppt.py` | 参考图 OCR | +| PPTX/预览渲染 | `rfs/composition/`;根目录 `ppt_compiler.py` 仅作兼容 wrapper | workflow 中复制渲染逻辑 | +| 产品级流程编排 | `rfs/workflows/` | 在 CLI 分支中塞业务逻辑 | +| 质量检查/benchmark | `rfs/evaluation/` | 用 presentations QA 反向改合同或 PPTX | +| VLM/API 调用 | `rfs/providers/`(根目录 `vlm_client.py` 仅兼容) | 在业务模块中散落 requests 调用 | +| professional DSL | `rfs/professional_*.py`(迁移期),目标为独立 professional 子包 | 普通 rebuild 模块 | +| Creator/Judge 演化实验 | `rfs/coevolution/`、`experiments/` | 产品 workflow 默认路径 | + +## 4. 权威性规则 + +- CLI 只负责参数解析、依赖组装、结果打印;业务逻辑必须进入 workflow 或领域模块。 +- `rfs/workflows/` 是跨阶段编排层,不重新实现合同、规划或编译。 +- `rfs/contracts/` 是稳定边界;新增字段先改合同/validator,再改生产者和消费者。 +- `rfs/composition/` 是 PPTX 编译权威;`ppt_compiler.py` 不再增加新功能。 +- `rfs/evaluation/` 只检查和报告,不偷偷修写生产合同。 +- `experiments/` 可以试错,但不得被生产包隐式 import。 +- 同一职责只能有一个“新实现”;兼容文件只能转发并标注迁移目标。 + +## 5. 不改变功能的迁移清单 + +### 当前已落地的 rebuild 权威边界 + +后续定位可编辑重建问题时,优先从下表的 canonical 模块进入;根目录旧模块和 +`rfs.editable_rebuild` 仍是兼容入口,不应新增业务逻辑。 + +| 关注点 | Canonical 位置 | 兼容入口 | +|---|---|---| +| 产品编排 | `rfs.workflows.rebuild_editable` | `rfs.editable_rebuild` | +| 参考图布局、控件、文本分析 | `rfs.analysis.rebuild.{layout,controls,reference_text}` | 原 `layout_*` / `control_*` / `reference_*` 模块 | +| 设计和语义规划 | `rfs.planning.rebuild.{design,semantics}` | 原 rebuild planner 模块 | +| 箭头、文本层、figure program | `rfs.composition.rebuild.{arrows,text,program}` | 原 `arrow_router` / `program_builder` 路径 | +| 槽级资产策略和生成 | `rfs.generation.{rebuild_policy,rebuild_assets}` | `rfs.asset_generator`、`rfs.editable_rebuild` | +| 组合覆盖审计与质量检查 | `rfs.evaluation.rebuild_coverage`、`rfs.evaluation` | `rfs.editable_rebuild` | +| PPTX 编译、预览和导出 | `rfs.composition`、`rfs.composition.rebuild.export` | `rfs.ppt_compiler` | +| controlled professional DSL | `rfs.professional` | `rfs.professional_*` | + +这些边界不改变三条产品路线:`paper-to-image`、`paper-to-editable`、 +`rebuild-editable`。Creator/Judge co-evolution 与 Judge 训练仍保留在 +`rfs.coevolution`、`rfs.judge_training` 和 `experiments/`,不被产品默认路径调用。 + +### Phase 0:冻结边界(当前批次) + +- [x] 建立本文档。 +- [ ] 为每个 CLI 命令补充一条“入口 → 产物 → 权威合同 → 编译器 → QA”链路测试。 +- [x] 给已迁移的根目录兼容模块增加模块级 docstring:标明 canonical target 和禁止新增逻辑。 +- [ ] 在 README 增加本文档链接,作为代码定位入口。 + +### Phase 1:先收口入口,不移动文件 + +- [x] `cli.py` 保留总 parser、dispatch 与结果输出;命令参数注册和 rebuild 参数装配迁到 `cli_commands/`。 +- [x] 为 `rebuild-editable` 建立显式 `rfs/workflows/rebuild_editable.py` facade,内部暂时调用旧实现。 +- [x] 为 `rebuild-editable-pro` 建立显式 `rfs/workflows/professional_rebuild.py` facade。 +- [x] 为 workflow facade 增加稳定入口和长期兼容导入测试。 + +### Phase 2:迁移重建实现(兼容导入) + +- [ ] 将 `layout_*`、`control_*`、`reference_*`、`arrow_router.py`、`text_*` 按职责迁入 `analysis/`、`planning/` 或 `composition/` 的明确子模块。(已迁入 analysis/rebuild:layout/control/reference text;已迁入 composition/rebuild:arrows/text/program。) +- [x] 将 `editable_rebuild.py` 拆成:输入/分析、资产策略、figure program、编译 orchestration;总编排已迁入 `workflows/rebuild_editable.py`,figure program、资产基础操作和 combined coverage audit 已分别归入 `composition/`、`generation/`、`evaluation/`。 +- [x] 根目录旧路径保留 re-export,并加入 deprecation 注释;所有内部 import 已改用新路径。 + +### Phase 3:迁移编译和基础设施 + +- [x] 将 `program_builder.py`、`validator.py`、`exporter.py`、`rebuild_preview_renderer.py` 的权威实现归入 `composition/` 或 `evaluation/`。(`program_builder.py` 已迁入 `composition/rebuild/`;`validator.py` 已迁入 `evaluation/`;rebuild export/coverage canonical boundaries 已建立。) +- [x] 将 `vlm_client.py`、模型解析和重试逻辑归入 `providers/`;根目录 `vlm_client.py` 保留兼容 alias。 +- [ ] 保持 `rfs.ppt_compiler`、`rfs.vlm_client` 等旧导入可用,至少跨一个版本周期。 + +### Phase 4:professional 与实验隔离 + +- [x] 将 `professional_*` 收口为一个显式子包和 workflow facade(旧根模块保留兼容 alias)。`professional_rebuild.py` 的权威编排已迁入 `rfs.professional.rebuild`。 +- [ ] 将只服务研究实验的 co-evolution、legacy rebuild、文献脚本与产品代码隔离。 +- [x] CLI help 和文档明确标记 production、experimental、compatibility 三种状态。 + +### Phase 5:删除兼容层(最后做) + +- [ ] 连续两个版本无内部代码使用旧路径。 +- [ ] 全量 unit/integration/e2e、benchmark smoke、CLI help 通过。 +- [ ] 更新插件 skill、README、architecture 文档。 +- [ ] 单独提交删除兼容 wrapper;不与功能修改混合。 + +## 6. 每次迁移的验收门槛 + +迁移只允许改变 import 路径和目录归属,不允许同时改变: + +- CLI 参数名、默认值和命令行为; +- JSON/Markdown/PPTX 输出合同; +- asset policy、smart-api 默认策略和 fallback 记录; +- 语义绑定、箭头路由、文本可编辑性; +- 生产/实验路线的默认选择。 + +每个迁移 PR/commit 必须至少运行: + +```powershell +python -m compileall -q rfs +python -m unittest discover -s tests -q +rfs doctor --json +``` + +涉及 workflow 或合同时,再运行对应 offline smoke、validator 和 benchmark case。 + +## 7. 推荐的提交顺序 + +1. 文档与模块级说明(无行为变化)。 +2. facade/re-export 与测试(无行为变化)。 +3. 内部 import 迁移(无行为变化)。 +4. 小批量文件移动(每批一个职责域)。 +5. 删除空兼容层(单独提交)。 + +不要把“文件移动 + 逻辑重写 + 输出合同变化”放在同一个 commit;这样才能在任何阶段安全回退。 + +## 8. 当前迁移状态(2026-08) + +生产代码的目标归属已经确定,且不改变 CLI 行为或输出合同。已迁移领域的根目录 +模块是薄 alias;`cli.py` 和历史 `workflow.py` 仍是过渡基础设施。 +新代码使用以下位置: + +| Status | Location | Rule | +|---|---|---| +| Production | `rfs.contracts`, `analysis`, `planning`, `generation`, `composition`, `evaluation`, `providers`, `workflows`, `professional` | New behavior belongs here. | +| Experimental | `rfs.coevolution`, `rfs.judge_training`, `experiments/` | May support research experiments, but is not implicitly imported by a product workflow. | +| Compatibility | root-level `rfs` aliases such as `editable_rebuild`, `ppt_compiler`, and `asset_generator` | Preserve imports only; do not add business logic. | +| Transitional infrastructure | root-level `cli` and historical `workflow` | Preserve behavior while new work goes to the owning production package. | + +See [code-structure.md](code-structure.md) for the concise, current navigation +page. Retain compatibility aliases for at least one version cycle, keep +route-level contract tests, and make any eventual alias removal a standalone +backwards-incompatible change. diff --git a/docs/paper-to-image-reference-learning-plan.md b/docs/paper-to-image-reference-learning-plan.md new file mode 100644 index 0000000..032f683 --- /dev/null +++ b/docs/paper-to-image-reference-learning-plan.md @@ -0,0 +1,241 @@ +# Paper-To-Image Reference Learning Plan + +This plan records the staged roadmap for adding a human-reference learning layer to `paper-to-image`. + +The goal is to let Paperimage learn reusable design knowledge from strong scientific framework figures before generating a new paper figure. The system should learn layout grammar, structural division, information density, hierarchy, color use, text strategy, and connector behavior. It should not copy paper-specific labels, logos, or scientific content from reference figures. + +## Product Principle + +Paper content decides what the figure says. + +Reference learning decides how the figure is visually organized. + +The image model generates candidates from the paper contract plus retrieved design knowledge. + +The critic checks both scientific correctness and alignment with human-quality design standards. + +## Target Pipeline + +```text +project reference figures +-> reference analysis workflow +-> API-backed library build +-> paper_to_image_reference_library.jsonl +-> exemplar selection for the current paper +-> exemplar_style_brief.json +-> paper-to-image prompt +-> image candidates +-> scientific / OCR / template / aesthetic / exemplar-alignment review +-> selected_image.png +``` + +## Reference Figure Storage + +Reference images should live in project-level folders instead of requiring users to pass arbitrary local paths for the built-in reference set. + +Stage 1 uses this local folder: + +```text +reference-figures/ + excellent/ + negative/ + user/ +``` + +The real image files should remain local-only by default because paper figures may have copyright or private-data constraints. The repository should commit reusable structured contracts and documentation, not the original reference images. + +## Stage 1: Build The Analysis Workflow + +Purpose: create the code path and schema for analyzing reference figures, before spending API calls on the actual library. + +Deliverables: + +- A reference-learning schema for reusable design contracts. +- A module such as `rfs/paper_to_image/reference_learning.py`. +- A CLI command such as `rfs analyze-reference-figures`. +- Offline placeholder or fixture-backed tests that verify schema shape and output paths. +- Documentation for expected input folders and outputs. +- Output files named `reference_analysis_records.jsonl` and `reference_analysis_report.json`. + +Important: this stage creates the analysis workflow only. It should not require real API calls to pass tests. + +Commit boundary: one commit for the Stage 1 workflow/scaffold. + +## Stage 2: API-Backed Library Build + +Purpose: run the analysis workflow with a VLM/API to analyze the prepared excellent and negative figures. + +Deliverables: + +- `paper_to_image_reference_library.jsonl` generated from project reference folders. +- A build report recording model, input counts, failures, skipped files, and API status without secrets. +- Safety fields for copied terms, copied objects, and content that must not be reused. +- Optional human-editable review notes for correcting VLM analysis. + +Important: this stage is where API analysis and actual library creation happen. + +Stage 2 command: + +```powershell +rfs analyze-reference-figures ` + --references reference-figures ` + --out output\reference_learning_stage2 ` + --mode vlm ` + --model gpt-5.6-sol ` + --json +``` + +The command writes both `reference_analysis_records.jsonl` and `paper_to_image_reference_library.jsonl`. API credentials come from environment variables already loaded into the shell, not from automatic `.env` parsing in code. + +Commit boundary: one commit for API-backed library building support and any non-private generated contract files that are safe to commit. + +## Stage 3: Exemplar Selection For Current Paper + +Purpose: before `paper-to-image` generation, select relevant design examples from the built library. + +Selection should consider: + +- domain profile, such as AI method, system, dataset, empirical science, or survey; +- module count; +- expected topology, such as linear, tree, loop, tripanel, or dense multimodal; +- aspect ratio; +- information density; +- retrieval, multimodal, feedback, or comparison structure; +- positive versus negative quality labels. + +Deliverables: + +- An exemplar retrieval module. +- `exemplar_selection.json`. +- `exemplar_style_brief.json`. +- Tests for deterministic selection from small fixture libraries. + +Stage 3 command: + +```powershell +rfs select-reference-exemplars ` + --paper-review output\paper_to_image\paper_review.json ` + --library output\reference_learning_stage2\paper_to_image_reference_library.jsonl ` + --out output\paper_to_image\reference_exemplars ` + --selected-template output\paper_to_image\selected_template.json ` + --top-k 5 ` + --json +``` + +`paper-to-image` may also receive `--reference-library` and `--reference-top-k` to write the two Stage 3 artifacts during a run. Stage 4 feeds the resulting brief into image prompts. + +Commit boundary: one commit for exemplar selection and brief generation. + +## Stage 4: Prompt Integration + +Purpose: make the selected design knowledge actively guide image generation. + +The generated prompt should include: + +- layout rules; +- structure and section-division rules; +- information-density target; +- hierarchy and emphasis rules; +- palette and contrast rules; +- text length and label-style rules; +- connector grammar; +- hard anti-copy constraints. + +Deliverables: + +- `paper-to-image` loads `exemplar_style_brief.json`. +- `compile_image_prompt()` includes the brief in a structured section. +- `image_prompt.txt` records the reference-derived design guidance. +- Candidate prompt files include the same brief so all generated candidates receive consistent visual guidance. +- Tests confirm the prompt contains the exemplar brief and anti-copy rules. + +Commit boundary: one commit for prompt integration. + +## Stage 5: Exemplar-Aware Critic + +Purpose: make candidate review judge whether the output matches human-quality design standards learned from the reference library. + +Add a review section similar to: + +```json +{ + "exemplar_alignment": { + "layout": 0.0, + "structure": 0.0, + "density": 0.0, + "hierarchy": 0.0, + "color": 0.0, + "text": 0.0, + "connectors": 0.0, + "copied_reference_content": [], + "score": 0.0, + "passed": false + } +} +``` + +Deliverables: + +- Critic prompt includes `exemplar_style_brief.json`. +- Aggregated review writes an exemplar-alignment report. +- Production gate rejects candidates when exemplar alignment fails, and the violations feed the existing repair round. +- Tests cover pass/fail normalization. + +Commit boundary: one commit for critic integration. + +## Stage 6: User-Provided Style Learning + +Purpose: allow a user to upload one style reference image and ask Paperimage to learn visual style preferences without replacing the project-wide design method. + +Suggested flow: + +```text +user style reference image +-> user_style_brief.json +-> combined_style_brief.json +-> paper-to-image generation +``` + +The user style brief is additive and style-only. It can influence color, line, shape, shadow/depth, typography, texture, icon/object rendering, panel surfaces, and overall mood. It must not decide layout topology, section structure, scientific relations, paper modules, visible labels, or copy reference text/content. + +Deliverables: + +- A command such as `rfs learn-reference-style --style-reference`. +- A `--user-style-brief` option for `paper-to-image`. +- `combined_style_brief.json` that distinguishes project-library design rules from user-provided style preferences. +- Tests for style-only learning and combining project exemplar brief with user style brief. + +Commit boundary: one commit for user-provided style learning. + +## Stage 7: Preference Feedback Loop + +Purpose: preserve human judgments over generated candidates so the system can improve later. + +Deliverables: + +- A `record-paper-image-preference` command that records one human-chosen candidate from an existing `paper-to-image` run. +- `preference_pairs.jsonl` for chosen-vs-rejected candidate comparisons generated from the chosen candidate and the other candidates in the run. +- `human_preference_feedback.json` with the reason, reviewer, tags, candidate metadata, and run metadata. +- Optional future training or tuning path for a stronger judge model. + +Commit boundary: one commit for preference logging. + +## Stage 8: Human-Preference-Aware Selection + +Purpose: separate scientific hard gates from human visual preference so visually stronger candidates are not rejected only because template or exemplar alignment is imperfect. + +Deliverables: + +- A critic schema with `scientific_gate`, `visual_preference`, `selection_status`, `hard_gate_errors`, and `soft_alignment_warnings`. +- Soft template and exemplar alignment scoring; copied reference content remains a hard failure. +- Candidate selection by weighted `selection_score`, with repair source selection preferring visually strong repairable candidates. +- Optional `--preference-history` input for prompt-safe summaries of prior `preference_pairs.jsonl`. +- Reports for human visual preference, selection breakdown, selection policy, and repair source selection. + +Commit boundary: one commit for human-preference-aware selection. + +## First Implementation Rule + +Implementation should start with Stage 1. Each stage must be completed and committed separately before moving to the next stage. + +Do not let later-stage work silently enter an earlier-stage commit. diff --git a/docs/paper-to-image.md b/docs/paper-to-image.md index ce5ca36..3bc73a3 100644 --- a/docs/paper-to-image.md +++ b/docs/paper-to-image.md @@ -1,6 +1,6 @@ # Summary -`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Production mode requires successful VLM paper review, a paper-grounded semantic blueprint, reference-conditioned Image2 visual generation, deterministic exact-label/connector composition, and the full production quality gates. +`rfs paper-to-image` produces a reviewed raster scientific framework image without creating PPTX. Image2 itself renders the complete visible figure: exact paper terminology, arrows, modules, illustrations, grouping, and layout. Programmatic label/connector composition is not supported in this route. ## Three-Minute Fast Contract @@ -8,7 +8,7 @@ rfs fast-framework-prompt --paper paper.pdf --out output/fast --deadline 180 --json ``` -The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/contract caches. Layout, style, prompt, and editable overlays are compiled deterministically after the semantic graph. A warm paper normally completes in well under a second. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. +The fast path stops before image generation. It uses structured block-level PDF extraction, complete overview captions, stable page/content-hash evidence IDs, a semantic-only VLM request, evidence-gated generic contract completion, and separate document/raw-semantic-plan caches. Layout, style, prompt, and audit/rebuild contracts are compiled deterministically after the raw semantic graph. A warm paper normally completes in well under a second. Older compiled semantic caches are deliberately not migrated because applying current normalization to an already compiled contract can duplicate or mutate relations. An affected paper therefore needs one fresh VLM compilation; later runs reuse the versioned raw-plan cache and remain stable and repeatable. Cached strings are recursively repaired for common UTF-8 mojibake before any JSON or Markdown artifact is written. If no VLM succeeds, the command returns an engineering result with explicit uncertainties instead of claiming production readiness. Use `rfs inspect-pdf --paper paper.pdf --out output/inspection --json` to diagnose reading order, Unicode, OCR, section coverage, captions, parser agreement, and evidence-page coverage without model calls. Rotated native-text pages are transformed into displayed-page coordinates before ordering, and body text is conservatively classified as one, two, or three columns; spanning titles and captions remain explicit ordering boundaries. OCR lines use the same detector, including a three-to-two-column fallback for first pages where centered title fragments otherwise resemble a third column. Quality gates count all Unicode letters and numbers. English, Spanish, French, German, Portuguese, Chinese, Japanese, and Korean section aliases are recognized; localized `Figure/Table` prefixes and Chinese `图/表` captions are indexed. Caption prefixes must include a real numeric, Roman, or appendix identifier, so ordinary prose such as `figura editable` is not misclassified. CJK cross-extractor agreement uses character bigrams so per-character spacing differences do not resemble corruption. Cross-extractor quality uses directional lexical coverage while retaining order-sensitive agreement for diagnostics, preventing multi-column order differences, equation variables, and Poppler-only hidden text from causing wasteful OCR. The quality report exposes the maximum detected column count, the number of multi-column pages, and all rotated page numbers. For image-only/scanned papers, `--ocr-engine auto` prefers RapidOCR. Up to three selected pages run concurrently, and short narrow or vertical fragments in the outer page margins are removed as likely watermarks/page furniture. OCR candidates are scheduled by semantic signals first (overview figures, Method, Abstract, Conclusion), then by document-wide coverage anchors instead of simply taking the first six pages. `RFS_OCR_WORKERS` overrides page concurrency. @@ -47,11 +47,11 @@ Born-digital and OCR-recovered pages also pass through a conservative repeated-m For pages carrying a PDF rotation flag, native blocks are ordered in the unrotated media-box coordinate system and only then transformed into displayed page coordinates. This preserves Abstract-to-Method-to-Figure reading order on 90/180/270-degree pages while keeping overlay coordinates aligned with rendered output. -Fast VLM planning preserves the paper's original writing system for every visible scientific label. Entity names and relation labels must either occur verbatim in their cited evidence or remain in the same source script; cross-script translations such as `文档编码器` to `Document encoder` fail planning validation and cannot be marked production-ready. Explanatory prose may follow the requested output language, but editable overlay labels do not. +Fast VLM planning preserves the paper's original writing system for every visible scientific label. Entity names and relation labels must either occur verbatim in their cited evidence or remain in the same source script; cross-script translations such as `文档编码器` to `Document encoder` fail planning validation and cannot be marked production-ready. Explanatory prose may follow the requested output language, but Image2-visible labels do not. For Spanish, French, German, and Portuguese papers, validation also detects same-script English translation. An English-looking visible label that does not occur verbatim anywhere in the paper evidence fails production validation even though both strings use Latin characters. This closes the gap where `codificador de documentos` could otherwise be silently changed to `Document Encoder`. -Relation labels are optional: if a model supplies a connector label that is not verbatim in the relation's cited evidence, normalization clears the label while preserving the grounded direction and type. Overlay compilation also emits an exact scientific label only once when the same evidence-backed concept is both a module and an innovation callout. +Relation labels are optional: if a model supplies a connector label that is not verbatim in the relation's cited evidence, normalization clears the label while preserving the grounded direction and type. Contract normalization also requires an exact scientific label only once when the same evidence-backed concept is both a module and an innovation callout. Native PDF lines split by publisher hyphenation are joined inside the same source text block before evidence IDs are created. Known scientific words such as `trans-` + `former` and `de-` + `coder` lose the layout-only hyphen, while unknown long compound parts retain a literal hyphen. The extraction report records `native_hyphenation_repair_count`. @@ -127,10 +127,10 @@ rfs paper-to-image ` `--repair-source` skips fresh initial candidate generation. The source is first reviewed against the current exact-label and scientific contract. If it fails, the best source is edited once and reviewed again. The request manifest marks -the reused source separately from the real Image2 repair request. This remains a -compatibility route for older images whose labels and arrows are already baked -in, so deterministic overlay mode is disabled for an explicit external repair -source. +the reused source separately from the real Image2 repair request. Because the +primary route treats labels and arrows as part of the complete Image2 figure, +an explicit external repair source is reviewed and repaired as a whole image; +deterministic overlay mode stays disabled. Re-audit an existing complete or failed run without regenerating an image: @@ -180,38 +180,46 @@ density, palette, and forbidden copied content. Automatic selection uses module count, loop/tree/parallel-branch structure, multimodality, retrieval structure, and requested ratio. -The selected profile is rendered as `layout_blueprint.png` for review. When a -semantic plan is available, Image2 receives `visual_substrate_blueprint.png` -instead. That guide contains the exact node and panel geometry plus reserved -header bands, but no scientific text and no connectors. Reference-derived -profiles remain free of copied reference text and reference-specific objects. - -## Deterministic Structure Overlay - -New semantic-plan runs enable deterministic overlay mode by default: - -1. Image2 generates `candidates/candidate_XX_raw.png`, containing only the - image-rich visual substrate. -2. The geometry normalizer detects the visual cards, requires the detected card - count to equal the semantic node count, rejects ambiguous matching, removes - only the non-semantic generated header band, and contain-fits each visual - region into the exact semantic node box. It writes - `candidates/candidate_XX_substrate.png` and - `geometry/candidate_XX_geometry.json`. -3. `overlay_spec.json` supplies exact paper terminology and declared directed - relations. -4. `layout_blueprint.json.semantic_plan` supplies normalized node boxes and - connector `path_percent` geometry. -5. The compiler writes `candidates/candidate_XX.png` and - `overlays/candidate_XX_overlay.json`. -6. OCR, scientific, topology, template, aesthetic, and substrate-geometry gates review the - composed candidate, not the raw substrate. - -Image2 prompts explicitly forbid words, letters, numbers, formulas, arrows, -lines, loops, and pseudo-text. Exact labels and relation geometry are therefore -owned by the deterministic compiler. The same semantic structure can later be -emitted as editable PowerPoint labels and connectors; the raster overlay exists -so the current whole-image review path scans the same final appearance. +The selected profile is rendered as `layout_blueprint.png`. By default the CLI +uses direct Image2 generation with the semantic blueprint embedded in the Prompt; +Image2 must render the final visible labels, +connectors, modules, and visual explanations itself. Reference-derived profiles +remain free of copied reference text and reference-specific objects. + +## Complete Image2 Ownership and Density + +Image2 is explicitly required to render the complete figure, including exact +visible labels, arrows, arrowheads, loops, modules, illustrations, and grouping. +`overlay_spec.json` only audits what the image must already contain and supplies +semantic guidance to later image-to-PPT reconstruction. It is never painted onto +the primary raster result. + +The generation prompt targets a balanced compact layout: normally 92-96% +canvas-width and 90-95% canvas-height utilization with 2-4% functional breathing +room. It explicitly forbids adding modules, charts, labels, repeated pipelines, +or decorations merely to increase density. Production review still requires at +least 0.10 active-region change from the bare blueprint. The detailed-tile ratio +defaults to 0.80, while feedback-loop figures use 0.75 because necessary return +arrow corridors otherwise create a systematic false whitespace penalty. Once +the applicable threshold passes, extra density is not treated as an improvement; +readability, balance, exact text, and connector correctness decide between +candidates. Flat color-filled cards and label-only boxes still fail. + +The deterministic density audit records `active_detail_grid_stddev` on a 24x16 +active-region grid and `semantic_node_density` on a 4x4 grid inside every +contracted node. Card borders are excluded from the node interior measurement, +so a crisp outline around an empty card does not count as useful detail. When +the global density gate fails, nodes below the 0.80 diagnostic target are added +to `sparse_semantic_nodes` and `repair_regions`; the repair compiler then emits +node-specific Image-2 instructions and localized masks. + +Semantic normalization also separates external inputs from learned internal +conditioning. Class/query tokens and positional embeddings feeding an internal +model move to the module contract with `role=conditioning`. A generic leaf such +as `Head` is removed when it duplicates its upstream head and has one concrete +sibling output; the exact short output label printed in the selected overview +figure is preferred. Topology is re-inferred after late normalization so a +removed false branch cannot leave stale `topology=branch` metadata. The connector compiler uses global candidate routing rather than independent per-edge elbows. It sorts fan-in/fan-out ports by geometry, reserves distinct @@ -225,35 +233,6 @@ segments, bend count, and path length. `semantic_plan.route_quality` records: - per-connector `lane_assignments` - the routing algorithm and pass state -Geometry normalization is a hard contract step before VLM review. Production -requires exact card/node cardinality, unique matching, exact post-normalization -semantic boxes, restored connector corridors, and `semantic_crop_used=false`. -Failures use `failure_stage=substrate_geometry`, never provider failure. A VLM -critic cannot override this gate. - -Overlay compilation is also a hard contract step. Missing label targets, missing -connector geometry, or incomplete paths fail as `failure_stage=overlay_compilation` -and are not misreported as Image2 provider failures. - -Repair ownership is also separated. OCR, spelling, exact-label, arrow, and -topology findings are assigned to `overlay_compiler`; visual density, mechanism -illustration, hierarchy, balance, and substrate aesthetics remain assigned to -`image2_visual_substrate`. Image2 repair prompts never ask the model to modify -text or connectors in deterministic overlay mode. If only compiler-owned issues -remain, the repair loop stops with `overlay_recompile_required` instead of -waiting on an unnecessary image-provider edit. - -`--resume-candidates` prefers the raw/normalized/composed chain. A raw candidate is -re-normalized, re-overlaid, and re-reviewed using the current semantic boxes and routes. A legacy run that -contains only the composed PNG is reviewed without applying the overlay twice; -it cannot pass the new substrate-geometry gate or produce an editable substrate -PPT until the missing raw candidate is regenerated. - -With `--editable-overlay-ppt`, the selected normalized substrate becomes the -bottom PPT image layer. Node outlines, exact labels, relation labels, halo paths, -and arrowheads are native editable PowerPoint objects. The PPT never uses the -geometrically drifting raw Image2 canvas as its structural source. - For supported normalized topologies (`linear`, `branch`, `multimodal`, `feedback`, and `dense_multiframe`), contracts with 2-16 visible entities are compiled into `layout_blueprint.json.semantic_plan`. The plan records exact @@ -347,7 +326,13 @@ input shortcuts that bypass an explicit intermediate representation. The general production review and focused topology review execute concurrently for every contract that declares visible relations, including linear pipelines. -Their default request timeout is 90 seconds with no automatic retry, preventing +Both default to `gemini-2.5-flash` when no review-specific model environment +variable is set. The focused review uses two passes: first verify all mandatory +directed relations, then exhaustively enumerate visible solid, dashed, curved, +faint, bundled, and cross-panel connectors. The second-pass inventory is +compared deterministically with the contract, so an undeclared visible edge can +only become an invented relation and cannot be reinterpreted as a missing edge. +Their default request timeout is 30 seconds with no automatic retry, preventing two independent judges from creating a multi-minute sequential stall. Configure `RFS_PAPER_TO_IMAGE_REVIEW_TIMEOUT` and `RFS_PAPER_TO_IMAGE_TOPOLOGY_TIMEOUT` when a slower provider requires a larger @@ -355,21 +340,59 @@ budget. Existing candidates can be passed through `--repair-source` so review or localized repair does not regenerate the correct parts of the image. The focused topology review is authoritative only for visible connector -geometry. When it cleanly passes every declared relation, connector-only false -positives from the general critic are suppressed and recorded in -`cross_critic_reconciliation`; paper semantics, entity roles, OCR, and -aesthetics remain under their original hard gates. If both critics identify the -same missing, bypassed, reversed, or invented connector, the failure remains -blocking. - -Image-2 edit requests default to a 120-second timeout controlled by -`RFS_IMAGE2_TIMEOUT`. Network timeouts stop the candidate immediately instead of -blindly regenerating the entire image. Retryable HTTP or provider failures may +geometry. The default production review now asks one VLM call for OCR, +paper alignment, aesthetics, and structured topology. Every relation must return +`source_departure_side`, `target_arrival_side`, and `arrowhead_contact`; complex +feedback, return-loop, and cross-panel edges cannot pass on a vague description +such as `direct arrow`. If the integrated response omits endpoint evidence, one +short focused topology call is used as a fallback. Set +`RFS_PAPER_TO_IMAGE_FOCUSED_TOPOLOGY=1` only when an independent topology call +should always run. Paper semantics, entity roles, OCR, and aesthetics remain +hard gates. + +OCR review records one item per physical text occurrence in +`visible_text_occurrences`. This lets the deterministic gate reject duplicated +non-repeatable labels even when the critic's distinct-string summary lists the +word only once. Labels in `repeatable_labels` remain explicitly exempt. + +Every candidate review writes `paper_image_comparison_report.json`. This single +report combines exact-text findings, paper-evidence alignment, declared relation +coverage, missing/reversed/bypassed/invented connectors, template alignment, +information density, aesthetics, blocking issues, and an explicit next repair +action while recording that Image2 owns the complete visible raster. + +`paper-to-image` defaults to topology-aware initialization through +`--image-init-mode topology`. Feedback-loop, linear, branch, multimodal, and dense-multiframe figures use +blueprint-conditioned full redraw after passing their five-seed stability +benchmarks; other topologies use direct generation until their own multi-seed +benchmarks support a safer default. Explicit `generate`, `edit`, and +timeout-fallback `auto` remain available. + +The SAM dense-multiframe benchmark is recorded in +`output/evolution/sam_dense_stability_v21.json`: five completed Image2 seeds, +five production passes, and zero incorrect selections. Its blueprint keeps the +Task, Model, and Data graphs separate, anchors panel-level dashed arrows on the +outer panel boundary, and fills only existing cards with mandatory unlabeled +paper-grounded visual cues. The density gate is frozen: a candidate just below +the generic texture threshold may pass only when it also shows material Image2 +enrichment over the engineering blueprint; near-blank blueprint copies remain +blocked. +The CLI default per-candidate timeout is 45 seconds through `--image-timeout`. +Low-level edit and generation timeouts may also be controlled by +`RFS_IMAGE2_TIMEOUT` and `RFS_IMAGE2_GENERATION_TIMEOUT`. Network timeouts stop +the candidate immediately instead of blindly regenerating the entire image. Retryable HTTP or provider failures may retry up to `--image-retries`; the default is one. +`RFS_IMAGE_QUALITY` optionally sends `low`, `medium`, or `high` to the +OpenAI-compatible Image2 endpoint. When unset, the provider default is preserved. +The selected value is recorded in `generation_parameters.json` and every request +record in `image2_request_manifest.json`. Quality affects only generation cost +and latency; exact text, paper alignment, topology, density, and aesthetics use +the same production gates at every quality level. + A full provider read/connect timeout also stops the multi-round repair loop with `repair_stop_reason=provider_timeout`; another nominal repair round must not -repeat the same 120-second blind wait. Fast failures such as HTTP 429 remain +repeat the same provider wait. Fast failures such as HTTP 429 remain provider failures rather than semantic non-improvement and may use later repair rounds. @@ -427,9 +450,8 @@ rfs paper-to-image ` --resume-candidates ``` -Existing raw candidates are re-composed and re-reviewed with the current gates; -legacy composed-only candidates are re-reviewed without duplicate overlay, and -only missing candidates are generated. A three-seed stability run also permits +Existing complete candidates are re-reviewed with the current gates, and only +missing candidates are generated. A three-seed stability run also permits one bounded replacement for a provider-failed seed; successful candidates are never regenerated by that replacement path. @@ -439,17 +461,20 @@ geometry while the generation canvas uses the nearest native Image2 ratio ## Production Gates -Every Image2 candidate must pass: +Every Image2 candidate in the primary complete-image route must pass: -- exact one-to-one visual-card detection and semantic geometry normalization, - with no ambiguous mapping and no semantic crop; +- complete-image ownership: visible text, arrows, modules, and illustrations + are present in the Image2 result rather than added by a later overlay; +- target-framework alignment against `target_figure.json`, including direct + caption evidence and sufficient target-caption terminology coverage; - exact-label OCR with no missing, misspelled, duplicate, or copied reference labels; - scientific module and relation checks with no invention or reversal; - for every declared visible relation, a second focused topology review that verifies visible connector endpoints and arrowheads, rejects bypasses, and writes `topology_critic_report.json`; -- template alignment score of at least `0.70` with no copied reference content; -- aesthetic score of at least `0.70`; +- template alignment score of at least `0.72`, including visual-density match of at least `0.85`, with no copied reference content; +- aesthetic score of at least `0.80`, visual-information density of at least `0.88`, mechanism visualization of at least `0.80`, and whitespace score of at least `0.84`; +- active-region blueprint enrichment of at least `0.10` and detailed-tile ratio of at least `0.80`; - valid image resolution and blueprint aspect ratio. Scientific score must be at least `0.95`; OCR must be exact. If three candidates fail, the best candidate receives one localized Image2 edit repair. If the repair still fails, the command stops without writing `selected_image.png`. diff --git a/docs/rebuild-editable.md b/docs/rebuild-editable.md index 2d9a42a..7fc1d00 100644 --- a/docs/rebuild-editable.md +++ b/docs/rebuild-editable.md @@ -60,7 +60,7 @@ rfs rebuild-editable-pro --reference input.png --out output\demo_pro_placeholder --regenerate-slots Comma-separated slot ids to rerun. --text-mode ocr | manual | off. Default: ocr. --layout-mode heuristic | vlm | hybrid. Default: hybrid. ---ocr-engine paddle | easyocr | off. Default: paddle. +--ocr-engine paddle | easyocr | off. Default: easyocr. Paddle remains available and falls back to EasyOCR if the local runtime fails. --ocr-lang en | ch | en_ch. Default: en_ch. --control-mode heuristic | vlm | hybrid | manual. Default: hybrid. --skip-analysis Reuse existing JSON contracts in --out. @@ -95,6 +95,7 @@ Hybrid VLM layout/control/semantic planning uses an OpenAI-compatible chat compl API_BASE API_KEY or GEMINI_API_KEY MODEL_VLM +RFS_REBUILD_DESIGN_MODEL optional, global design and panel skeleton RFS_REBUILD_LAYOUT_MODEL optional RFS_REBUILD_CONTROL_MODEL optional RFS_REBUILD_SEMANTIC_MODEL optional @@ -107,7 +108,8 @@ PowerShell example: $env:API_BASE='https://your-openai-compatible-provider/v1' $env:API_KEY='' $env:MODEL_VLM='your-vision-language-model' -$env:RFS_REBUILD_LAYOUT_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_DESIGN_MODEL=$env:MODEL_VLM +$env:RFS_REBUILD_LAYOUT_MODEL='gemini-3.5-flash' $env:RFS_REBUILD_CONTROL_MODEL=$env:MODEL_VLM $env:RFS_REBUILD_SEMANTIC_MODEL=$env:MODEL_VLM $env:RFS_PROFESSIONAL_REBUILD_MODEL=$env:MODEL_VLM @@ -125,6 +127,8 @@ The workflow writes these files: input_manifest.json reference_geometry.json reference_geometry_overlay.png +reference_panel_layout_plan.json +reference_panel_layout_overlay.png reference_text_geometry.json reference_controls.json reference_controls_overlay.png diff --git a/docs/rebuild-process-judge.md b/docs/rebuild-process-judge.md new file mode 100644 index 0000000..463a450 --- /dev/null +++ b/docs/rebuild-process-judge.md @@ -0,0 +1,68 @@ +# Rebuild 过程监督 Judge + +这是「分块进化 / 过程监督」的第一版:给 `rebuild-editable` 管线加一层 append-only +运行日志,并配一个能「看过程、按阶段归因」的 Judge。它对应 `docs/paperimage.docx` +里的痛点二(「缺少从论文理解…到视觉渲染的过程证据,难以确定缺陷产生的具体环节」)。 + +当前它是 **prompt-as-judge**:不依赖训练好的模型,而是调通用 VLM 扮演评委。它产出的 +`stage_feedback` 本身就是将来微调「过程监督 Judge 模型」的标注数据。 + +## 产物(都在 rebuild 输出目录里) + +| 文件 | 给谁看 | 内容 | +|---|---|---| +| `run_log.jsonl` | 机器 | append-only 事件流(每个阶段 started/completed/failed) | +| `run_report.md` | 人 | 阶段→状态→耗时→产物的 Markdown 表 | +| `rebuild_process_judgement.json` | 机器 | Judge 的阶段归因:`stage_feedback` | +| `rebuild_repair_plan.json` | 人/机器 | v1 粗修复计划(fault 阶段 → 动作) | + +事件字段约定见 [`docs/run-log-events.md`](run-log-events.md)。 + +## 用法 + +```powershell +# 1. 跑一次重建(会自动写 run_log.jsonl + run_report.md) +rfs rebuild-editable --reference "C:\path\ref.png" --out "output\demo" ` + --asset-mode api --asset-policy smart-api --layout-mode hybrid ` + --control-mode hybrid --text-mode ocr --export-preview + +# 2. 让过程 Judge 看过程、按阶段归因(需要 API_BASE / API_KEY) +rfs rebuild-judge --out "output\demo" --repair-plan --json +``` + +`rebuild-judge` 的 `--reference` / `--preview` 可覆盖,默认从 `rebuild_result.json` 读取; +`--model` / `--timeout` / `--retries` 可覆盖 VLM 配置;`--repair-plan` 才额外生成修复计划。 + +## 设计要点 + +- **事实与视角分离**:`run_log.jsonl` 只存事实;`rfs/run_projection.py` 把同一条日志 + 投影成「给人看的报告」和「给 Judge 的轨迹」两个视图,不是两套日志。 +- **日志非权威**:快照 JSON(`figure_program.json` 等)仍是可信状态来源;删掉日志重跑 + 不应改变任何快照。 +- **阶段归因**:Judge 输出 `stage_feedback`(`{stage, phase, verdict, problem, suggested_fix}`), + 把缺陷定位到具体环节,而不是只说「图难看」。 +- **续跑升级**:`--skip-analysis` 的续跑判断从「文件存在」升级为「日志里该阶段上次 + `completed` 且 `ok`」,旧目录无日志仍回退文件存在判断。 + +## v1 / v2 边界 + +**v1(已实现)**:fault 阶段映射到三个现有粗开关—— + +| Judge 判错在 | 动作 | +|---|---| +| 分析段(design/layout/control/text/semantic) | `reanalyze`(整段重分析) | +| `asset_generation` | `regen_assets` | +| `compile` | `recompile` | + +**v2(未实现,后续单独做)**: + +- 细粒度「只重跑某一个阶段」的开关(现在 pipeline 没有按阶段重跑能力)。 +- 自动修复闭环 `--loop N`(重跑 → 再判 → 按 fault 只修 → 再判,直到 ok 或到轮数)。 + +## 关联模块 + +- `rfs/run_log.py` —— `EventLog`(append-only 日志 + fold/last_event)。 +- `rfs/run_projection.py` —— 投影(`render_run_report` / `project_judge_trajectory`)。 +- `rfs/rebuild_process_judge.py` —— `judge_rebuild_run`(过程监督 Judge)。 +- `rfs/rebuild_judge.py` —— `run_rebuild_judge`(编排 + v1 修复计划)。 +- `rfs/coevolution/judges.py` —— 已有的「整图 Judge」(全局进化),与此处「过程 Judge」互补。 diff --git a/docs/run-log-events.md b/docs/run-log-events.md new file mode 100644 index 0000000..2370a7b --- /dev/null +++ b/docs/run-log-events.md @@ -0,0 +1,67 @@ +# run_log.jsonl 事件 Schema + +`rebuild-editable`(后续扩展到 `paper-to-editable` 全链路)在运行目录下追加写一个 +`run_log.jsonl`,作为**运行事实的 append-only 日志**。它本身不是权威数据——各阶段的 +快照 JSON(`figure_program.json`、`reference_geometry.json` 等)仍然是可信状态来源, +日志只是「索引 + 历史」。删除日志重跑,不应改变任何快照产物。 + +设计原则(本项目从 Harness 范式学习的部分): + +1. **事实与视角分离**:日志只存事实,每个消费者写自己的投影(`fold`)来得到视图。 + 给人看的是 `run_report.md`,给 Judge 做过程监督的是压缩后的轨迹摘要——它们是同一条 + 日志上的两个投影,不是两套日志。 +2. **whole-value 规则**:携带状态的事件(`stage/completed`)必须携带该阶段产出的 + **最小完整摘要**(`summary`),不能只写「done」;全量数据在快照文件里,靠 + `artifacts` 指针关联。 +3. **可归因**:每个阶段事件带 `stage` 和 `phase`,让 Judge 能把缺陷定位到具体环节 + (对应 `docs/paperimage.docx` 痛点二的过程监督要求)。 + +## 通用字段 + +每条事件都是单行 JSON,UTF-8、`ensure_ascii=False`,追加顺序写入: + +| 字段 | 类型 | 含义 | +|---|---|---| +| `seq` | int | 本次运行内单调递增的事件序号 | +| `ts` | float | Unix 时间戳(墙钟,便于人看) | +| `type` | string | 事件类型(见下表) | + +## 事件类型 + +| type | 必需字段 | 可选字段 | 含义 | +|---|---|---|---| +| `run/started` | `workflow`, `reference`, `out_dir` | `params{}` | 一次运行开始 | +| `stage/started` | `stage` | `phase`, `mode`, 其他 meta | 某阶段开始 | +| `stage/completed` | `stage`, `ok=true`, `duration_ms` | `phase`, `artifacts[]`, `summary{}` | 某阶段成功结束 | +| `stage/failed` | `stage`, `ok=false`, `duration_ms`, `error`, `error_type` | `phase` | 某阶段失败(异常不吞,仍上抛) | +| `run/completed` | `ok=true` | `pptx`, `preview` | 运行成功结束 | +| `run/failed` | `ok=false`, `error` | — | 运行失败 | + +`summary{}` 是 whole-value 的最小完整摘要,示例: + +```json +{"seq": 12, "ts": 1719000031.1, "type": "stage/completed", "stage": "build_program", + "phase": "layout", "ok": true, "duration_ms": 18200, + "artifacts": ["reference_geometry.json", "slot_inventory.json"], + "summary": {"slots": 12, "panels": 3, "arrows": 5}} +``` + +## phase 映射(docx 五个概念环节 → rebuild 具体阶段) + +`docs/paperimage.docx` 把过程监督的环节定为: +`论文理解 → 内容抽取 → 拓扑构建 → 布局规划 → 视觉渲染`。rebuild 段的映射如下 +(后续扩展到 paper-to-image 段时补 `paper_understanding` 与完整的 `content_extraction`): + +| phase | 环节 | rebuild 阶段 | +|---|---|---| +| `topology` | 拓扑构建 | `design_plan` | +| `layout` | 布局规划 | `build_program`, `localize_controls` | +| `content_extraction` | 内容抽取 | `slot_semantics` | +| `rendering` | 视觉渲染 | `text_layer`, `compile` | + +## 消费者(投影) + +- 人看:`run_projection.render_run_report()` → `run_report.md`。 +- Judge 过程监督:`run_projection.project_judge_trajectory()` → 压缩轨迹文本注入 Judge prompt。 +- 续跑水位线:`EventLog.last_event(path, "stage/completed", stage=...)` 或 + `run_projection.last_completed_stage()`。 diff --git a/docs/target_evolution_pipeline_plan.md b/docs/target_evolution_pipeline_plan.md new file mode 100644 index 0000000..dc67694 --- /dev/null +++ b/docs/target_evolution_pipeline_plan.md @@ -0,0 +1,429 @@ +# Paperimage 目标链路构建计划 + +> 状态:目标架构与分期实施计划。 +> 范围:论文 → 整体框架图 → 可编辑 PPTX 的科学性与美观性协同进化。 +> 本计划以 `docs/paperimage.docx` 的方法主张为准,并与当前 Paperimage 的工作流和合同文件对齐。 + +## 1. 目标与边界 + +Paperimage 的目标不是生成一张看起来合理的图片,而是稳定地产出: + +1. 科学上忠实表达论文方法的整体框架图; +2. 视觉上清晰、专业、有完整叙事的整体图; +3. 保留文本、箭头、连接器、面板、公式、分组等关键对象可编辑性的 PPTX; +4. 可追溯、可归因、可回退、可跨论文验证的进化过程。 + +系统的优化优先级固定如下: + +```text +科学事实与关键关系正确 + > 方法覆盖和叙事清晰 + > 可编辑性与结构保留 + > 全局美观性 + > 局部视觉细节 +``` + +科学硬错误不能被审美高分抵消。核心模块缺失、虚构科学模块、关键关系反向、训练/推理混淆、核心术语错误均直接触发 `gate_fail`。 + +## 2. 总体双阶段链路 + +```text +阶段 A:论文 → 整体框架图(全局进化) + +论文 PDF + → 论文证据与科学合同 + → 全图结构/叙事/风格候选 + → 整体框架图候选 + → 科学 Grounding Gate + 全图 Judge + → 选优、修复、跨论文验证 + → approved_image.png + +阶段 B:整体框架图 → 可编辑 PPTX(全局—分块协同进化) + +approved_image.png + 论文科学合同 + → 全局图理解、布局与分块候选 + → 选择分块/接口/对象职责方案 + → 分块资产重建 + 原生 PPT 对象生成 + → 全局 PPT 装配 + → 科学保留 Gate + 全局/分块/接口/PPT Judge + → 局部修复或全局重规划 + → editable_composition.pptx +``` + +阶段 A 的生成单位始终是**整图**,不要求提前生成局部块。阶段 B 才将已通过科学门禁的整图拆为可编辑重建所需的局部资产和 PPT 原生对象。 + +阶段 B 不是“先全局、后分块”的一次性流水线,而是下列交替循环: + +```text +全局规划 → 分块重建 → 全局装配 → 联合评判 + ↓ + 局部修复 ← 归因 → 全局重规划 + ↓ + 再装配、再评判 +``` + +只有上游整图存在科学错误,或经验证无法以合理对象边界拆解时,阶段 B 才向阶段 A 发出“建议重生成”的反馈;正常的 PPT 重建问题不应触发整图重画。 + +## 3. 阶段 A:论文到整体框架图的全局进化 + +### 3.1 输入与科学合同 + +首先建立可审计的论文合同,而不是直接生成图。建议固定以下产物: + +```text +paper_review.json +figure_specification.json +scientific_ground_truth.json +``` + +合同至少定义:研究问题、核心贡献、必须出现的模块、输入输出、训练/推理阶段、实体、关系、术语、允许简化项、禁止虚构项和对应论文证据。 + +每个科学节点和关系必须有稳定 ID 与证据链接: + +```json +{ + "id": "node_encoder", + "label": "Feature Encoder", + "role": "core_method_module", + "must_show": true, + "evidence": [{"page": 3, "quote": "..."}] +} +``` + +### 3.2 全图候选与全局进化变量 + +每轮输出多张整体候选图。候选之间可改变: + +- 模块层级、分组与阅读方向; +- 主流程、支路、反馈环的表达; +- 内容密度、留白、视觉隐喻和配色; +- 全局风格与视觉节奏; +- 模块边界清晰度。 + +整图生成阶段不切局部资产,但应考虑下游重建,额外优化 `decomposability_score`(可分解性): + +- 模块区块边界是否清楚; +- 文本、箭头、面板和局部视觉是否可区分; +- 关键关系是否依赖难以分离的复杂纹理; +- 跨区连接是否可识别; +- 是否有足够留白让 PPT 中的可编辑连接器路由。 + +可分解性是图质量的一项正向标准,不是要求牺牲图的视觉表现来迎合下游。 + +### 3.3 阶段 A 评判与选择 + +图像 Judge 输入为:论文证据摘要、科学合同、候选整图及审美 Profile。输出必须分项、可引用、可执行: + +```json +{ + "coverage_score": 0.0, + "relation_correctness": 0.0, + "terminology_accuracy": 0.0, + "narrative_score": 0.0, + "aesthetic_score": 0.0, + "decomposability_score": 0.0, + "blocking_issues": [], + "evidence": [], + "preserve": [], + "repair": [] +} +``` + +独立 Grounding Gate 根据 `must_show`、`must_not_invent`、实体、关系及术语进行确定性/半确定性核验。其失败优先级高于所有软分数。 + +Online Judge 用于提出修复与选择候选;Frozen Judge 用于决定是否接受新最佳候选。两者不得在同一进化周期同步升级。 + +### 3.4 阶段 A 的停止与经验沉淀 + +整图通过条件:无科学 blocker,达到科学与审美的最低阈值,且可分解性不低于产品阈值。每条修复经验必须记录为“条件 → 行动 → 结果”,并先在本案例、再在留出论文上验证;只在跨论文不退化时升级为通用策略。 + +## 4. 阶段 B:整体图到可编辑 PPTX 的全局—分块协同进化 + +### 4.1 全局规划的职责 + +阶段 B 的全局进化不重新创作图片,而是为 PPT 重建选择全局可编辑结构: + +- 阅读顺序、面板层级与页面区域; +- 模块与视觉区域如何映射; +- 分块方案及每块边界; +- 关键文本、公式、箭头、面板、控制符应由何种 PPT 原生对象承载; +- raster 局部资产的职责边界; +- slot 的输入/输出端口; +- 连接器的路由、避让、虚线反馈环; +- 装配后的视觉还原约束。 + +全局规划输出: + +```text +reference_logic_plan.json +reference_layer_plan.json +reference_flow_graph.json +segmentation_candidates.json +segmentation_plan.json +``` + +`segmentation_plan.json` 是新增的显式合同。它不只是图像切块坐标,而是“语义—视觉—可编辑对象”的职责分配: + +```json +{ + "plan_id": "segmentation_02", + "slots": [ + { + "slot_id": "encoder_visual", + "semantic_node_ids": ["node_encoder"], + "bbox_percent": {"x": 0.1, "y": 0.3, "w": 0.2, "h": 0.3}, + "visual_asset_scope": ["encoder_illustration"], + "editable_object_ids": ["label_encoder", "edge_input_encoder", "edge_encoder_decoder"], + "input_ports": ["left_center"], + "output_ports": ["right_center"] + } + ], + "rationale": [] +} +``` + +### 4.2 分块策略是进化变量 + +同一整图可以存在多个分块候选。例如按科学模块、按视觉卡片、按面板层级、或将装饰性视觉与科学模块分离。分块策略的优化目标为: + +```text +分块质量 = +语义模块一致性 ++ 视觉区域完整性 ++ 原生对象可编辑性 ++ 连接可恢复性 ++ 局部可修复性 ++ 装配后的整体还原度 +- 过度切分 +- 不当合并 +- 文本/箭头/公式被烘焙进 asset +- 跨块科学关系丢失 +``` + +以下情况应重新分块,而非重画上游整图: + +- 一个 slot 合并了本应独立编辑的两个科学模块; +- 一个模块被拆得过碎,导致接口和风格失控; +- 标签、箭头或公式残留在 raster asset 中; +- slot 边界切断视觉主体或破坏端口; +- 分块后无法保持关键关系的可编辑连接。 + +### 4.3 分块与全局装配的协同循环 + +每次选择一个全局候选后,执行: + +```text +1. 生成/选择 segmentation plan; +2. 对每个 slot 生成或提取局部视觉资产; +3. 用 PPT 原生对象创建文字、箭头、面板、公式与分组; +4. 根据 ports 和 flow graph 路由连接器; +5. 编译 figure_program.json 与 editable_composition.pptx; +6. 渲染 preview; +7. 运行科学保留 Gate、全局 Judge、分块 Judge、接口 Judge、PPT 编辑性检查; +8. 按归因执行局部修复或全局重规划; +9. 重新装配并比较修复前后结果。 +``` + +局部最优不能替代全局回评:某 slot 单独合格,也可能在装配后造成拥挤、风格断裂、箭头交叉或主流程弱化。 + +### 4.4 阶段 B 的问题归因与修复范围 + +| 问题 | 默认修复动作 | +|---|---| +| 论文核心模块或关系在 PPT 中丢失/改变 | `reanalyze_semantics`,必要时 `resegment` | +| 分块边界不合理、过拆或合并 | `resegment` | +| 整体布局拥挤、主流程弱、面板层级混乱 | `relayout` | +| 局部图像资产错误、模糊或风格不一致 | `regen_assets(slot_ids)` | +| 标签、公式、端口或箭头错位 | `repair_interfaces(object_ids, edge_ids)` | +| 连接器路由不正确 | `reroute(edge_ids)` | +| PPT 对象缺失、顺序或导出异常 | `recompile` | +| 上游整图科学错误或不可合理分解 | `request_regenerate_image` | + +正常情况下,阶段 B 只修复当前对象或重新规划 PPT 结构;`request_regenerate_image` 是向阶段 A 的升级反馈,不是默认路径。 + +## 5. Judge 系统与训练路线 + +### 5.1 Judge 的职责分层 + +```text +训练 VLM Judge:分项评分、证据引用、错误定位、修复建议 + ↓ +Grounding Gate:实体、关系、术语、must_show、must_not_invent 的硬门禁 + ↓ +代码聚合:按固定权重得分、阈值和接受/回退决策 +``` + +Judge 不自行决定最终总分,也不能让审美分抵消科学 blocker。 + +### 5.2 两个 Judge 任务阶段 + +**图像 Judge(优先建设)** + +输入:论文证据、科学 Ground Truth、候选整图和审美 Profile。 +任务:论文方法覆盖、实体关系正确、术语、整体叙事、美观性、可分解性、A/B 偏好、全图修复建议。 + +**PPT Judge(图像 Judge 通过后扩展)** + +输入:论文科学合同、`approved_image.png`、PPT 渲染预览、`figure_program.json`、对象树/统计。 +任务:科学保留、视觉还原、文本/箭头/公式/分组可编辑率、分块合理性、接口与对象级修复建议。 + +### 5.3 训练数据和人工反馈 + +训练样本必须让标注者同时看到论文证据摘要和候选图,不能只按图片主观打分。人工标签至少拆为: + +- 科学状态:`correct / incomplete / incorrect / unverifiable`; +- blocker:缺核心模块、虚构模块、关系反向、训练推理混淆、术语错误; +- 审美:布局、可读性、信息密度、配色、风格偏好; +- 修复:保留什么、修改什么、修改后是否真正提高。 + +重点数据集包括:真实生成轨迹、人工认可图、失败候选、成对偏好、修复前后轨迹,以及可控合成错误。必须覆盖 `beautiful_but_wrong`、`correct_but_ugly`、`relation_reversal`、`missing_core_stage`、`invented_module`、`terminology_corruption` 等对抗集。 + +训练顺序保持:结构化 SFT → Pairwise 排序 → 科学硬错误加权 → 建议有效性 → 分数校准。按论文/GT ID 切分训练、验证、隐藏测试,禁止同一论文跨集合泄漏。 + +## 6. 与现有代码的映射和改造计划 + +| 目标能力 | 当前基础 | 下一步改造 | +|---|---|---| +| 论文到整图全局进化 | `rfs/coevolution/`、`rfs/coevolution/groundtruth.py` | 把 `figure_specification`/论文证据稳定接入 Ground Truth;增加可分解性评分 | +| 图像 Judge 训练 | `rfs/judge_training/`、训练计划文档 | 落地 `JudgeExample`、数据 builder、corruption generator、grounding gate、split/evaluator/registry | +| 参考图到 PPT 规划 | `rfs/planning/rebuild/design.py` | 输出多套 `segmentation_candidates` 和选中的 `segmentation_plan`;绑定 semantic nodes、ports、editable scopes | +| 分块资产重建 | `rfs/workflows/rebuild_editable.py`、`rfs/generation/rebuild_assets.py` | 从 `segmentation_plan` 派生 asset specs;支持按 slot 重生与资产验收 | +| PPT 原生编译 | `rfs/composition/rebuild/`、`figure_program.json` | 保持本地 deterministic compiler 权威;增强对象 ID 与科学节点/关系绑定 | +| 过程监督 | `rfs/run_log.py`、`rfs/run_projection.py`、`rfs/rebuild_process_judge.py` | 记录 scientific、segmentation、interface 事件;Judge 输出 node/slot/edge/object 级归因 | +| 自动修复 | `rfs/rebuild_judge.py`、`rfs/rebuild_evolution.py` | 从 coarse `reanalyze/regen_assets/recompile` 升级为 `resegment/relayout/reroute/repair_interfaces`,并保留 coarse fallback | + +现有稳定合同和编译边界不应被打破:VLM 只生成 JSON 规划、评价和修复建议;本地 Python 继续编译 PPTX;文字、箭头、连接线、面板和公式保持 PPT 可编辑对象;局部图像使用 contain/no-crop。 + +## 7. 新增合同、运行事件和核心输出 + +建议新增或标准化下列产物: + +```text +scientific_ground_truth.json +segmentation_candidates.json +segmentation_plan.json +scientific_grounding_report.json +joint_judgement.json +repair_decision.json +experience_records.jsonl +``` + +每个运行日志应能记录: + +```text +paper_contract/created +image_candidate/generated +image_candidate/judged +image_candidate/approved +segmentation/candidate_created +segmentation/selected +slot/generation_started +slot/judged +interface/routed +ppt/compiled +scientific_gate/passed | scientific_gate/failed +repair/accepted | repair/rolled_back +experience/promoted | experience/rejected +``` + +最终 `paper_to_editable_result.json` 应至少汇总科学覆盖、关系正确性、全局叙事、可分解性、分块选择、slot 修复、PPT 可编辑率、修复轮次和最终 gate 状态。 + +## 8. 分期实施与门禁 + +### P0:合同与观测性 + +- 冻结论文科学合同和统一 JudgeExample 结构; +- 在现有 run log 中记录论文合同、整图候选和阶段 B 的对象级产物; +- 为现有轨迹建立可追溯 manifest。 + +门禁:所有结论都能追溯到论文证据、候选、Judge 输出和编译产物。 + +### P1:图像 Judge 的科学 Gate + +- 实现 Grounding Gate v0:OCR 术语、必须实体、虚构实体、关键关系; +- 建设 D0 数据和人工复核的科学负样本; +- 将 Gate 接到 `coevolve-image` 的 Frozen 接受条件。 + +门禁:已知注入科学错误有高召回,且美观但错误的对抗图不能通过。 + +### P2:整图全局进化 + +- 将 `paper-to-image` / `coevolve-image` 明确为整图候选循环; +- 增加论文覆盖、叙事、可分解性评分; +- 记录全图修复经验和跨论文验证结果。 + +门禁:留出论文上的科学质量不退化,且没有明显审美模板坍缩。 + +### P3:PPT 分块合同与全局—分块协同 + +- 在 `planning/rebuild/design.py` 增加 segmentation candidate/plan; +- 在 rebuild workflow 消费该合同; +- 对每个 slot 建立语义节点、端口和可编辑职责绑定; +- 增加装配后全局回评。 + +门禁:分块不丢失科学关系,关键对象可编辑,PPT 预览还原达到阈值。 + +### P4:对象级修复进化 + +- 扩展过程 Judge 输出 `semantic_node_ids`、`slot_ids`、`edge_ids`、`object_ids`; +- 扩展 evolution action:`resegment`、`relayout`、`reroute`、`repair_interfaces`; +- 记录前后分数、blocker 变化和副作用,拒绝无效修复。 + +门禁:局部修复不引入新科学错误;全局问题不会被局部修饰掩盖。 + +### P5:训练 Judge 与 Shadow 接入 + +- 用真实进化轨迹、合成错误、人工筛选与修复结果训练 Judge; +- 先以 Shadow 模式和当前 Bootstrap Judge 并行; +- 达到隐藏集和对抗集门禁后才作为 Online Judge;旧稳定版本继续担任 Frozen Judge。 + +门禁:科学硬错误漏检受控、输出 JSON 稳定、人工审计无系统性偏差;新模型随时可回滚。 + +### P6:PPT Judge 与用户编辑反馈 + +- 在图像 Judge 稳定后,增加 PPT 对象树和渲染预览样本; +- 采集用户在 PPT 中的移动、改字、删箭头、替换资产等编辑事件; +- 将编辑前后状态及 Judge 分数变化转为可学习的修复轨迹。 + +门禁:用户偏好只能影响审美与布局策略,不得降低科学硬门禁。 + +## 9. 接受、回退与跨论文泛化规则 + +一次修复被接受,必须同时满足: + +1. 不产生新的科学 blocker; +2. 修复目标的分项指标提升; +3. 已正确的内容被 preserve; +4. 整体或局部改动的副作用在阈值内; +5. Frozen Judge 和 Grounding Gate 都同意; +6. 对候选通用经验,在留出论文验证后不退化。 + +经验状态: + +```text +candidate +→ current_case_validated +→ cross_paper_validated +→ promoted + +candidate/current_case_validated +→ overfit_or_failed +→ rollback +``` + +经验应保存为带适用条件的规则,而不是某张图的坐标或单次 prompt。例如:“在横向三阶段主流程中,反馈边优先放在主链下方并使用虚线;仅适用于存在明确反馈语义且下方有足够留白的布局”。 + +## 10. 最终完成定义 + +目标链路完成并不等于能跑通一次 demo,而是同时满足: + +- 论文事实、模块和关键关系可追溯并经科学 Gate 守护; +- 整图全局进化能够产出科学正确、清晰、美观且可分解的框架图; +- PPT 阶段能够通过全局—分块协同进化完成可编辑重建; +- 关键文本、箭头、关系、公式和面板保持为 PPT 原生可编辑对象; +- Judge 能定位全局、分块、接口、语义与编译问题并给出可执行修复范围; +- 修复操作能够被接受、回退和跨论文验证; +- 训练出的 Judge 与数据、配置、评测、模型版本及 Git commit 绑定; +- 用户编辑反馈可成为后续进化数据,但不突破科学性约束。 + +最终系统应能证明的不是“它生成了一张漂亮图”,而是:这张图忠实表达了论文,整体叙事清楚,能被可靠重建为可编辑 PPTX,且每次优化都有可追溯证据并能在新论文上验证其泛化性。 diff --git a/docs/workflow.md b/docs/workflow.md index 94d846e..dddc136 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -1,30 +1,40 @@ # Summary -The workflow is designed to keep scientific content, layout, image generation, and PPT editability separated. This prevents the project from becoming a fragile script pile where coordinates, prompts, and rendered assets are mixed together. +The default workflow asks Image2 to generate one complete paper figure, including text, arrows, modules, and visual explanations, then reconstructs the approved image into editable PowerPoint layers. ## Full Workflow -1. Archive the paper and reference image into the output directory. -2. Extract the paper brief: title, method story, named modules, variables, and figure goal. -3. Analyze the reference image and create 25-50 non-arrow image slot targets. -4. Detect arrow/control candidates, write `reference_control_candidates.json`, and render `slot_overlay.png` plus `reference_control_overlay.png` for review or VLM binding. -5. Bind controls to source/target slots with `--control-localizer-mode hybrid` when API access exists, or deterministic heuristic fallback otherwise. -6. Create `arrow_style_profile.json`, `selected_arrow_routes.json`, and `arrow_quality_report.json`; this stage may soften and classify arrows, and may use orthogonal obstacle-aware fallback routing only for missing or explicitly fallback-allowed paths. In `--arrow-style-mode aesthetic`, it may apply curve connectors and halo underlays by default; bundle lane offsets require explicit route opt-in and must stay inside the recorded reference tunnel. It must preserve reference-image flow logic. -7. Create the style sheet before any image prompt is written. -8. Create `layout_plan.json`; use VLM only for normalized coordinate estimation when enabled. -9. Create `figure_program.json`; this is the only layout source for the PPT compiler. -10. Write `reference_slot_prompt_brief.json`; it records what each local reference slot does and what paper concept it carries. -11. Write `slot_prompt_plan.json`; default `--prompt-plan-mode vlm` calls the model for each slot and produces `image_prompt_core`. Use `--prompt-plan-workers` to run these per-slot VLM calls in parallel. -12. Generate multiple image candidates per slot from the model-planned prompt core. -13. Select assets by fill, margin, ratio, and cutoff metrics; never use semantic cropping. -14. Review selected assets with heuristic or VLM review. -15. Compile `editable_composition.pptx` deterministically from the figure program. -16. Export PDF/PNG for review where possible. -17. Run visual critic against the reference image and rendered output. -18. Write alignment review and critic report. -19. Run `rfs validate` before delivery. - -## Recommended Development Runs +1. Archive the paper and optional positive/negative visual references. +2. Extract and validate the evidence-grounded semantic contract. +3. Compile a full layout blueprint and Prompt whose whitelist contains every visible label and whose connector checklist contains every required relation. +4. Ask Image2 to generate the complete raster figure, including visible text and arrows. +5. Review candidates for paper faithfulness, terminology, topology, information density, and visual quality; repair failed regions with Image2. +6. Promote only a passing candidate to `selected_image.png`. +7. Reconstruct the approved complete image with layout analysis, OCR, connector localization, asset extraction, and paper-semantic binding. +8. Compile `editable_composition.pptx`, export review files with a hard timeout, and validate the result. + +## Recommended Product Runs + +Complete default product run: + +```powershell +rfs paper-to-editable --paper "C:\path\paper.pdf" --out .\output\paper_to_editable --editable-route rebuild --image-asset-mode image2 --review-mode vlm --json +``` + +The explicit `--editable-route rebuild` is optional because it is the default. + +Generate and review the complete Image2 figure without rebuilding PPT: + +```powershell +rfs paper-to-image --paper "C:\path\paper.pdf" --out .\output\paper_to_image --asset-mode image2 --candidates 3 --review-mode vlm --json +``` + +## Optional Multi-Asset Experiment + +The commands and failure policy below apply only to `make-framework`. They are +not the primary complete-image product route. + +Offline engineering check: Offline engineering check: @@ -33,16 +43,16 @@ rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.pn rfs validate --out .\output\offline_check --json ``` -Contract-driven engineering check (recommended after a fast/paper-to-image run): +Paper-native contract-driven engineering check (recommended after a fast/paper-to-image run): ```powershell -rfs make-framework --paper "C:\path\paper.pdf" --reference ".\output\paper_to_image\run\selected_image.png" --paper-contract-dir ".\output\paper_to_image\run" --out .\output\contract_check --asset-mode placeholder --asset-workers 3 --locator-mode heuristic --control-localizer-mode off --arrow-style-mode reference --prompt-plan-mode heuristic --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --text-extractor-mode heuristic --ocr-engine off --no-export --json +rfs make-framework --paper "C:\path\paper.pdf" --paper-contract-dir ".\output\paper_to_image\run" --out .\output\contract_check --slot-source paper-native --complexity-profile paper-native-dense --asset-mode placeholder --asset-workers 3 --locator-mode heuristic --control-localizer-mode off --prompt-plan-mode heuristic --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --text-extractor-mode heuristic --ocr-engine off --no-export --json ``` -This route does not call the legacy paper planner. `panel_graphs` instance IDs -remain distinct across panels, local relations are authoritative, exact labels -are editable PPT text, and any extra slots needed to reach 25 are reference -visual splits rather than summary-sentence fillers. +This route does not call the legacy paper planner or generate a whole reference +figure. `panel_graphs` instance IDs remain distinct, relations remain +evidence-backed, exact labels remain editable PPT text, and extra slots are +grounded visual facets of real entities rather than invented modules. Small real image check: @@ -63,6 +73,7 @@ rfs make-framework --paper "C:\path\paper.pdf" --reference "C:\path\reference.pn - `--asset-retries` controls retries per candidate. Retry handles transient 429, 5xx, and timeout failures. - Candidate failures are recorded as rejected candidates. A slot only blocks the run when all of its candidates fail. - The pipeline does not silently switch to placeholder mode when Gemini fails. +- An uncached 25-slot Image2 run is expected to take minutes. A warm run can reuse the prompt plan and validated assets and should normally return in tens of seconds. ## Failure Policy diff --git a/examples/banana_game.md b/examples/banana_game.md index 3ad9844..6006013 100644 --- a/examples/banana_game.md +++ b/examples/banana_game.md @@ -4,32 +4,38 @@ This example runs the BananaGame paper through ResearchFigureStudio using the cu ## Inputs -- Paper: `C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf` -- Reference image: `D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png` +- Paper: `path/to/BananaGame.pdf` +- Reference image: `path/to/reference_architecture.png` If the reference path is unavailable, provide a new user reference image. The workflow should not generate its own full reference image unless explicitly requested. ## Offline Validation Run ```powershell -$out='D:\ResearchFigureStudio\output\banana_full_pipeline_placeholder_36' +$paper='path/to/BananaGame.pdf' +$reference='path/to/reference_architecture.png' +$out='output/banana_full_pipeline_placeholder_36' if (Test-Path $out) { Remove-Item -LiteralPath $out -Recurse -Force } -rfs make-framework --paper 'C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf' --reference 'D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png' --out $out --asset-mode placeholder --locator-mode heuristic --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json +rfs make-framework --paper $paper --reference $reference --out $out --asset-mode placeholder --locator-mode heuristic --prompt-plan-mode heuristic --slot-count 36 --candidates-per-slot 3 --asset-review-mode heuristic --critic-mode heuristic --json rfs validate --out $out --json ``` ## Small Real Image Run ```powershell -$out='D:\ResearchFigureStudio\output\banana_real_small_25x1' -rfs make-framework --paper 'C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf' --reference 'D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png' --out $out --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json +$paper='path/to/BananaGame.pdf' +$reference='path/to/reference_architecture.png' +$out='output/banana_real_small_25x1' +rfs make-framework --paper $paper --reference $reference --out $out --asset-mode gemini --asset-workers 3 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 25 --candidates-per-slot 1 --asset-review-mode heuristic --critic-mode heuristic --json ``` ## Full Quality Run ```powershell -$out='D:\ResearchFigureStudio\output\banana_real_36x3_vlm' -rfs make-framework --paper 'C:\Users\zhang\Downloads\BananaGame_EMNLP26_YichiChuyu (1).pdf' --reference 'D:\LiveFigure\output\test_run_banana_architecture\00_reference_gemini.png' --out $out --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json +$paper='path/to/BananaGame.pdf' +$reference='path/to/reference_architecture.png' +$out='output/banana_real_36x3_vlm' +rfs make-framework --paper $paper --reference $reference --out $out --asset-mode gemini --asset-workers 4 --asset-retries 2 --locator-mode vlm --prompt-plan-mode vlm --prompt-plan-workers 4 --slot-count 36 --candidates-per-slot 3 --asset-review-mode vlm --critic-mode vlm --critic-iterations 1 --json ``` ## Expected Outputs diff --git a/experiments/legacy_reference_rebuilds/README.md b/experiments/legacy_reference_rebuilds/README.md index 0eeed27..7c63f2f 100644 --- a/experiments/legacy_reference_rebuilds/README.md +++ b/experiments/legacy_reference_rebuilds/README.md @@ -1,5 +1,5 @@ # Legacy reference-specific rebuilds -These scripts document the repository's early image-to-PPT experiments. They intentionally remain outside the reusable `rfs` package because they contain sample-specific assumptions and historical local paths. +This directory documents the repository's early image-to-PPT experiments. The old sample-specific rebuild scripts were removed from the public repository because they depended on local reference images and hard-coded machine paths. -Use them only as implementation references for visual details. Do not import them from production workflows, plugin skills, tests, or packaging code. New reusable behavior belongs in `rfs/` with tests and path-independent inputs. +Use this directory only for historical context. Do not import anything from here into production workflows, plugin skills, tests, or packaging code. New reusable behavior belongs in `rfs/` with tests and path-independent inputs. diff --git a/experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py b/experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py deleted file mode 100644 index 9216464..0000000 --- a/experiments/legacy_reference_rebuilds/rebuild_autofigure_architecture_ai_assets.py +++ /dev/null @@ -1,1894 +0,0 @@ -from __future__ import annotations - -import base64 -import argparse -import json -import math -import mimetypes -import os -import shutil -import time -from collections import deque -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path - -import requests -from PIL import Image, ImageDraw, ImageStat -from pptx import Presentation -from pptx.dml.color import RGBColor -from pptx.enum.dml import MSO_LINE_DASH_STYLE -from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE -from pptx.enum.text import MSO_ANCHOR, PP_ALIGN -from pptx.oxml import parse_xml -from pptx.util import Inches, Pt - - -ROOT = Path(__file__).resolve().parents[1] -REFERENCE_CANDIDATES = [ - Path(r"C:\Users\zhang\Documents\xwechat_files\wxid_h824xk1qpfoh22_adc2\temp\RWTemp\2026-06\860847045f7ccc2e7052eb182bab099b\8fc4985d14085a49b20f61b3d56d0bea.png"), - ROOT / "output" / "autofigure_reference_real_v10_layout" / "inputs" / "reference.png", - ROOT / "output" / "autofigure_reference_contract_v2" / "inputs" / "reference.png", -] -REFERENCE = next((path for path in REFERENCE_CANDIDATES if path.exists()), REFERENCE_CANDIDATES[0]) -OUT = ROOT / "output" / "autofigure_architecture_ai_rebuild" -CROP_DIR = OUT / "reference_slot_crops" -ASSET_DIR = OUT / "assets" -CANDIDATE_DIR = OUT / "asset_candidates" - -REF_W = 1770 -REF_H = 975 -SLIDE_W = 15.60 -SLIDE_H = SLIDE_W * REF_H / REF_W - -FONT = "Arial" -TEXT_SIZE_SCALE = 1.08 -INTERNAL_CONTENT_FILL_TARGET = 0.88 -INTERNAL_ACCEPTABLE_FILL_MIN = 0.80 -INTERNAL_ACCEPTABLE_FILL_MAX = 0.95 -INTERNAL_REGENERATE_FILL_THRESHOLD = 0.70 -INTERNAL_MAX_MARGIN_PERCENT = 0.10 -SLOT_FIT_WASTE_ACCEPTABLE_MAX = 0.16 -GENERATION_ASPECT_RATIO_VALUES = { - "1:1": 1.0, - "3:4": 3 / 4, - "4:3": 4 / 3, - "9:16": 9 / 16, - "16:9": 16 / 9, -} -TEXT_STYLES = { - "title": {"font": FONT, "size": 23.0, "bold": True}, - "subtitle": {"font": FONT, "size": 10.8, "bold": False}, - "stage_title": {"font": FONT, "size": 13.0, "bold": True}, - "label": {"font": FONT, "size": 10.5, "bold": True}, - "body": {"font": FONT, "size": 8.6, "bold": False}, - "small": {"font": FONT, "size": 7.4, "bold": False}, - "callout": {"font": FONT, "size": 9.0, "bold": True}, - "legend": {"font": FONT, "size": 9.2, "bold": True}, -} - -SLOTS = [ - { - "id": "input_text_stack", - "bbox": (66, 254, 200, 400), - "asset_bbox": (68, 255, 205, 398), - "prompt": "a stack of off-white paper documents with simple abstract pseudo-lines and two tiny pseudo-number marks, cute academic flat illustration, no readable text", - "fill_guidance": "Make the paper stack very large, nearly full height, with page corners close to all four safe margins. Avoid tiny centered sheets and avoid showing broad blue background around the documents.", - "background_color_hex": "#EAF3FF", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "vlm_agent_robot", - "bbox": (280, 258, 458, 404), - "asset_bbox": (294, 250, 454, 407), - "prompt": "a cute round robot assistant head with small antennae and soft blue-gray casing, pastel academic diagram style, no text", - "background_color_hex": "#EAF3FF", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "ai_designer", - "bbox": (642, 441, 793, 629), - "asset_bbox": (650, 425, 790, 627), - "prompt": "a cute AI designer character wearing a beret and holding a blueprint tablet, pastel orange academic illustration, no readable text", - "background_color_hex": "#FFF1DF", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "ai_critic", - "bbox": (1059, 438, 1212, 632), - "asset_bbox": (1062, 434, 1208, 632), - "prompt": "a cute robot critic wearing round glasses and holding a clipboard, pastel academic illustration, no readable text", - "fill_guidance": "Use a large close-up robot critic with head, torso, glasses, and clipboard filling the portrait canvas. The robot should nearly touch the side safe margins and bottom safe margin.", - "background_color_hex": "#FFF1DF", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "synthesis_tools", - "bbox": (1375, 132, 1625, 247), - "asset_bbox": (1375, 126, 1628, 250), - "prompt": "a magic wand and cheerful painter palette with soft sparkles, pastel scientific illustration, no text", - "background_color_hex": "#ECF7E9", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "erase_text_tool", - "bbox": (1288, 493, 1420, 604), - "asset_bbox": (1290, 486, 1425, 606), - "prompt": "one large front-facing rounded pink eraser icon, cute academic diagram illustration, no readable text, no letters, no numbers, no pseudo-letters", - "fill_guidance": "Make one oversized front-facing rounded eraser dominate the square canvas, with its body occupying roughly 85-90% of the canvas width and height. Use a broad blocky eraser shape rather than a thin diagonal object. Keep the eraser close to all safe margins without cutting it off. Do not add ABC, letters, numbers, words, pseudo-text, distant dust, particles, side objects, or broad empty background.", - "background_color_hex": "#ECF7E9", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "ocr_verify", - "bbox": (1570, 492, 1712, 604), - "asset_bbox": (1566, 490, 1718, 608), - "prompt": "a magnifying glass inspecting pseudo-numbers with a green check badge and short gray pseudo-lines, clean pastel academic illustration, no readable text", - "fill_guidance": "Make the magnifying glass lens very large, occupying most of the canvas height, with the handle and check badge tightly grouped into one compact foreground silhouette. Avoid broad side margins and keep decorative side lines minimal.", - "background_color_hex": "#ECF7E9", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, - { - "id": "final_autofigure_card", - "bbox": (1402, 682, 1590, 812), - "asset_bbox": (1408, 686, 1588, 808), - "prompt": "a small polished scientific figure card with abstract pie chart, smooth shapes, tiny pseudo-lines and pleasing pastel colors, no readable text", - "fill_guidance": "Make the figure card itself large and nearly full canvas, with its rounded rectangle edges close to the safe margins. Internal charts should be big enough to count as foreground, not tiny decorations.", - "background_color_hex": "#ECF7E9", - "internal_content_fill_target": INTERNAL_CONTENT_FILL_TARGET, - "internal_max_margin_percent": INTERNAL_MAX_MARGIN_PERCENT, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - }, -] - -CONTROL_SPECS = [ - {"id": "input_to_vlm", "kind": "line", "source_id": "input_text_stack", "target_id": "vlm_agent_robot", "points": [(222, 346), (278, 346)], "color": "#607998", "width_pt": 2.5, "arrow": True, "dash": False, "confidence": 0.92}, - {"id": "vlm_to_blueprint", "kind": "line", "source_id": "vlm_agent_robot", "target_id": "initial_blueprint", "points": [(365, 407), (365, 508)], "color": "#607998", "width_pt": 2.8, "arrow": True, "dash": False, "confidence": 0.91}, - {"id": "stage_i_to_stage_ii", "kind": "line", "source_id": "stage_i", "target_id": "stage_ii", "points": [(565, 493), (626, 493)], "color": "#555555", "width_pt": 4.0, "arrow": True, "dash": False, "confidence": 0.95}, - {"id": "stage_ii_to_stage_iii", "kind": "line", "source_id": "stage_ii", "target_id": "stage_iii", "points": [(1228, 493), (1270, 493)], "color": "#555555", "width_pt": 4.0, "arrow": True, "dash": False, "confidence": 0.95}, - {"id": "critique_refinement_loop", "kind": "oval", "source_id": "ai_critic", "target_id": "ai_designer", "bbox": (735, 335, 350, 382), "color": "#D99045", "width_pt": 2.0, "arrow": False, "dash": True, "flow": "clockwise", "confidence": 0.86}, - {"id": "designer_refine_arrow", "kind": "line", "source_id": "critique_refinement_loop", "target_id": "ai_designer", "points": [(814, 685), (774, 648)], "color": "#D99045", "width_pt": 2.0, "arrow": True, "dash": False, "confidence": 0.84}, - {"id": "critic_feedback_arrow", "kind": "line", "source_id": "critique_refinement_loop", "target_id": "ai_critic", "points": [(1038, 388), (1072, 430)], "color": "#D99045", "width_pt": 2.0, "arrow": True, "dash": False, "confidence": 0.84}, - {"id": "synthesis_to_raw", "kind": "line", "source_id": "synthesis_tools", "target_id": "raw_image", "points": [(1500, 280), (1500, 350)], "color": "#6F9A66", "width_pt": 3.0, "arrow": True, "dash": False, "confidence": 0.9}, - {"id": "raw_to_erase_route", "kind": "polyline", "source_id": "raw_image", "target_id": "erase_text_tool", "points": [(1428, 405), (1362, 405), (1348, 428), (1345, 462)], "color": "#6F9A66", "width_pt": 1.8, "arrow": True, "dash": True, "confidence": 0.9}, - {"id": "raw_to_ocr_route", "kind": "polyline", "source_id": "raw_image", "target_id": "ocr_verify", "points": [(1588, 405), (1652, 405), (1684, 426), (1692, 462)], "color": "#6F9A66", "width_pt": 1.8, "arrow": True, "dash": True, "confidence": 0.9}, - {"id": "erase_to_final_route", "kind": "polyline", "source_id": "erase_text_tool", "target_id": "final_autofigure_card", "points": [(1345, 624), (1350, 650), (1382, 680)], "color": "#6F9A66", "width_pt": 1.8, "arrow": True, "dash": True, "confidence": 0.88}, - {"id": "ocr_to_final_route", "kind": "polyline", "source_id": "ocr_verify", "target_id": "final_autofigure_card", "points": [(1644, 624), (1638, 650), (1608, 680)], "color": "#6F9A66", "width_pt": 1.8, "arrow": True, "dash": True, "confidence": 0.88}, - {"id": "finalization_arrow", "kind": "line", "source_id": "erase_text_tool", "target_id": "final_autofigure_card", "points": [(1306, 737), (1386, 737)], "color": "#6F9A66", "width_pt": 3.5, "arrow": True, "dash": False, "confidence": 0.86}, -] - - -def px(v: float, total: float, slide_total: float) -> float: - return v / total * slide_total - - -def x(v: float) -> float: - return px(v, REF_W, SLIDE_W) - - -def y(v: float) -> float: - return px(v, REF_H, SLIDE_H) - - -def w(v: float) -> float: - return px(v, REF_W, SLIDE_W) - - -def h(v: float) -> float: - return px(v, REF_H, SLIDE_H) - - -def rgb(value: str) -> RGBColor: - text = value.strip().lstrip("#") - return RGBColor(int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16)) - - -def hex_to_tuple(value: str) -> tuple[int, int, int]: - text = value.strip().lstrip("#") - return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) - - -def tuple_to_hex(value: tuple[int, int, int]) -> str: - return f"#{value[0]:02X}{value[1]:02X}{value[2]:02X}" - - -def set_fill(shape, color: str, transparency: int = 0) -> None: - shape.fill.solid() - shape.fill.fore_color.rgb = rgb(color) - shape.fill.transparency = transparency - - -def no_fill(shape) -> None: - shape.fill.background() - - -def set_line(shape, color: str, width_pt: float = 1.2, dash: bool = False) -> None: - shape.line.color.rgb = rgb(color) - shape.line.width = Pt(width_pt) - if dash: - shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH - - -def add_text(slide, text: str, left: float, top: float, width: float, height: float, *, style: str = "body", color: str = "#333333", align=PP_ALIGN.CENTER, size: float | None = None, bold: bool | None = None, valign=MSO_ANCHOR.MIDDLE): - spec = TEXT_STYLES[style] - font_size = float(size if size is not None else spec["size"]) * TEXT_SIZE_SCALE - box = slide.shapes.add_textbox(Inches(left), Inches(top), Inches(width), Inches(height)) - tf = box.text_frame - tf.clear() - tf.word_wrap = True - tf.vertical_anchor = valign - tf.margin_left = Inches(0.01) - tf.margin_right = Inches(0.01) - tf.margin_top = Inches(0.0) - tf.margin_bottom = Inches(0.0) - for idx, line in enumerate(text.split("\n")): - p = tf.paragraphs[0] if idx == 0 else tf.add_paragraph() - p.alignment = align - r = p.add_run() - r.text = line - r.font.name = str(spec["font"]) - r.font.size = Pt(font_size) - r.font.bold = bool(bold if bold is not None else spec["bold"]) - r.font.color.rgb = rgb(color) - return box - - -def add_round(slide, left_px, top_px, width_px, height_px, fill, stroke, width_pt=1.2, radius=True): - shape = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE if radius else MSO_SHAPE.RECTANGLE, Inches(x(left_px)), Inches(y(top_px)), Inches(w(width_px)), Inches(h(height_px))) - set_fill(shape, fill) - set_line(shape, stroke, width_pt) - shape.shadow.inherit = False - return shape - - -def add_arrowhead(connector, size="med") -> None: - ln = connector.line._get_or_add_ln() - ln.append(parse_xml(f'')) - - -def add_line(slide, x1, y1, x2, y2, color="#333333", width_pt=1.2, arrow=False, dash=False): - conn = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(x(x1)), Inches(y(y1)), Inches(x(x2)), Inches(y(y2))) - set_line(conn, color, width_pt, dash=dash) - if arrow: - add_arrowhead(conn) - return conn - - -def add_poly(slide, points, color="#333333", width_pt=1.2, arrow=False, dash=False): - for idx, (a, b) in enumerate(zip(points[:-1], points[1:])): - add_line(slide, a[0], a[1], b[0], b[1], color=color, width_pt=width_pt, arrow=arrow and idx == len(points) - 2, dash=dash) - - -def bbox_percent_from_points(points: list[tuple[float, float]]) -> dict: - xs = [point[0] for point in points] - ys = [point[1] for point in points] - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) - return {"x": round(left / REF_W, 4), "y": round(top / REF_H, 4), "w": round((right - left) / REF_W, 4), "h": round((bottom - top) / REF_H, 4)} - - -def direction_label(dx: float, dy: float) -> str: - if abs(dx) < 1e-6 and abs(dy) < 1e-6: - return "none" - if abs(dx) >= abs(dy) * 1.8: - return "right" if dx > 0 else "left" - if abs(dy) >= abs(dx) * 1.8: - return "down" if dy > 0 else "up" - vertical = "down" if dy > 0 else "up" - horizontal = "right" if dx > 0 else "left" - return f"{vertical}_{horizontal}" - - -def anchor_from_direction(dx: float, dy: float, at_source: bool) -> str: - if abs(dx) >= abs(dy): - if dx > 0: - return "right_center" if at_source else "left_center" - return "left_center" if at_source else "right_center" - if dy > 0: - return "bottom_center" if at_source else "top_center" - return "top_center" if at_source else "bottom_center" - - -def position_direction_metadata(control: dict) -> dict: - if control["kind"] == "oval": - left, top, width, height = control["bbox"] - return { - "position": { - "bbox_px": [left, top, width, height], - "center_px": [round(left + width / 2, 2), round(top + height / 2, 2)], - "bbox_percent": {"x": round(left / REF_W, 4), "y": round(top / REF_H, 4), "w": round(width / REF_W, 4), "h": round(height / REF_H, 4)}, - }, - "direction": { - "path_type": "closed_loop", - "flow": control.get("flow", "clockwise"), - "arrowhead_at": control.get("arrowhead_at", "none"), - "direction_confidence": control.get("confidence", 0.0), - }, - "source_anchor": control.get("source_anchor", "loop_right_side"), - "target_anchor": control.get("target_anchor", "loop_left_side"), - } - - points = control["points"] - start = points[0] - end = points[-1] - dx = end[0] - start[0] - dy = end[1] - start[1] - length = max((dx * dx + dy * dy) ** 0.5, 1e-6) - angle = math.degrees(math.atan2(dy, dx)) - return { - "position": { - "start_px": [start[0], start[1]], - "end_px": [end[0], end[1]], - "points_px": [[px_, py_] for px_, py_ in points], - "center_px": [round((start[0] + end[0]) / 2, 2), round((start[1] + end[1]) / 2, 2)], - "bbox_percent": bbox_percent_from_points(points), - }, - "direction": { - "vector_px": [round(dx, 2), round(dy, 2)], - "unit_vector": [round(dx / length, 4), round(dy / length, 4)], - "angle_deg": round(angle, 2), - "label": direction_label(dx, dy), - "arrowhead_at": "end" if control.get("arrow") else "none", - "direction_confidence": control.get("confidence", 0.0), - }, - "source_anchor": control.get("source_anchor", anchor_from_direction(dx, dy, True)), - "target_anchor": control.get("target_anchor", anchor_from_direction(dx, dy, False)), - } - - -def normalize_control(control: dict) -> dict: - item = dict(control) - item.update(position_direction_metadata(control)) - if control["kind"] == "oval": - left, top, width, height = control["bbox"] - item["bbox_percent"] = {"x": round(left / REF_W, 4), "y": round(top / REF_H, 4), "w": round(width / REF_W, 4), "h": round(height / REF_H, 4)} - item["path_percent"] = [] - else: - item["bbox_percent"] = bbox_percent_from_points(control["points"]) - item["path_percent"] = [{"x": round(px_ / REF_W, 4), "y": round(py_ / REF_H, 4)} for px_, py_ in control["points"]] - item["render_policy"] = "ppt_shape_not_image_asset" - item["route_policy"] = "reference_locked" - return item - - -def render_control(slide, control: dict): - if control["kind"] == "oval": - left, top, width, height = control["bbox"] - shape = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left)), Inches(y(top)), Inches(w(width)), Inches(h(height))) - no_fill(shape) - set_line(shape, control["color"], control["width_pt"], dash=control["dash"]) - return shape - if control["kind"] == "polyline": - return add_poly(slide, control["points"], control["color"], control["width_pt"], arrow=control["arrow"], dash=control["dash"]) - start, end = control["points"] - return add_line(slide, start[0], start[1], end[0], end[1], control["color"], control["width_pt"], arrow=control["arrow"], dash=control["dash"]) - - -def render_controls(slide, control_ids: list[str]) -> None: - lookup = {control["id"]: control for control in CONTROL_SPECS} - for control_id in control_ids: - render_control(slide, lookup[control_id]) - - -def draw_dashed_line(draw: ImageDraw.ImageDraw, points: list[tuple[float, float]], color: str, width: int = 3, dash: int = 14, gap: int = 9) -> None: - fill = hex_to_tuple(color) - for start, end in zip(points[:-1], points[1:]): - x1, y1 = start - x2, y2 = end - length = max(((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5, 1) - steps = int(length // (dash + gap)) + 1 - for step in range(steps): - a = step * (dash + gap) / length - b = min((step * (dash + gap) + dash) / length, 1) - if a >= 1: - continue - draw.line((x1 + (x2 - x1) * a, y1 + (y2 - y1) * a, x1 + (x2 - x1) * b, y1 + (y2 - y1) * b), fill=fill, width=width) - - -def write_reference_overlays() -> None: - with Image.open(REFERENCE).convert("RGB") as image: - slot_overlay = image.copy() - slot_draw = ImageDraw.Draw(slot_overlay) - for slot in SLOTS: - slot_draw.rectangle(slot["bbox"], outline=hex_to_tuple("#2A7BD1"), width=4) - slot_draw.text((slot["bbox"][0] + 4, slot["bbox"][1] + 4), slot["id"], fill=hex_to_tuple("#2A7BD1")) - slot_overlay.save(OUT / "slot_overlay.png") - - control_overlay = image.copy() - control_draw = ImageDraw.Draw(control_overlay) - for control in CONTROL_SPECS: - color = control["color"] - if control["kind"] == "oval": - left, top, width, height = control["bbox"] - control_draw.ellipse((left, top, left + width, top + height), outline=hex_to_tuple(color), width=5) - label_xy = (left, top) - else: - points = control["points"] - if control["dash"]: - draw_dashed_line(control_draw, points, color, width=5) - else: - control_draw.line(points, fill=hex_to_tuple(color), width=5) - label_xy = points[0] - control_draw.text((label_xy[0] + 5, label_xy[1] + 5), control["id"], fill=hex_to_tuple("#111111")) - control_overlay.save(OUT / "reference_control_overlay.png") - - -def crop_reference_slots() -> list[dict]: - CROP_DIR.mkdir(parents=True, exist_ok=True) - with Image.open(REFERENCE) as image: - image = image.convert("RGB") - for slot in SLOTS: - crop = image.crop(slot["bbox"]) - crop_path = CROP_DIR / f"{slot['id']}.png" - crop.save(crop_path) - slot["crop_path"] = str(crop_path) - x0, y0, x1, y1 = slot["bbox"] - slot["bbox_percent"] = {"x": round(x0 / REF_W, 4), "y": round(y0 / REF_H, 4), "w": round((x1 - x0) / REF_W, 4), "h": round((y1 - y0) / REF_H, 4)} - return SLOTS - - -def estimate_edge_background(image: Image.Image) -> tuple[int, int, int]: - rgb_image = image.convert("RGB") - width, height = rgb_image.size - strip = max(4, min(width, height) // 20) - samples = [] - for crop_box in [ - (0, 0, width, strip), - (0, height - strip, width, height), - (0, 0, strip, height), - (width - strip, 0, width, height), - ]: - crop = rgb_image.crop(crop_box) - if hasattr(crop, "get_flattened_data"): - samples.extend(crop.get_flattened_data()) - else: - samples.extend(crop.getdata()) - samples.sort(key=lambda color: sum(color)) - mid = samples[len(samples) // 2] - return int(mid[0]), int(mid[1]), int(mid[2]) - - -def harmonize_asset_background(asset_path: Path, background_hex: str) -> dict: - image = Image.open(asset_path).convert("RGB") - target = hex_to_tuple(background_hex) - source_bg = estimate_edge_background(image) - mask = edge_connected_background_mask(image, source_bg, tolerance=42) - output = image.copy() - bg_layer = Image.new("RGB", image.size, target) - output.paste(bg_layer, mask=mask) - output.save(asset_path) - stat = ImageStat.Stat(mask) - replaced_percent = round(stat.mean[0] / 255 * 100, 2) - return { - "asset": str(asset_path), - "estimated_source_background": tuple_to_hex(source_bg), - "target_background": background_hex, - "replaced_percent": replaced_percent, - } - - -def measure_internal_content_fill(image_path: Path, background_hex: str) -> dict: - image = Image.open(image_path).convert("RGB") - width, height = image.size - target_bg = hex_to_tuple(background_hex) - edge_bg = estimate_edge_background(image) - pixels = image.load() - tolerance = 54 - xs: list[int] = [] - ys: list[int] = [] - for py_ in range(height): - for px_ in range(width): - color = pixels[px_, py_] - target_dist = abs(color[0] - target_bg[0]) + abs(color[1] - target_bg[1]) + abs(color[2] - target_bg[2]) - edge_dist = abs(color[0] - edge_bg[0]) + abs(color[1] - edge_bg[1]) + abs(color[2] - edge_bg[2]) - if min(target_dist, edge_dist) > tolerance: - xs.append(px_) - ys.append(py_) - if not xs: - return { - "candidate_path": str(image_path), - "foreground_bbox_px": None, - "foreground_bbox_fill_percent": 0.0, - "margin_left_percent": 1.0, - "margin_right_percent": 1.0, - "margin_top_percent": 1.0, - "margin_bottom_percent": 1.0, - "estimated_edge_background": tuple_to_hex(edge_bg), - "crop_applied": False, - "warning": "no_foreground_detected", - } - left, right = min(xs), max(xs) - top, bottom = min(ys), max(ys) - bbox_width = right - left + 1 - bbox_height = bottom - top + 1 - margins = { - "margin_left_percent": round(left / width, 4), - "margin_right_percent": round((width - 1 - right) / width, 4), - "margin_top_percent": round(top / height, 4), - "margin_bottom_percent": round((height - 1 - bottom) / height, 4), - } - return { - "candidate_path": str(image_path), - "foreground_bbox_px": [left, top, right, bottom], - "foreground_bbox_fill_percent": round((bbox_width * bbox_height) / (width * height), 4), - **margins, - "max_margin_percent": max(margins.values()), - "estimated_edge_background": tuple_to_hex(edge_bg), - "crop_applied": False, - } - - -def score_internal_fill(metrics: dict) -> float: - fill = float(metrics.get("foreground_bbox_fill_percent") or 0.0) - max_margin = float(metrics.get("max_margin_percent") or 1.0) - cutoff_penalty = 0.0 - for key in ["margin_left_percent", "margin_right_percent", "margin_top_percent", "margin_bottom_percent"]: - value = float(metrics.get(key) or 0.0) - if value < 0.015: - cutoff_penalty += 0.08 - if INTERNAL_ACCEPTABLE_FILL_MIN <= fill <= INTERNAL_ACCEPTABLE_FILL_MAX: - fill_score = 1.0 - abs(fill - INTERNAL_CONTENT_FILL_TARGET) - elif fill < INTERNAL_ACCEPTABLE_FILL_MIN: - fill_score = fill / INTERNAL_ACCEPTABLE_FILL_MIN - else: - fill_score = max(0.0, 1.0 - (fill - INTERNAL_ACCEPTABLE_FILL_MAX) * 3.0) - margin_score = max(0.0, 1.0 - max(0.0, max_margin - INTERNAL_MAX_MARGIN_PERCENT) * 2.0) - return round(fill_score * 0.78 + margin_score * 0.22 - cutoff_penalty, 4) - - -def evaluate_candidate(slot: dict, candidate_path: Path) -> dict: - metrics = measure_internal_content_fill(candidate_path, slot["background_color_hex"]) - metrics.update(compute_slot_fit_metrics(candidate_path, slot)) - fill = float(metrics.get("foreground_bbox_fill_percent") or 0.0) - slot_fit_waste = float(metrics.get("slot_fit_waste_percent") or 1.0) - edge_touch_count = sum( - 1 for key in ["margin_left_percent", "margin_right_percent", "margin_top_percent", "margin_bottom_percent"] - if float(metrics.get(key) or 0.0) < 0.01 - ) - metrics["edge_touch_count"] = edge_touch_count - metrics["background_or_edge_artifact_risk"] = fill > 0.97 and edge_touch_count >= 2 - internal_score = score_internal_fill(metrics) - slot_fit_score = max(0.0, 1.0 - slot_fit_waste) - metrics["score"] = round(internal_score * 0.72 + slot_fit_score * 0.28, 4) - metrics["internal_fill_score"] = internal_score - metrics["slot_fit_score"] = round(slot_fit_score, 4) - metrics["pass"] = ( - INTERNAL_ACCEPTABLE_FILL_MIN <= fill <= INTERNAL_ACCEPTABLE_FILL_MAX - and slot_fit_waste <= SLOT_FIT_WASTE_ACCEPTABLE_MAX - and not metrics["background_or_edge_artifact_risk"] - ) - metrics["needs_regeneration"] = ( - fill < INTERNAL_REGENERATE_FILL_THRESHOLD - or fill > INTERNAL_ACCEPTABLE_FILL_MAX - or slot_fit_waste > SLOT_FIT_WASTE_ACCEPTABLE_MAX - or metrics["background_or_edge_artifact_risk"] - ) - return metrics - - -def select_best_candidate(candidate_metrics: list[dict]) -> dict | None: - if not candidate_metrics: - return None - slot_fit_candidates = [ - item for item in candidate_metrics - if float(item.get("slot_fit_waste_percent") or 1.0) <= SLOT_FIT_WASTE_ACCEPTABLE_MAX - ] - if slot_fit_candidates: - artifact_free_candidates = [ - item for item in slot_fit_candidates - if not item.get("background_or_edge_artifact_risk") - ] - candidates = artifact_free_candidates or slot_fit_candidates - def slot_fit_key(item: dict) -> tuple[float, float, float, float]: - fill = float(item.get("foreground_bbox_fill_percent") or 0.0) - edge_touch_count = int(item.get("edge_touch_count") or 0) - overfill_penalty = max(0.0, fill - INTERNAL_ACCEPTABLE_FILL_MAX) * 4.0 - fill_priority = min(fill, INTERNAL_ACCEPTABLE_FILL_MAX) - overfill_penalty - return ( - 1.0 if INTERNAL_ACCEPTABLE_FILL_MIN <= fill <= INTERNAL_ACCEPTABLE_FILL_MAX else 0.0, - fill_priority, - -float(edge_touch_count), - float(item.get("score") or 0.0), - ) - return max(candidates, key=slot_fit_key) - return max(candidate_metrics, key=lambda item: item["score"]) - - -ECONOMY_ACCEPTANCE_PROFILES = { - "character": {"min_fill": 0.80, "max_fill": 0.95}, - "document_stack": {"min_fill": 0.75, "max_fill": 0.95}, - "tool_combo": {"min_fill": 0.75, "max_fill": 0.95}, - "chart_card": {"min_fill": 0.75, "max_fill": 0.95}, - "inspection": {"min_fill": 0.70, "max_fill": 0.95}, - "thin_tool": {"min_fill": 0.50, "max_fill": 0.95}, -} - - -SLOT_ECONOMY_TYPES = { - "input_text_stack": "document_stack", - "vlm_agent_robot": "character", - "ai_designer": "character", - "ai_critic": "character", - "synthesis_tools": "tool_combo", - "erase_text_tool": "thin_tool", - "ocr_verify": "inspection", - "final_autofigure_card": "chart_card", -} - - -ASSET_TYPE_PROMPT_TEMPLATES = { - "character": { - "subject_framing": "large bust or upper-body character framing", - "composition": "The character must be zoomed in so the head and upper body dominate the canvas, with the foreground bbox spanning at least 80% of both canvas width and canvas height. Do not show a small full-body figure floating in the center.", - "allowed_auxiliary_objects": "Only one small carried prop is allowed when it identifies the role.", - }, - "document_stack": { - "subject_framing": "large stacked-document framing", - "composition": "The document stack must be oversized and zoomed in, with page corners close to all four safe margins and the stack spanning at least 80% of canvas width and height.", - "allowed_auxiliary_objects": "No auxiliary objects.", - }, - "tool_combo": { - "subject_framing": "compact grouped tool framing", - "composition": "Make the wand and palette large, overlapping, and tightly grouped into one foreground cluster that spans most of the landscape canvas. Do not leave the tools as tiny separated objects.", - "allowed_auxiliary_objects": "Only tiny sparkles attached to the main cluster are allowed.", - }, - "chart_card": { - "subject_framing": "large card-object framing", - "composition": "The chart card rectangle itself should be large, with card edges close to the safe margins and internal chart marks large enough to count as foreground.", - "allowed_auxiliary_objects": "No objects outside the chart card.", - }, - "inspection": { - "subject_framing": "large compact inspection-symbol framing", - "composition": "The magnifying lens should be oversized and the handle and verification badge should be tightly grouped into one compact foreground silhouette that fills the 4:3 canvas.", - "allowed_auxiliary_objects": "Only short side lines close to the lens are allowed.", - }, - "thin_tool": { - "subject_framing": "oversized single-tool framing", - "composition": "Use a broad front-facing version of the tool when possible so it fills the canvas visually without relying on scattered particles.", - "allowed_auxiliary_objects": "No auxiliary objects.", - }, -} - - -def build_asset_generation_spec(slot: dict, candidate_index: int = 1) -> dict: - slot_type = SLOT_ECONOMY_TYPES.get(slot["id"], "character") - profile = ECONOMY_ACCEPTANCE_PROFILES[slot_type] - template = ASSET_TYPE_PROMPT_TEMPLATES[slot_type] - aspect_ratio = choose_generation_aspect_ratio(slot) - return { - "slot_id": slot["id"], - "object_type": slot_type, - "target_aspect_ratio": aspect_ratio, - "slot_aspect_ratio": slot_aspect_ratio(slot), - "target_fill_range": [profile["min_fill"], profile["max_fill"]], - "target_fill_percent_text": f"{int(profile['min_fill'] * 100)}-{int(profile['max_fill'] * 100)}%", - "max_margin_percent": 0.12, - "background_color_hex": slot["background_color_hex"], - "subject_framing": template["subject_framing"], - "composition": template["composition"], - "allowed_auxiliary_objects": template["allowed_auxiliary_objects"], - "main_request": slot["prompt"], - "slot_guidance": slot.get("fill_guidance", ""), - "forbidden": [ - "readable text", - "letters", - "numbers", - "labels", - "watermark", - "large empty background", - "wide margins", - "distant detached decorations", - "black or transparent corner artifacts", - ], - "candidate_index": candidate_index, - } - - -def prompt_from_asset_generation_spec(spec: dict) -> str: - forbidden = ", ".join(spec["forbidden"]) - return ( - "Use the provided crop only as a visual reference for object identity and style, not for its scale, framing, empty margins, or amount of background. " - "Before generating, internally plan the icon layout from this generation specification and satisfy every framing constraint. " - f"Asset id: {spec['slot_id']}. Object type: {spec['object_type']}. Main request: {spec['main_request']}. " - f"Canvas aspect ratio: {spec['target_aspect_ratio']}. Target effective foreground fill: {spec['target_fill_percent_text']}. " - f"Maximum intended uniform background margin: {int(spec['max_margin_percent'] * 100)}%. " - f"Subject framing: {spec['subject_framing']}. Composition: {spec['composition']} " - f"Auxiliary object rule: {spec['allowed_auxiliary_objects']} " - f"{spec['slot_guidance']} " - "Preserve the crop's object identity, approximate pose, pastel academic illustration style, soft shadows, rounded friendly shapes, and color family. " - "Create a standalone PowerPoint scientific architecture icon where the main subject fills the image canvas according to the target fill range. " - "The background should be only a narrow border around the subject, not a large field. Prefer a close-up crop-like composition over a full-scene composition. " - "If the first draft would contain broad empty background, internally zoom in before finalizing the image. " - "Keep all important details visible inside the canvas and avoid cutting off the subject. " - f"Use a perfectly flat background color matching {spec['background_color_hex']} so the asset blends into its stage panel. " - f"Do not include: {forbidden}. No extra unrelated objects." - ) - - -def economy_acceptance_decision(slot: dict, metrics: dict) -> dict: - slot_type = SLOT_ECONOMY_TYPES.get(slot["id"], "character") - profile = ECONOMY_ACCEPTANCE_PROFILES[slot_type] - fill = float(metrics.get("foreground_bbox_fill_percent") or 0.0) - slot_fit_waste = float(metrics.get("slot_fit_waste_percent") or 1.0) - artifact_risk = bool(metrics.get("background_or_edge_artifact_risk")) - accepted = ( - profile["min_fill"] <= fill <= profile["max_fill"] - and slot_fit_waste <= SLOT_FIT_WASTE_ACCEPTABLE_MAX - and not artifact_risk - ) - if accepted: - reason = f"{slot_type}_threshold_met" - elif artifact_risk: - reason = "reject_background_or_edge_artifact" - elif slot_fit_waste > SLOT_FIT_WASTE_ACCEPTABLE_MAX: - reason = "reject_slot_fit_waste" - elif fill < profile["min_fill"]: - reason = f"below_{slot_type}_min_fill" - else: - reason = f"above_{slot_type}_max_fill" - return { - "slot_type": slot_type, - "min_fill": profile["min_fill"], - "max_fill": profile["max_fill"], - "accepted": accepted, - "reason": reason, - } - - -def run_asset_cost_experiment(retries: int, output_path: Path) -> dict: - crop_reference_slots() - existing_report_items: dict[str, dict] = {} - generation_report_path = OUT / "asset_generation_report.json" - if generation_report_path.exists(): - try: - generation_report = json.loads(generation_report_path.read_text(encoding="utf-8")) - existing_report_items = {item["slot_id"]: item for item in generation_report.get("assets", [])} - except Exception: - existing_report_items = {} - items = [] - strict_max_requests = 0 - economy_max_requests = 0 - for slot in SLOTS: - report_item = existing_report_items.get(slot["id"]) - if report_item and report_item.get("candidate_metrics"): - candidate_metrics = report_item["candidate_metrics"] - else: - candidate_metrics = [evaluate_candidate(slot, path) for path in candidate_paths_for_slot(slot["id"])] - best = report_item.get("selected_internal_fill") if report_item and report_item.get("selected_internal_fill") else select_best_candidate(candidate_metrics) - if best is None: - strict_would_generate = True - economy_would_generate = True - economy_decision = {"accepted": False, "reason": "no_existing_candidate", "slot_type": SLOT_ECONOMY_TYPES.get(slot["id"], "character")} - selected = {} - else: - strict_would_generate = not best["pass"] or best["needs_regeneration"] - economy_decision = economy_acceptance_decision(slot, best) - economy_would_generate = not economy_decision["accepted"] - selected = { - "candidate_path": best.get("candidate_path"), - "foreground_bbox_fill_percent": best.get("foreground_bbox_fill_percent"), - "slot_fit_waste_percent": best.get("slot_fit_waste_percent"), - "generation_aspect_ratio": best.get("generation_aspect_ratio"), - "generated_image_size": best.get("generated_image_size"), - "background_or_edge_artifact_risk": best.get("background_or_edge_artifact_risk"), - "strict_pass": best.get("pass"), - "strict_needs_regeneration": best.get("needs_regeneration"), - } - if strict_would_generate: - strict_max_requests += retries - if economy_would_generate: - economy_max_requests += 1 - items.append({ - "slot_id": slot["id"], - "candidate_count": len(candidate_metrics), - "selected": selected, - "current_strict_policy": { - "would_generate_next_run": strict_would_generate, - "max_api_requests_next_run": retries if strict_would_generate else 0, - }, - "economy_policy": { - **economy_decision, - "would_generate_next_run": economy_would_generate, - "max_api_requests_next_run": 1 if economy_would_generate else 0, - }, - }) - report = { - "summary": "Dry-run cost experiment. No image API calls are made; this compares next-run generation decisions using existing candidates.", - "baseline_policy": { - "name": "current_strict_80_95_policy", - "max_candidates_per_failed_slot": retries, - }, - "economy_policy": { - "name": "type_aware_acceptance_with_one_candidate_retry", - "profiles": ECONOMY_ACCEPTANCE_PROFILES, - "max_candidates_per_failed_slot": 1, - "locks_existing_accepted_assets": True, - }, - "estimated_next_run_api_requests": { - "current_strict_policy_max": strict_max_requests, - "economy_policy_max": economy_max_requests, - "estimated_requests_saved": strict_max_requests - economy_max_requests, - "estimated_saving_percent": round((strict_max_requests - economy_max_requests) / max(strict_max_requests, 1) * 100, 2), - }, - "items": items, - } - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - return report - - -def parse_slot_ids(slot_ids: str) -> list[str]: - if slot_ids.strip().lower() == "all": - return [slot["id"] for slot in SLOTS] - return [item.strip() for item in slot_ids.split(",") if item.strip()] - - -def create_preflight_contact_sheet(items: list[dict], output_path: Path) -> None: - thumb = 180 - label_h = 64 - gap = 16 - columns = 4 - rows = max(1, math.ceil(len(items) / columns)) - sheet = Image.new("RGB", (columns * (thumb + gap) + gap, rows * (thumb + label_h + gap) + gap), "#F3F1EC") - draw = ImageDraw.Draw(sheet) - for idx, item in enumerate(items): - col = idx % columns - row = idx // columns - x0 = gap + col * (thumb + gap) - y0 = gap + row * (thumb + label_h + gap) - if item.get("candidate_path") and Path(item["candidate_path"]).exists(): - image = Image.open(item["candidate_path"]).convert("RGB") - image.thumbnail((thumb, thumb), Image.Resampling.LANCZOS) - sheet.paste(image, (x0 + (thumb - image.width) // 2, y0 + (thumb - image.height) // 2)) - fill = item.get("metrics", {}).get("foreground_bbox_fill_percent") - accepted = item.get("economy_decision", {}).get("accepted") - label = f"{item['slot_id']}\nfill: {fill * 100:.1f}%" if isinstance(fill, (int, float)) else item["slot_id"] - label += f"\n{'accepted' if accepted else 'review'}" - draw.multiline_text((x0, y0 + thumb + 6), label, fill="#333333", spacing=3) - output_path.parent.mkdir(parents=True, exist_ok=True) - sheet.save(output_path) - - -def run_preflight_first_pass_experiment(slot_ids: str, output_dir: Path, workers: int = 4) -> dict: - crop_reference_slots() - output_dir.mkdir(parents=True, exist_ok=True) - candidate_dir = output_dir / "candidates" - selected_ids = parse_slot_ids(slot_ids) - selected_slots = [slot_by_id(slot_id) for slot_id in selected_ids] - specs = [build_asset_generation_spec(slot, 1) for slot in selected_slots] - (output_dir / "asset_generation_specs.json").write_text(json.dumps({ - "summary": "Preflight first-pass generation specs. These specs are used before any image API request.", - "slots": specs, - }, indent=2, ensure_ascii=False), encoding="utf-8") - - def run_one(slot: dict) -> dict: - try: - candidate_path = gemini_generate_from_crop(slot, 1, candidate_dir=candidate_dir) - background_report = harmonize_asset_background(candidate_path, slot["background_color_hex"]) - metrics = evaluate_candidate(slot, candidate_path) - economy_decision = economy_acceptance_decision(slot, metrics) - return { - "slot_id": slot["id"], - "status": "ok", - "candidate_path": str(candidate_path), - "asset_generation_spec": build_asset_generation_spec(slot, 1), - "metrics": metrics, - "economy_decision": economy_decision, - "strict_decision": { - "accepted": bool(metrics.get("pass")), - "needs_regeneration": bool(metrics.get("needs_regeneration")), - }, - "background": background_report, - } - except Exception as exc: - return { - "slot_id": slot["id"], - "status": "generation_failed", - "asset_generation_spec": build_asset_generation_spec(slot, 1), - "error": str(exc), - "economy_decision": {"accepted": False, "reason": "generation_failed"}, - "strict_decision": {"accepted": False, "needs_regeneration": True}, - } - - items = [] - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = {executor.submit(run_one, slot): slot for slot in selected_slots} - for future in as_completed(futures): - items.append(future.result()) - items.sort(key=lambda item: item["slot_id"]) - accepted = [item for item in items if item.get("economy_decision", {}).get("accepted")] - strict_accepted = [item for item in items if item.get("strict_decision", {}).get("accepted")] - contact_sheet = output_dir / "preflight_first_pass_contact_sheet.png" - create_preflight_contact_sheet(items, contact_sheet) - report = { - "summary": "One-shot preflight generation experiment. Each selected slot gets one planned API generation in an isolated experiment directory; main assets and PPT are not overwritten.", - "selected_slots": selected_ids, - "workers": workers, - "main_assets_overwritten": False, - "main_ppt_overwritten": False, - "contact_sheet": str(contact_sheet), - "result": { - "generated_count": len([item for item in items if item["status"] == "ok"]), - "failed_count": len([item for item in items if item["status"] != "ok"]), - "economy_first_pass_accept_count": len(accepted), - "economy_first_pass_accept_rate": round(len(accepted) / max(len(items), 1), 4), - "strict_first_pass_accept_count": len(strict_accepted), - "strict_first_pass_accept_rate": round(len(strict_accepted) / max(len(items), 1), 4), - "estimated_requests_if_one_shot_then_manual_review": len(items), - "estimated_requests_if_strict_five_retry_for_nonpass": len(items) + (len(items) - len(strict_accepted)) * 4, - }, - "items": items, - } - (output_dir / "preflight_first_pass_experiment_report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - return report - - -def edge_connected_background_mask(image: Image.Image, background: tuple[int, int, int], tolerance: int = 42) -> Image.Image: - rgb_image = image.convert("RGB") - width, height = rgb_image.size - pixels = rgb_image.load() - visited = bytearray(width * height) - mask = Image.new("L", (width, height), 0) - mask_pixels = mask.load() - queue: deque[tuple[int, int]] = deque() - - def close_to_background(px_: int, py_: int) -> bool: - color = pixels[px_, py_] - return abs(color[0] - background[0]) + abs(color[1] - background[1]) + abs(color[2] - background[2]) <= tolerance - - def push(px_: int, py_: int) -> None: - index = py_ * width + px_ - if visited[index] or not close_to_background(px_, py_): - return - visited[index] = 1 - mask_pixels[px_, py_] = 255 - queue.append((px_, py_)) - - for px_ in range(width): - push(px_, 0) - push(px_, height - 1) - for py_ in range(height): - push(0, py_) - push(width - 1, py_) - - while queue: - px_, py_ = queue.popleft() - if px_ > 0: - push(px_ - 1, py_) - if px_ + 1 < width: - push(px_ + 1, py_) - if py_ > 0: - push(px_, py_ - 1) - if py_ + 1 < height: - push(px_, py_ + 1) - return mask - - -def latest_candidate_for_slot(slot_id: str) -> Path | None: - slot_dir = CANDIDATE_DIR / slot_id - if not slot_dir.exists(): - return None - candidates = sorted(slot_dir.glob("candidate_*.png")) - return candidates[-1] if candidates else None - - -def candidate_paths_for_slot(slot_id: str) -> list[Path]: - slot_dir = CANDIDATE_DIR / slot_id - if not slot_dir.exists(): - return [] - return sorted(slot_dir.glob("candidate_*.png")) - - -def candidate_meta_path(candidate_path: Path) -> Path: - return candidate_path.with_suffix(".generation.json") - - -def slot_aspect_ratio(slot: dict) -> float: - left, top, right, bottom = slot["asset_bbox"] - return round((right - left) / max(bottom - top, 1), 4) - - -def choose_generation_aspect_ratio(slot: dict) -> str: - override = slot.get("generation_aspect_ratio_override") - if override: - return override - ratio = slot_aspect_ratio(slot) - if ratio >= 1.55: - return "16:9" - if ratio >= 1.18: - return "4:3" - if ratio >= 0.85: - return "1:1" - if ratio >= 0.62: - return "3:4" - return "9:16" - - -def aspect_ratio_prompt_instruction(aspect_ratio: str) -> str: - if aspect_ratio in {"16:9", "4:3"}: - return ( - f"Use a {aspect_ratio} landscape canvas matched to the target slot. " - "Compose the subject horizontally so it naturally spans the canvas width without needing rotation, stacking, or artificial square filling. " - "Keep the top and bottom margins narrow but preserve all important details. " - ) - if aspect_ratio in {"3:4", "9:16"}: - return ( - f"Use a {aspect_ratio} portrait canvas matched to the target slot. " - "Compose the subject vertically so it fills the height while keeping left and right margins narrow. " - "Use a bust or full-height framing as appropriate, but avoid a tiny centered character with broad side margins. " - ) - return ( - "Use a square 1:1 canvas. " - "Compose the subject to fill the square naturally with narrow, even margins. " - ) - - -def compute_slot_fit_metrics(image_path: Path, slot: dict) -> dict: - with Image.open(image_path) as image: - image_width, image_height = image.size - image_ratio = image_width / max(image_height, 1) - slot_ratio = slot_aspect_ratio(slot) - left, top, right, bottom = slot["asset_bbox"] - slot_width = right - left - slot_height = bottom - top - if image_ratio > slot_ratio: - fit_width = slot_width - fit_height = slot_width / max(image_ratio, 0.0001) - else: - fit_height = slot_height - fit_width = slot_height * image_ratio - used_area = fit_width * fit_height - slot_area = max(slot_width * slot_height, 1) - waste = max(0.0, 1.0 - used_area / slot_area) - target_ratio = GENERATION_ASPECT_RATIO_VALUES[choose_generation_aspect_ratio(slot)] - ratio_error = abs(math.log(max(image_ratio, 0.0001) / target_ratio)) - if waste <= SLOT_FIT_WASTE_ACCEPTABLE_MAX and ratio_error <= 0.18: - status = "matched" - elif waste <= SLOT_FIT_WASTE_ACCEPTABLE_MAX: - status = "slot_fit_ok_generated_ratio_drift" - else: - status = "mismatch" - generation_meta = {} - meta_path = candidate_meta_path(image_path) - if meta_path.exists(): - try: - generation_meta = json.loads(meta_path.read_text(encoding="utf-8")) - except Exception: - generation_meta = {"generation_meta_warning": "failed_to_read_generation_meta"} - return { - "slot_aspect_ratio": slot_ratio, - "generation_aspect_ratio": choose_generation_aspect_ratio(slot), - "requested_generation_aspect_ratio": generation_meta.get("requested_generation_aspect_ratio", choose_generation_aspect_ratio(slot)), - "actual_generation_request_aspect_ratio": generation_meta.get("actual_generation_request_aspect_ratio"), - "generation_aspect_ratio_fallback_used": generation_meta.get("generation_aspect_ratio_fallback_used", False), - "generated_image_size": [image_width, image_height], - "generated_image_aspect_ratio": round(image_ratio, 4), - "slot_fit_waste_percent": round(waste, 4), - "aspect_ratio_match_status": status, - **({"generation_meta_warning": generation_meta["generation_meta_warning"]} if "generation_meta_warning" in generation_meta else {}), - } - - -def gemini_generate_from_crop(slot: dict, candidate_index: int = 1, candidate_dir: Path | None = None) -> Path: - api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") - url = os.getenv("GEMINI_GEN_IMG_URL") - if not api_key or not url: - raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required") - crop_path = Path(slot["crop_path"]) - mime = mimetypes.guess_type(crop_path.name)[0] or "image/png" - image_b64 = base64.b64encode(crop_path.read_bytes()).decode("ascii") - spec = build_asset_generation_spec(slot, candidate_index) - aspect_ratio = spec["target_aspect_ratio"] - prompt = prompt_from_asset_generation_spec(spec) - payload = { - "contents": [{ - "role": "user", - "parts": [ - {"text": prompt}, - {"inlineData": {"mimeType": mime, "data": image_b64}}, - ], - }], - "generationConfig": { - "responseModalities": ["IMAGE"], - "imageConfig": {"imageSize": "1K", "aspectRatio": aspect_ratio}, - }, - } - candidate_root = candidate_dir or CANDIDATE_DIR - out_path = candidate_root / slot["id"] / f"candidate_{candidate_index:02d}.png" - out_path.parent.mkdir(parents=True, exist_ok=True) - request_aspects = [aspect_ratio] - if aspect_ratio != "1:1": - request_aspects.append("1:1") - last_error: Exception | None = None - for request_aspect in request_aspects: - payload["generationConfig"]["imageConfig"]["aspectRatio"] = request_aspect - try: - response = requests.post(url, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), timeout=240) - response.raise_for_status() - except requests.HTTPError as exc: - last_error = exc - if request_aspect != "1:1": - continue - raise - data = response.json() - for candidate in data.get("candidates", []): - for part in candidate.get("content", {}).get("parts", []): - inline = part.get("inlineData") or part.get("inline_data") - if inline and inline.get("data"): - out_path.write_bytes(base64.b64decode(inline["data"])) - candidate_meta_path(out_path).write_text(json.dumps({ - "slot_id": slot["id"], - "requested_generation_aspect_ratio": aspect_ratio, - "actual_generation_request_aspect_ratio": request_aspect, - "generation_aspect_ratio_fallback_used": request_aspect != aspect_ratio, - "asset_generation_spec": spec, - }, indent=2, ensure_ascii=False), encoding="utf-8") - return out_path - if last_error: - raise last_error - raise RuntimeError(f"No image returned for {slot['id']}") - - -def generate_one_asset_with_retries(slot: dict, retries: int = 1, economy_mode: bool = True) -> dict: - final_path = ASSET_DIR / f"{slot['id']}.png" - errors = [] - candidate_metrics: list[dict] = [] - existing_candidates = candidate_paths_for_slot(slot["id"]) if os.getenv("RFS_REUSE_ASSETS", "1") == "1" else [] - for candidate_path in existing_candidates: - candidate_metrics.append(evaluate_candidate(slot, candidate_path)) - - best_existing = select_best_candidate(candidate_metrics) - existing_economy_decision = economy_acceptance_decision(slot, best_existing) if best_existing else {"accepted": False, "reason": "no_existing_candidate"} - should_generate_more = ( - best_existing is None - or (not existing_economy_decision["accepted"] if economy_mode else (not best_existing["pass"] or best_existing["needs_regeneration"])) - ) - next_index = len(existing_candidates) + 1 - api_requests_attempted = 0 - if should_generate_more: - for attempt in range(next_index, next_index + retries): - try: - api_requests_attempted += 1 - candidate_path = gemini_generate_from_crop(slot, attempt) - candidate_metrics.append(evaluate_candidate(slot, candidate_path)) - except Exception as exc: - errors.append(str(exc)) - time.sleep(min(2 * (attempt - next_index + 1), 6)) - current_best = select_best_candidate(candidate_metrics) - if current_best and (economy_acceptance_decision(slot, current_best)["accepted"] if economy_mode else current_best["pass"]): - break - - best = select_best_candidate(candidate_metrics) - if best is not None: - candidate_path = Path(best["candidate_path"]) - shutil.copyfile(candidate_path, final_path) - background_report = harmonize_asset_background(final_path, slot["background_color_hex"]) - fill = float(best.get("foreground_bbox_fill_percent") or 0.0) - if best["pass"]: - selected_reason = "selected_candidate_with_internal_fill_80_95_percent" - elif fill < INTERNAL_ACCEPTABLE_FILL_MIN: - selected_reason = "selected_best_available_but_subject_still_too_small" - else: - selected_reason = "selected_best_available_but_possible_edge_cutoff_risk" - economy_decision = economy_acceptance_decision(slot, best) - if economy_mode and economy_decision["accepted"] and not best["pass"]: - selected_reason = f"selected_by_economy_policy_{economy_decision['reason']}" - return { - "slot_id": slot["id"], - "status": "ok" if best["pass"] else ("ok_by_economy_policy" if economy_decision["accepted"] else "ok_with_internal_fill_warning"), - "asset_source": "candidate_selected", - "candidate": str(candidate_path), - "asset": str(final_path), - "candidate_metrics": sorted(candidate_metrics, key=lambda item: item["score"], reverse=True), - "selected_internal_fill": {**best, "selected_reason": selected_reason}, - "economy_decision": economy_decision, - "economy_mode": economy_mode, - "api_requests_attempted": api_requests_attempted, - "background": background_report, - **({"errors": errors} if errors else {}), - } - - fallback = ASSET_DIR / f"{slot['id']}.png" - # Keep the pipeline deliverable if API has a repeated transient failure, but make the fallback explicit. - Image.open(slot["crop_path"]).convert("RGB").save(fallback) - background_report = harmonize_asset_background(fallback, slot["background_color_hex"]) - fallback_metrics = evaluate_candidate(slot, fallback) - return { - "slot_id": slot["id"], - "status": "fallback_to_reference_crop", - "errors": errors, - "asset": str(fallback), - "selected_internal_fill": {**fallback_metrics, "selected_reason": "fallback_reference_crop_after_generation_failure"}, - "economy_decision": economy_acceptance_decision(slot, fallback_metrics), - "economy_mode": economy_mode, - "api_requests_attempted": api_requests_attempted, - "background": background_report, - } - - -def generate_assets_parallel(workers: int = 6, retries: int = 1, economy_mode: bool = True) -> dict: - ASSET_DIR.mkdir(parents=True, exist_ok=True) - CANDIDATE_DIR.mkdir(parents=True, exist_ok=True) - reports = [] - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = {executor.submit(generate_one_asset_with_retries, slot, retries, economy_mode): slot for slot in SLOTS} - for future in as_completed(futures): - slot = futures[future] - try: - reports.append(future.result()) - except Exception as exc: - fallback = ASSET_DIR / f"{slot['id']}.png" - Image.open(slot["crop_path"]).convert("RGB").save(fallback) - background_report = harmonize_asset_background(fallback, slot["background_color_hex"]) - fallback_metrics = evaluate_candidate(slot, fallback) - reports.append({ - "slot_id": slot["id"], - "status": "fallback_to_reference_crop", - "errors": [str(exc)], - "asset": str(fallback), - "selected_internal_fill": {**fallback_metrics, "selected_reason": "fallback_reference_crop_after_parallel_generation_failure"}, - "economy_decision": economy_acceptance_decision(slot, fallback_metrics), - "economy_mode": economy_mode, - "api_requests_attempted": 0, - "background": background_report, - }) - return { - "summary": "AI asset generation report for AutoFigure Architecture rebuild.", - "workers": workers, - "retries": retries, - "economy_mode": economy_mode, - "api_requests_attempted": sum(int(item.get("api_requests_attempted") or 0) for item in reports), - "assets": sorted(reports, key=lambda item: item["slot_id"]), - } - - -def add_picture_contain(slide, image_path: Path, left_px, top_px, width_px, height_px): - with Image.open(image_path) as img: - ratio = img.width / max(img.height, 1) - box_ratio = width_px / max(height_px, 1) - if ratio > box_ratio: - fit_w = width_px - fit_h = width_px / ratio - else: - fit_h = height_px - fit_w = height_px * ratio - return slide.shapes.add_picture(str(image_path), Inches(x(left_px + (width_px - fit_w) / 2)), Inches(y(top_px + (height_px - fit_h) / 2)), width=Inches(w(fit_w)), height=Inches(h(fit_h))) - - -def draw_document_blueprint(slide): - box = add_round(slide, 74, 514, 472, 248, "#F4F8FB", "#6D8BA4", 1.3) - box.line.dash_style = MSO_LINE_DASH_STYLE.DASH - add_text(slide, "Initial Blueprint (S0, A0)", x(160), y(528), w(270), h(34), style="stage_title", color="#111111", size=12.5) - code = '{\n "nodes": [{\n id: "v1",\n pos: [x,y]...\n }],\n "style": "academic_flat"\n}' - add_text(slide, code, x(98), y(574), w(220), h(150), style="small", color="#111111", align=PP_ALIGN.LEFT, size=8.3) - add_line(slide, 336, 656, 421, 656, color="#A6C8DE", width_pt=2.0) - brace = slide.shapes.add_shape(MSO_SHAPE.RIGHT_BRACE, Inches(x(388)), Inches(y(610)), Inches(w(55)), Inches(h(112))) - no_fill(brace) - set_line(brace, "#9FC6DE", 1.6) - add_round(slide, 424, 585, 112, 54, "#E9EEF1", "#7A8790", 1.0) - add_text(slide, "Node A", x(433), y(595), w(90), h(28), style="body", color="#333333", size=10.5) - add_round(slide, 424, 676, 112, 54, "#E9EEF1", "#7A8790", 1.0) - add_text(slide, "Node B", x(433), y(686), w(90), h(28), style="body", color="#333333", size=10.5) - - -def add_panel(slide, left, top, width, height, title, border, fill, header): - add_round(slide, left, top, width, height, fill, border, 1.8) - header_shape = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(left + 2)), Inches(y(top + 2)), Inches(w(width - 4)), Inches(h(62))) - set_fill(header_shape, header, 0) - set_line(header_shape, header, 0.1) - add_text(slide, title, x(left + 16), y(top + 16), w(width - 32), h(36), style="stage_title", color="#333333") - - -def draw_ppt() -> Path: - prs = Presentation() - prs.slide_width = Inches(SLIDE_W) - prs.slide_height = Inches(SLIDE_H) - slide = prs.slides.add_slide(prs.slide_layouts[6]) - slide.background.fill.solid() - slide.background.fill.fore_color.rgb = rgb("#F3F1EC") - - add_text(slide, "AutoFigure Architecture", x(50), y(45), w(620), h(60), style="title", color="#333333", align=PP_ALIGN.LEFT) - add_text(slide, "Decoupled Generative Paradigm: Reasoning, Refinement, and Layered Rendering", x(51), y(114), w(760), h(30), style="subtitle", color="#333333", align=PP_ALIGN.LEFT) - - add_panel(slide, 42, 170, 540, 650, "Stage I: Concept Extraction", "#8AA4BC", "#EAF3FF", "#D6E5F4") - add_panel(slide, 615, 170, 610, 658, "Stage II: Critique-and-Refine", "#E1A45D", "#FFF1DF", "#F8D1A3") - add_panel(slide, 1260, 48, 500, 892, "Stage III: Rendering Strategy", "#84A57F", "#ECF7E9", "#C9DFC1") - - asset = lambda name: ASSET_DIR / f"{name}.png" - add_picture_contain(slide, asset("input_text_stack"), 70, 258, 145, 148) - add_text(slide, "Input Text", x(92), y(421), w(105), h(26), style="label", color="#333333") - render_controls(slide, ["input_to_vlm"]) - add_picture_contain(slide, asset("vlm_agent_robot"), 285, 252, 165, 160) - add_text(slide, "VLM Agent", x(309), y(421), w(118), h(26), style="label", color="#333333") - add_round(slide, 465, 258, 90, 90, "#EAF0F5", "#94A6B9", 1.2) - add_text(slide, "Entities", x(476), y(293), w(68), h(24), style="body", color="#333333", bold=True) - add_round(slide, 465, 355, 90, 90, "#EAF0F5", "#94A6B9", 1.2) - add_text(slide, "Relations", x(474), y(389), w(72), h(24), style="body", color="#333333", bold=True) - render_controls(slide, ["vlm_to_blueprint"]) - draw_document_blueprint(slide) - add_text(slide, '"Map unstructured text to symbolic graph"', x(84), y(774), w(415), h(32), style="body", color="#111111", size=11.0) - render_controls(slide, ["stage_i_to_stage_ii"]) - - # Stage II - add_round(slide, 700, 260, 440, 50, "#FFE7C8", "#D99045", 1.2) - add_text(slide, "Critique: Alignment, Overlap, Balance", x(720), y(274), w(400), h(24), style="callout", color="#333333", size=10.6) - add_round(slide, 638, 350, 104, 70, "#FFE7C8", "#D99045", 1.1) - add_text(slide, "Refine\nLayout", x(657), y(360), w(64), h(42), style="callout", color="#333333", size=8.5) - add_round(slide, 1092, 350, 110, 70, "#FFE7C8", "#D99045", 1.1) - add_text(slide, "Feedback\nF(i)", x(1108), y(360), w(80), h(42), style="callout", color="#333333", size=8.5) - render_controls(slide, ["critique_refinement_loop"]) - add_picture_contain(slide, asset("ai_designer"), 650, 433, 145, 185) - add_text(slide, "AI Designer", x(670), y(642), w(120), h(24), style="label", color="#333333", size=8.7) - add_picture_contain(slide, asset("ai_critic"), 1060, 430, 145, 190) - add_text(slide, "AI Critic", x(1098), y(642), w(86), h(24), style="label", color="#333333", size=8.7) - add_round(slide, 802, 480, 235, 90, "#FFE7C8", "#D99045", 1.3) - add_text(slide, "Score Comparison", x(822), y(500), w(194), h(30), style="callout", color="#333333", size=10.0) - add_text(slide, "q(cand) > q(best)", x(822), y(532), w(194), h(24), style="small", color="#333333", size=8.0) - render_controls(slide, ["designer_refine_arrow", "critic_feedback_arrow"]) - add_round(slide, 690, 748, 460, 48, "#FFE7C8", "#D99045", 1.2) - add_text(slide, "Update: Re-interpret method & improve", x(710), y(761), w(420), h(22), style="callout", color="#333333", size=10.5) - render_controls(slide, ["stage_ii_to_stage_iii"]) - - # Stage III - add_picture_contain(slide, asset("synthesis_tools"), 1380, 128, 230, 120) - add_text(slide, "Synthesis", x(1455), y(251), w(120), h(28), style="label", color="#333333") - render_controls(slide, ["synthesis_to_raw"]) - add_round(slide, 1428, 346, 160, 108, "#F4F4EF", "#B5B5AC", 1.3) - add_text(slide, "Raw Image", x(1450), y(364), w(116), h(26), style="label", color="#333333", size=10.2) - add_text(slide, "Text Blur!", x(1460), y(404), w(96), h(24), style="callout", color="#E24D42", size=9.2) - add_picture_contain(slide, asset("erase_text_tool"), 1290, 486, 140, 125) - add_text(slide, "Erase Text", x(1320), y(595), w(108), h(28), style="label", color="#333333", size=9.5) - add_text(slide, "ABC", x(1404), y(516), w(55), h(24), style="label", color="#333333", size=9.0) - add_picture_contain(slide, asset("ocr_verify"), 1570, 490, 150, 125) - add_text(slide, "OCR + Verify", x(1578), y(595), w(140), h(28), style="label", color="#333333", size=9.5) - add_picture_contain(slide, asset("final_autofigure_card"), 1410, 684, 185, 128) - add_text(slide, "Final AutoFigure", x(1418), y(815), w(178), h(28), style="label", color="#333333", size=10.0) - add_text(slide, "A stunning, high-resolution scientific\ndiagram with crisp details and a pleasing\ncolor scheme, ready for publication.", x(1368), y(846), w(300), h(70), style="body", color="#333333", size=8.2) - render_controls(slide, ["raw_to_erase_route", "raw_to_ocr_route", "erase_to_final_route", "ocr_to_final_route", "finalization_arrow"]) - - # Bottom legend - add_round(slide, 42, 858, 1180, 80, "#F4F2ED", "#D2CEC8", 1.2) - add_text(slide, "Key Methodology:", x(105), y(890), w(210), h(24), style="legend", color="#333333", align=PP_ALIGN.LEFT) - add_round(slide, 328, 875, 280, 50, "#DFECF8", "#8AA4BC", 1.2) - add_text(slide, "Concept Extraction", x(380), y(890), w(180), h(22), style="legend", color="#333333") - add_round(slide, 628, 875, 280, 50, "#FFE6C9", "#D99045", 1.2) - add_text(slide, "Critique-and-Refine", x(670), y(890), w(200), h(22), style="legend", color="#333333") - add_round(slide, 925, 875, 280, 50, "#DCEED7", "#84A57F", 1.2) - add_text(slide, "Rendering Strategy", x(970), y(890), w(190), h(22), style="legend", color="#333333") - - target = OUT / "editable_composition.pptx" - try: - prs.save(target) - return target - except PermissionError: - fallback = OUT / "editable_composition_controls_calibrated.pptx" - prs.save(fallback) - return fallback - - -def export_preview(pptx_path: Path) -> None: - try: - import win32com.client # type: ignore - except Exception as exc: - (OUT / "preview_export_error.txt").write_text(str(exc), encoding="utf-8") - return - app = win32com.client.Dispatch("PowerPoint.Application") - app.Visible = 1 - presentation = app.Presentations.Open(str(pptx_path), WithWindow=False) - try: - presentation.Slides(1).Export(str(OUT / "rebuild_preview.png"), "PNG", REF_W, REF_H) - finally: - presentation.Close() - app.Quit() - - -def write_metadata(asset_report: dict) -> None: - controls = [normalize_control(control) for control in CONTROL_SPECS] - internal_fill_report = { - "summary": "Internal effective-content fill report for small AI icon assets. No local crop, auto-crop, PowerPoint crop, or layout enlargement is applied.", - "policy": { - "icon_type": "functional_icon_or_small_mark", - "target_fill_range": [INTERNAL_ACCEPTABLE_FILL_MIN, INTERNAL_ACCEPTABLE_FILL_MAX], - "target_fill_center": INTERNAL_CONTENT_FILL_TARGET, - "regenerate_below": INTERNAL_REGENERATE_FILL_THRESHOLD, - "no_crop_postprocess": True, - "layout_bbox_locked": True, - }, - "assets": [ - { - "slot_id": item["slot_id"], - **item.get("selected_internal_fill", {}), - } - for item in asset_report.get("assets", []) - ], - } - background_report = { - "summary": "Background harmonization report for generated/reused AI assets.", - "policy": "match_slot_local_background", - "assets": [ - { - "slot_id": item["slot_id"], - **item.get("background", {}), - } - for item in asset_report.get("assets", []) - ], - } - ratio_fit_report = { - "summary": "Slot-aware aspect-ratio fit report for generated icon assets. This distinguishes true small-subject problems from square-canvas letterboxing in non-square slots.", - "policy": { - "layout_bbox_locked": True, - "no_default_crop": True, - "slot_fit_waste_acceptable_max": SLOT_FIT_WASTE_ACCEPTABLE_MAX, - "aspect_ratio_selection": { - ">=1.55": "16:9", - "1.18-1.55": "4:3", - "0.85-1.18": "1:1", - "0.62-0.85": "3:4", - "<0.62": "9:16", - }, - }, - "assets": [ - { - "slot_id": item["slot_id"], - "slot_aspect_ratio": item.get("selected_internal_fill", {}).get("slot_aspect_ratio"), - "generation_aspect_ratio": item.get("selected_internal_fill", {}).get("generation_aspect_ratio"), - "requested_generation_aspect_ratio": item.get("selected_internal_fill", {}).get("requested_generation_aspect_ratio"), - "actual_generation_request_aspect_ratio": item.get("selected_internal_fill", {}).get("actual_generation_request_aspect_ratio"), - "generation_aspect_ratio_fallback_used": item.get("selected_internal_fill", {}).get("generation_aspect_ratio_fallback_used", False), - "generated_image_size": item.get("selected_internal_fill", {}).get("generated_image_size"), - "generated_image_aspect_ratio": item.get("selected_internal_fill", {}).get("generated_image_aspect_ratio"), - "slot_fit_waste_percent": item.get("selected_internal_fill", {}).get("slot_fit_waste_percent"), - "foreground_bbox_fill_percent": item.get("selected_internal_fill", {}).get("foreground_bbox_fill_percent"), - "aspect_ratio_match_status": item.get("selected_internal_fill", {}).get("aspect_ratio_match_status"), - "selected_candidate": item.get("candidate"), - "status": item.get("status"), - "economy_decision": item.get("economy_decision"), - "api_requests_attempted": item.get("api_requests_attempted", 0), - "generation_errors": item.get("errors", []), - } - for item in asset_report.get("assets", []) - ], - } - economy_report = { - "summary": "Economy asset generation report. Accepted assets are locked and skipped; failed slots use bounded one-shot generation unless strict mode is explicitly requested.", - "economy_mode": asset_report.get("economy_mode"), - "retries": asset_report.get("retries"), - "api_requests_attempted": asset_report.get("api_requests_attempted", 0), - "profiles": ECONOMY_ACCEPTANCE_PROFILES, - "assets": [ - { - "slot_id": item["slot_id"], - "status": item.get("status"), - "api_requests_attempted": item.get("api_requests_attempted", 0), - "economy_decision": item.get("economy_decision"), - "selected_candidate": item.get("candidate"), - "foreground_bbox_fill_percent": item.get("selected_internal_fill", {}).get("foreground_bbox_fill_percent"), - "slot_fit_waste_percent": item.get("selected_internal_fill", {}).get("slot_fit_waste_percent"), - "selected_reason": item.get("selected_internal_fill", {}).get("selected_reason"), - } - for item in asset_report.get("assets", []) - ], - } - inventory = { - "summary": "Slot inventory for AI-generated image assets in AutoFigure Architecture editable rebuild.", - "reference": str(REFERENCE), - "slots": [ - { - "slot_id": slot["id"], - "bbox_percent": slot["bbox_percent"], - "asset_bbox_px": slot["asset_bbox"], - "slot_aspect_ratio": slot_aspect_ratio(slot), - "generation_aspect_ratio": choose_generation_aspect_ratio(slot), - "generation_aspect_ratio_override": slot.get("generation_aspect_ratio_override"), - "reference_crop_path": slot["crop_path"], - "asset_path": str(ASSET_DIR / f"{slot['id']}.png"), - "background_color_hex": slot["background_color_hex"], - "internal_content_fill_target": slot["internal_content_fill_target"], - "internal_max_margin_percent": slot["internal_max_margin_percent"], - "layout_bbox_locked": slot["layout_bbox_locked"], - "no_crop_postprocess": slot["no_crop_postprocess"], - "prompt": slot["prompt"], - "fill_guidance": slot.get("fill_guidance"), - } - for slot in SLOTS - ], - } - (OUT / "slot_inventory.json").write_text(json.dumps(inventory, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "asset_generation_report.json").write_text(json.dumps(asset_report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "asset_internal_fill_report.json").write_text(json.dumps(internal_fill_report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "asset_ratio_fit_report.json").write_text(json.dumps(ratio_fit_report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "asset_economy_report.json").write_text(json.dumps(economy_report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "asset_background_report.json").write_text(json.dumps(background_report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "reference_control_candidates.json").write_text(json.dumps({ - "summary": "Reference-derived candidate arrows, connector lines, and dashed loops for the AutoFigure Architecture rebuild.", - "extraction_mode": "manual_reference_locked_first_pass", - "note": "This is the structured control layer that will later be replaced/refined by image-processing or VLM control extraction.", - "controls": controls, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "reference_controls.json").write_text(json.dumps({ - "summary": "Bound editable PPT control layer for arrows, connector lines, and dashed loops.", - "controls": controls, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "arrow_style_profile.json").write_text(json.dumps({ - "summary": "Arrow and connector style profile.", - "reference_image_hard_constraint": True, - "control_localizer_mode": "manual_reference_locked_first_pass", - "style_tokens": { - "stage_transition": {"color": "#555555", "width_pt": 4.0, "arrow_head": "triangle"}, - "stage_i_flow": {"color": "#607998", "width_pt": 2.5, "arrow_head": "triangle"}, - "critique_loop": {"color": "#D99045", "width_pt": 2.0, "dash": True}, - "rendering_loop": {"color": "#6F9A66", "width_pt": 1.8, "dash": True}, - }, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "selected_arrow_routes.json").write_text(json.dumps({ - "summary": "Selected editable PowerPoint routes for all reference controls.", - "route_generation_status": "reference_locked_manual_seed", - "routes": controls, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "arrow_quality_report.json").write_text(json.dumps({ - "summary": "First-pass arrow quality report.", - "status": "improved_structured_controls", - "known_limitations": [ - "This pass stores measured/control coordinates in JSON but does not yet run automatic computer-vision arrow extraction.", - "Dashed closed loops are still approximated with PPT oval/polyline controls.", - ], - "control_count": len(controls), - "fallback_routing_used": False, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "arrow_direction_report.json").write_text(json.dumps({ - "summary": "Arrow position and direction report. This file prioritizes arrow placement, target direction, anchors, and arrowhead endpoint over style details.", - "status": "position_direction_fields_added", - "direction_contract": { - "start_px": "tail position in reference-image pixel coordinates", - "end_px": "arrowhead target position in reference-image pixel coordinates", - "direction.label": "dominant direction inferred from start_px -> end_px", - "source_anchor": "inferred source edge from direction unless explicitly overridden", - "target_anchor": "inferred target edge from direction unless explicitly overridden", - }, - "arrows": [ - { - "id": control["id"], - "kind": control["kind"], - "source_id": control["source_id"], - "target_id": control["target_id"], - "source_anchor": control.get("source_anchor"), - "target_anchor": control.get("target_anchor"), - "position": control.get("position"), - "direction": control.get("direction"), - } - for control in controls - ], - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "reference_style_profile.json").write_text(json.dumps({ - "summary": "Reference style profile for AutoFigure Architecture editable rebuild.", - "canvas_background": "#F3F1EC", - "stage_backgrounds": {"stage_i": "#EAF3FF", "stage_ii": "#FFF1DF", "stage_iii": "#ECF7E9"}, - "asset_background_policy": "match_slot_local_background", - "font_family": FONT, - "text_size_scale": TEXT_SIZE_SCALE, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "layout_plan.json").write_text(json.dumps({ - "summary": "Reference-guided layout plan for the AutoFigure Architecture editable rebuild.", - "canvas": {"width_px": REF_W, "height_px": REF_H, "slide_width_in": SLIDE_W, "slide_height_in": SLIDE_H}, - "panels": [ - {"id": "stage_i", "bbox_px": [42, 170, 540, 650], "fill": "#EAF3FF"}, - {"id": "stage_ii", "bbox_px": [615, 170, 610, 658], "fill": "#FFF1DF"}, - {"id": "stage_iii", "bbox_px": [1260, 48, 500, 892], "fill": "#ECF7E9"}, - ], - "slots": inventory["slots"], - "controls": controls, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "reference_text_geometry.json").write_text(json.dumps({ - "summary": "Reference text geometry seed for editable PPT text.", - "detection_mode": "manual_reference_seed", - "font_size_estimation": "reference bbox height converted to PPT points, then calibrated with TEXT_SIZE_SCALE", - "text_size_scale": TEXT_SIZE_SCALE, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "text_program.json").write_text(json.dumps({ - "summary": "Editable text program for AutoFigure Architecture rebuild.", - "font_family_guess": FONT, - "fit_strategy": "bbox_bound_with_global_scale", - "text_style_tokens": TEXT_STYLES, - "text_size_scale": TEXT_SIZE_SCALE, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "text_alignment_report.json").write_text(json.dumps({ - "summary": "Text alignment report for the AutoFigure Architecture editable rebuild.", - "status": "scaled_up", - "scale_applied": TEXT_SIZE_SCALE, - "reason": "Prior preview text was visually small; this pass applies a uniform calibration factor while preserving editable PPT text boxes.", - "next_recommended_step": "render-and-measure calibration against OCR/reference text masks.", - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "figure_program.json").write_text(json.dumps({ - "summary": "PowerPoint-first reconstruction with AI-generated local illustration assets and editable PPT structure/text.", - "reference_size_px": [REF_W, REF_H], - "slide_size_in": [SLIDE_W, SLIDE_H], - "editable_layers": ["text", "stage panels", "cards", "arrows", "dashed loops", "methodology legend"], - "raster_asset_layers": [slot["id"] for slot in SLOTS], - "controls": controls, - "asset_background_policy": "match_slot_local_background", - "asset_internal_fill_policy": { - "target_fill_range": [INTERNAL_ACCEPTABLE_FILL_MIN, INTERNAL_ACCEPTABLE_FILL_MAX], - "target_fill_center": INTERNAL_CONTENT_FILL_TARGET, - "no_crop_postprocess": True, - "layout_bbox_locked": True, - }, - "asset_aspect_ratio_policy": { - "mode": "slot_aware_generation_canvas", - "slot_fit_waste_acceptable_max": SLOT_FIT_WASTE_ACCEPTABLE_MAX, - "supported_generation_aspect_ratios": list(GENERATION_ASPECT_RATIO_VALUES.keys()), - "report": str(OUT / "asset_ratio_fit_report.json"), - }, - "asset_economy_policy": { - "enabled": asset_report.get("economy_mode"), - "api_requests_attempted": asset_report.get("api_requests_attempted", 0), - "type_aware_profiles": ECONOMY_ACCEPTANCE_PROFILES, - "report": str(OUT / "asset_economy_report.json"), - }, - "text_style_tokens": TEXT_STYLES, - "text_size_scale": TEXT_SIZE_SCALE, - }, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "rebuild_notes.md").write_text( - "# Summary\n" - "AutoFigure Architecture was rebuilt as an editable PowerPoint composition with AI-generated local illustration assets.\n\n" - "## Editable PPT Layers\n" - "- All titles, labels, captions, callouts, methodology legend text, panels, cards, arrows, and dashed loops are PowerPoint objects.\n\n" - "## AI Raster Asset Layers\n" - "- Complex illustrations such as robot characters, document stack, synthesis tools, eraser, OCR magnifier, and final figure card were generated as local image assets from reference crops.\n" - "- Critical text is not trusted to the generated images; it is added as editable PPT text.\n\n" - "## Generation\n" - f"- Parallel workers: {asset_report.get('workers')}\n" - f"- Economy mode: {asset_report.get('economy_mode')}; image API requests attempted: {asset_report.get('api_requests_attempted', 0)}\n" - "- If an API asset failed, the report marks it explicitly and uses the reference crop as a temporary fallback.\n\n" - "## Control Layer\n" - "- Arrows, connector lines, and dashed loops are now recorded in reference_controls.json and selected_arrow_routes.json before PPT rendering.\n" - "- reference_control_overlay.png visualizes the current reference-locked control layer for manual QA.\n\n" - "## Background and Text Calibration\n" - "- Each generated or reused AI asset is harmonized to the local slot background color recorded in slot_inventory.json.\n" - f"- Editable PPT text uses TEXT_SIZE_SCALE={TEXT_SIZE_SCALE} as a first-pass visual calibration factor.\n\n" - "## Internal Icon Fill\n" - f"- Small icon assets target {INTERNAL_ACCEPTABLE_FILL_MIN:.0%}-{INTERNAL_ACCEPTABLE_FILL_MAX:.0%} internal effective-content fill.\n" - "- Candidate selection is based on asset_internal_fill_report.json.\n" - "- Generation canvas aspect ratio is selected from each locked asset slot; see asset_ratio_fit_report.json.\n" - "- No local crop, automatic crop, PowerPoint crop, or PPT layout enlargement is used.\n", - encoding="utf-8", - ) - - -def slot_by_id(slot_id: str) -> dict: - lookup = {slot["id"]: slot for slot in SLOTS} - if slot_id not in lookup: - raise ValueError(f"Unknown trim slot: {slot_id}") - return lookup[slot_id] - - -def selected_candidate_from_internal_report(slot_id: str) -> Path | None: - report_path = OUT / "asset_internal_fill_report.json" - if not report_path.exists(): - return None - try: - data = json.loads(report_path.read_text(encoding="utf-8")) - except Exception: - return None - for item in data.get("assets", []): - if item.get("slot_id") == slot_id and item.get("candidate_path"): - path = Path(item["candidate_path"]) - if path.exists(): - return path - return None - - -def source_icon_for_trim_experiment(slot: dict) -> Path: - return selected_candidate_from_internal_report(slot["id"]) or latest_candidate_for_slot(slot["id"]) or (ASSET_DIR / f"{slot['id']}.png") - - -def trim_icon_candidate_for_experiment(slot: dict, source_path: Path, out_path: Path, padding_percent: float) -> dict: - before = measure_internal_content_fill(source_path, slot["background_color_hex"]) - bbox = before.get("foreground_bbox_px") - if not bbox: - shutil.copyfile(source_path, out_path) - after = measure_internal_content_fill(out_path, slot["background_color_hex"]) - return { - "slot_id": slot["id"], - "source_path": str(source_path), - "output_path": str(out_path), - "crop_applied": False, - "trim_rejected": True, - "risk_flags": ["invalid_foreground_bbox"], - "before": before, - "after": after, - "fill_gain": round((after.get("foreground_bbox_fill_percent") or 0) - (before.get("foreground_bbox_fill_percent") or 0), 4), - "recommendation": "reject", - } - - image = Image.open(source_path).convert("RGB") - width, height = image.size - left, top, right, bottom = bbox - bbox_w = right - left + 1 - bbox_h = bottom - top + 1 - pad = int(max(bbox_w, bbox_h) * padding_percent) - crop_left = max(0, left - pad) - crop_top = max(0, top - pad) - crop_right = min(width - 1, right + pad) - crop_bottom = min(height - 1, bottom + pad) - crop_w = crop_right - crop_left + 1 - crop_h = crop_bottom - crop_top + 1 - risk_flags: list[str] = [] - if crop_left / width < 0.02 or crop_top / height < 0.02 or (width - 1 - crop_right) / width < 0.02 or (height - 1 - crop_bottom) / height < 0.02: - risk_flags.append("edge_cutoff_risk") - if crop_w <= 8 or crop_h <= 8: - risk_flags.append("invalid_crop_box") - - if "invalid_crop_box" in risk_flags: - shutil.copyfile(source_path, out_path) - crop_applied = False - else: - crop = image.crop((crop_left, crop_top, crop_right + 1, crop_bottom + 1)) - scale = min(width / crop.width, height / crop.height) - fit_w = max(1, int(crop.width * scale)) - fit_h = max(1, int(crop.height * scale)) - resized = crop.resize((fit_w, fit_h), Image.Resampling.LANCZOS) - canvas = Image.new("RGB", (width, height), hex_to_tuple(slot["background_color_hex"])) - canvas.paste(resized, ((width - fit_w) // 2, (height - fit_h) // 2)) - canvas.save(out_path) - crop_applied = True - - after = measure_internal_content_fill(out_path, slot["background_color_hex"]) - after_fill = float(after.get("foreground_bbox_fill_percent") or 0.0) - before_fill = float(before.get("foreground_bbox_fill_percent") or 0.0) - if after_fill > INTERNAL_ACCEPTABLE_FILL_MAX: - risk_flags.append("too_tight_risk") - fill_gain = round(after_fill - before_fill, 4) - if crop_applied and INTERNAL_ACCEPTABLE_FILL_MIN <= after_fill <= INTERNAL_ACCEPTABLE_FILL_MAX and fill_gain >= 0.10 and not risk_flags: - recommendation = "accept" - elif crop_applied and fill_gain > 0 and "invalid_crop_box" not in risk_flags: - recommendation = "needs_visual_review" - else: - recommendation = "reject" - return { - "slot_id": slot["id"], - "source_path": str(source_path), - "output_path": str(out_path), - "crop_applied": crop_applied, - "trim_rejected": recommendation == "reject", - "padding_percent": padding_percent, - "crop_box_px": [crop_left, crop_top, crop_right, crop_bottom], - "risk_flags": risk_flags, - "before": before, - "after": after, - "before_fill": before_fill, - "after_fill": after_fill, - "fill_gain": fill_gain, - "recommendation": recommendation, - } - - -def create_trim_contact_sheet(rows: list[dict], output_path: Path) -> None: - thumb = 220 - label_h = 54 - gap = 18 - width = thumb * 2 + gap * 3 - height = (thumb + label_h + gap) * max(len(rows), 1) + gap - sheet = Image.new("RGB", (width, height), "#F3F1EC") - draw = ImageDraw.Draw(sheet) - for idx, row in enumerate(rows): - y0 = gap + idx * (thumb + label_h + gap) - before = Image.open(row["before_path"]).convert("RGB") - after = Image.open(row["after_path"]).convert("RGB") - before.thumbnail((thumb, thumb), Image.Resampling.LANCZOS) - after.thumbnail((thumb, thumb), Image.Resampling.LANCZOS) - x_before = gap + (thumb - before.width) // 2 - x_after = gap * 2 + thumb + (thumb - after.width) // 2 - sheet.paste(before, (x_before, y0 + (thumb - before.height) // 2)) - sheet.paste(after, (x_after, y0 + (thumb - after.height) // 2)) - label = f"{row['slot_id']} | {row['before_fill']:.3f} -> {row['after_fill']:.3f} | {row['recommendation']}" - draw.text((gap, y0 + thumb + 8), label, fill="#333333") - draw.text((gap, y0 - 2), "before", fill="#666666") - draw.text((gap * 2 + thumb, y0 - 2), "after", fill="#666666") - output_path.parent.mkdir(parents=True, exist_ok=True) - sheet.save(output_path) - - -def run_icon_trim_experiment(trim_slots: str, trim_padding_percent: float, trim_output_dir: Path) -> None: - trim_output_dir.mkdir(parents=True, exist_ok=True) - before_dir = trim_output_dir / "before" - after_dir = trim_output_dir / "after" - before_dir.mkdir(parents=True, exist_ok=True) - after_dir.mkdir(parents=True, exist_ok=True) - rows = [] - report_items = [] - for slot_id in [item.strip() for item in trim_slots.split(",") if item.strip()]: - slot = slot_by_id(slot_id) - source_path = source_icon_for_trim_experiment(slot) - if not source_path.exists(): - report_items.append({"slot_id": slot_id, "error": f"source image not found: {source_path}", "recommendation": "reject"}) - continue - before_path = before_dir / f"{slot_id}.png" - after_path = after_dir / f"{slot_id}.png" - shutil.copyfile(source_path, before_path) - item = trim_icon_candidate_for_experiment(slot, before_path, after_path, trim_padding_percent) - report_items.append(item) - rows.append({ - "slot_id": slot_id, - "before_path": str(before_path), - "after_path": str(after_path), - "before_fill": item["before_fill"], - "after_fill": item["after_fill"], - "recommendation": item["recommendation"], - }) - contact_sheet = trim_output_dir / "contact_sheet_before_after.png" - create_trim_contact_sheet(rows, contact_sheet) - report = { - "summary": "Controlled icon trim experiment. Main assets and main PPT are not overwritten.", - "trim_slots": [item.strip() for item in trim_slots.split(",") if item.strip()], - "trim_padding_percent": trim_padding_percent, - "output_dir": str(trim_output_dir), - "main_assets_overwritten": False, - "main_ppt_overwritten": False, - "contact_sheet": str(contact_sheet), - "items": report_items, - } - (trim_output_dir / "icon_trim_experiment_report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser(description="Rebuild AutoFigure Architecture as an editable PPTX.") - parser.add_argument("--icon-trim-experiment", action="store_true", help="Run a controlled icon trim A/B experiment without changing main assets or PPT.") - parser.add_argument("--trim-slots", default="input_text_stack,ai_designer", help="Comma-separated slot ids for icon trim experiment.") - parser.add_argument("--trim-padding-percent", type=float, default=0.08, help="Padding added around detected foreground bbox before experimental trim.") - parser.add_argument("--trim-output-dir", default=str(OUT / "icon_trim_experiment"), help="Output directory for trim experiment artifacts.") - parser.add_argument("--asset-cost-experiment", action="store_true", help="Dry-run the asset generation cost-saving policy without calling the image API.") - parser.add_argument("--asset-cost-experiment-output", default=str(OUT / "asset_cost_experiment_report.json"), help="Output JSON path for the cost-saving dry-run report.") - parser.add_argument("--asset-cost-baseline-retries", type=int, default=5, help="Baseline retries per failed slot used for cost comparison.") - parser.add_argument("--asset-preflight-first-pass-experiment", action="store_true", help="Generate one planned candidate per selected slot in an isolated experiment directory.") - parser.add_argument("--preflight-slots", default="all", help="Comma-separated slot ids, or 'all', for the first-pass preflight experiment.") - parser.add_argument("--preflight-output-dir", default=str(OUT / "preflight_first_pass_experiment"), help="Output directory for first-pass preflight experiment artifacts.") - parser.add_argument("--preflight-workers", type=int, default=4, help="Parallel workers for the first-pass preflight experiment.") - parser.add_argument("--asset-retries", type=int, default=1, help="Maximum image-generation attempts per slot that is not accepted by the active asset policy.") - parser.add_argument("--asset-workers", type=int, default=6, help="Parallel workers for main asset generation.") - parser.add_argument("--strict-asset-regeneration", action="store_true", help="Use strict 80-95 percent fill acceptance instead of economy type-aware acceptance.") - args = parser.parse_args() - OUT.mkdir(parents=True, exist_ok=True) - if args.icon_trim_experiment: - run_icon_trim_experiment(args.trim_slots, args.trim_padding_percent, Path(args.trim_output_dir)) - return - if args.asset_cost_experiment: - run_asset_cost_experiment(args.asset_cost_baseline_retries, Path(args.asset_cost_experiment_output)) - return - if args.asset_preflight_first_pass_experiment: - run_preflight_first_pass_experiment(args.preflight_slots, Path(args.preflight_output_dir), args.preflight_workers) - return - crop_reference_slots() - write_reference_overlays() - asset_report = generate_assets_parallel( - workers=args.asset_workers, - retries=args.asset_retries, - economy_mode=not args.strict_asset_regeneration, - ) - write_metadata(asset_report) - pptx_path = draw_ppt() - export_preview(pptx_path) - - -if __name__ == "__main__": - main() diff --git a/experiments/legacy_reference_rebuilds/rebuild_c60_editable.py b/experiments/legacy_reference_rebuilds/rebuild_c60_editable.py deleted file mode 100644 index 853eb06..0000000 --- a/experiments/legacy_reference_rebuilds/rebuild_c60_editable.py +++ /dev/null @@ -1,551 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from pptx import Presentation -from pptx.dml.color import RGBColor -from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE -from pptx.enum.text import MSO_ANCHOR, PP_ALIGN -from pptx.oxml import parse_xml -from pptx.util import Inches, Pt - - -ROOT = Path(__file__).resolve().parents[1] -REFERENCE = Path(r"C:\Users\zhang\Documents\xwechat_files\wxid_h824xk1qpfoh22_adc2\temp\RWTemp\2026-06\860847045f7ccc2e7052eb182bab099b\c60bea4f7eb67de472ee6cf6bcbd09d6.png") -OUT = ROOT / "output" / "c60_editable_rebuild" - -REF_W = 1806 -REF_H = 920 -SLIDE_W = 15.60 -SLIDE_H = SLIDE_W * REF_H / REF_W - -FONT_REGULAR = "Arial" -FONT_UI = "Microsoft YaHei UI" -TEXT_STYLES = { - "panel_title": {"font": FONT_REGULAR, "size": 11.2, "bold": True}, - "panel_title_small": {"font": FONT_REGULAR, "size": 10.3, "bold": True}, - "module_label": {"font": FONT_REGULAR, "size": 7.7, "bold": True}, - "module_output": {"font": FONT_REGULAR, "size": 7.3, "bold": False}, - "body": {"font": FONT_REGULAR, "size": 8.7, "bold": False}, - "body_small": {"font": FONT_REGULAR, "size": 7.7, "bold": False}, - "legend": {"font": FONT_REGULAR, "size": 8.8, "bold": True}, - "trait": {"font": FONT_REGULAR, "size": 6.7, "bold": False}, - "badge": {"font": FONT_REGULAR, "size": 10.5, "bold": True}, -} - - -def x(px: float) -> float: - return px / REF_W * SLIDE_W - - -def y(px: float) -> float: - return px / REF_H * SLIDE_H - - -def w(px: float) -> float: - return px / REF_W * SLIDE_W - - -def h(px: float) -> float: - return px / REF_H * SLIDE_H - - -def rgb(hex_color: str) -> RGBColor: - value = hex_color.strip().lstrip("#") - return RGBColor(int(value[:2], 16), int(value[2:4], 16), int(value[4:6], 16)) - - -def set_line(shape, color: str, width: float = 1.4, dash: bool = False) -> None: - shape.line.color.rgb = rgb(color) - shape.line.width = Pt(width) - if dash: - from pptx.enum.dml import MSO_LINE_DASH_STYLE - - shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH - - -def set_fill(shape, color: str, transparency: int = 0) -> None: - shape.fill.solid() - shape.fill.fore_color.rgb = rgb(color) - shape.fill.transparency = transparency - - -def no_fill(shape) -> None: - shape.fill.background() - - -def add_text(slide, text: str, left: float, top: float, width: float, height: float, *, size: float | None = None, color: str = "#111111", bold: bool | None = None, align=PP_ALIGN.CENTER, font: str | None = None, valign=MSO_ANCHOR.MIDDLE, style: str | None = None): - spec = TEXT_STYLES.get(style or "", {}) - font = font or str(spec.get("font") or FONT_REGULAR) - size = float(size if size is not None else spec.get("size", 11)) - bold = bool(bold if bold is not None else spec.get("bold", False)) - box = slide.shapes.add_textbox(Inches(left), Inches(top), Inches(width), Inches(height)) - tf = box.text_frame - tf.clear() - tf.word_wrap = True - tf.vertical_anchor = valign - tf.margin_left = Inches(0.01) - tf.margin_right = Inches(0.01) - tf.margin_top = Inches(0.0) - tf.margin_bottom = Inches(0.0) - for idx, raw_line in enumerate(text.split("\n")): - p = tf.paragraphs[0] if idx == 0 else tf.add_paragraph() - p.alignment = align - run = p.add_run() - run.text = raw_line - run.font.size = Pt(size) - run.font.bold = bold - run.font.name = font - run.font.color.rgb = rgb(color) - return box - - -def add_panel(slide, left_px, top_px, width_px, height_px, title: str, color="#0B3A91", title_size=12.5): - panel = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(left_px)), Inches(y(top_px)), Inches(w(width_px)), Inches(h(height_px))) - set_fill(panel, "#FFFFFF", 0) - set_line(panel, color, 1.55) - panel.shadow.inherit = False - add_text(slide, title, x(left_px + 10), y(top_px + 8), w(width_px - 20), y(58), size=title_size, color=color, style="panel_title") - return panel - - -def add_card(slide, left_px, top_px, width_px, height_px, border: str, fill: str = "#FFFFFF", radius=True): - shape_type = MSO_SHAPE.ROUNDED_RECTANGLE if radius else MSO_SHAPE.RECTANGLE - card = slide.shapes.add_shape(shape_type, Inches(x(left_px)), Inches(y(top_px)), Inches(w(width_px)), Inches(h(height_px))) - set_fill(card, fill, 0) - set_line(card, border, 1.2) - card.shadow.inherit = False - return card - - -def apply_arrow(connector, size: str = "sm") -> None: - ln = connector.line._get_or_add_ln() - ln.append(parse_xml(f'')) - - -def add_arrow(slide, x1, y1, x2, y2, color="#000000", width_pt=1.3): - conn = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(x(x1)), Inches(y(y1)), Inches(x(x2)), Inches(y(y2))) - set_line(conn, color, width_pt) - apply_arrow(conn) - return conn - - -def add_line(slide, x1, y1, x2, y2, color="#000000", width_pt=1.2, dash=False): - conn = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(x(x1)), Inches(y(y1)), Inches(x(x2)), Inches(y(y2))) - set_line(conn, color, width_pt, dash=dash) - return conn - - -def add_polyline(slide, points, color="#000000", width_pt=1.2, arrow=False): - lines = [] - for idx, (p1, p2) in enumerate(zip(points[:-1], points[1:])): - conn = add_line(slide, p1[0], p1[1], p2[0], p2[1], color=color, width_pt=width_pt) - if arrow and idx == len(points) - 2: - apply_arrow(conn) - lines.append(conn) - return lines - - -def icon_video(slide, left_px, top_px, width_px=76, height_px=62, color="#173B5D"): - box = add_card(slide, left_px, top_px, width_px, height_px, "#0B2844", "#2B4C6D", radius=False) - for i in range(5): - for side_x in (left_px + 6, left_px + width_px - 14): - sq = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(side_x)), Inches(y(top_px + 8 + i * 10)), Inches(w(8)), Inches(h(5))) - set_fill(sq, "#C8D8EA") - set_line(sq, "#C8D8EA", 0.2) - tri = slide.shapes.add_shape(MSO_SHAPE.ISOSCELES_TRIANGLE, Inches(x(left_px + width_px * 0.43)), Inches(y(top_px + height_px * 0.28)), Inches(w(width_px * 0.25)), Inches(h(height_px * 0.42))) - tri.rotation = 90 - set_fill(tri, "#FFFFFF") - set_line(tri, "#FFFFFF", 0.5) - return box - - -def icon_wave(slide, left_px, top_px, width_px=82, height_px=42, color="#0A459C"): - cx = left_px + width_px / 2 - for i in range(15): - px = left_px + 4 + i * (width_px - 8) / 14 - amp = (0.25 + 0.75 * abs(7 - i) / 7) * height_px * 0.42 - add_line(slide, px, top_px + height_px / 2 - amp / 2, px, top_px + height_px / 2 + amp / 2, color=color, width_pt=1.25) - add_line(slide, left_px, top_px + height_px / 2, left_px + 8, top_px + height_px / 2, color=color, width_pt=1) - add_line(slide, cx + width_px / 2 - 8, top_px + height_px / 2, cx + width_px / 2, top_px + height_px / 2, color=color, width_pt=1) - - -def icon_doc(slide, left_px, top_px, width_px=44, height_px=58, color="#111111"): - doc = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(left_px)), Inches(y(top_px)), Inches(w(width_px)), Inches(h(height_px))) - set_fill(doc, "#FFFFFF") - set_line(doc, color, 1.2) - fold = slide.shapes.add_shape(MSO_SHAPE.RIGHT_TRIANGLE, Inches(x(left_px + width_px - 15)), Inches(y(top_px)), Inches(w(15)), Inches(h(15))) - set_fill(fold, "#E9EEF3") - set_line(fold, color, 0.8) - for i in range(3): - add_line(slide, left_px + 8, top_px + 20 + i * 11, left_px + width_px - 8, top_px + 20 + i * 11, color="#1A1A1A", width_pt=0.9) - - -def icon_mic(slide, left_px, top_px, color="#173B5D"): - oval = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(left_px + 16)), Inches(y(top_px)), Inches(w(22)), Inches(h(48))) - set_fill(oval, color) - set_line(oval, color, 1.0) - add_line(slide, left_px + 7, top_px + 26, left_px + 7, top_px + 40, color=color, width_pt=2) - add_line(slide, left_px + 47, top_px + 26, left_px + 47, top_px + 40, color=color, width_pt=2) - add_line(slide, left_px + 8, top_px + 40, left_px + 27, top_px + 55, color=color, width_pt=2) - add_line(slide, left_px + 46, top_px + 40, left_px + 27, top_px + 55, color=color, width_pt=2) - add_line(slide, left_px + 27, top_px + 55, left_px + 27, top_px + 72, color=color, width_pt=2) - add_line(slide, left_px + 12, top_px + 72, left_px + 42, top_px + 72, color=color, width_pt=2) - - -def icon_face(slide, left_px, top_px, size_px=62): - frame = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(left_px)), Inches(y(top_px)), Inches(w(size_px)), Inches(h(size_px))) - no_fill(frame) - set_line(frame, "#6B57C8", 1.0) - head = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left_px + size_px * 0.28)), Inches(y(top_px + size_px * 0.18)), Inches(w(size_px * 0.44)), Inches(h(size_px * 0.48))) - set_fill(head, "#D7B08E") - set_line(head, "#6B4C35", 0.8) - hair = slide.shapes.add_shape(MSO_SHAPE.ARC, Inches(x(left_px + size_px * 0.23)), Inches(y(top_px + size_px * 0.13)), Inches(w(size_px * 0.54)), Inches(h(size_px * 0.32))) - no_fill(hair) - set_line(hair, "#202020", 2.5) - for ex in (left_px + size_px * 0.41, left_px + size_px * 0.58): - eye = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(ex)), Inches(y(top_px + size_px * 0.40)), Inches(w(3)), Inches(h(3))) - set_fill(eye, "#111111") - set_line(eye, "#111111", 0.2) - body = slide.shapes.add_shape(MSO_SHAPE.ARC, Inches(x(left_px + size_px * 0.25)), Inches(y(top_px + size_px * 0.64)), Inches(w(size_px * 0.50)), Inches(h(size_px * 0.25))) - no_fill(body) - set_line(body, "#284A7C", 2.2) - - -def icon_person(slide, left_px, top_px, scale=1.0, shirt="#546E7A"): - head = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left_px + 18 * scale)), Inches(y(top_px)), Inches(w(18 * scale)), Inches(h(18 * scale))) - set_fill(head, "#D0A079") - set_line(head, "#5A4030", 0.6) - body = slide.shapes.add_shape(MSO_SHAPE.TRAPEZOID, Inches(x(left_px + 12 * scale)), Inches(y(top_px + 20 * scale)), Inches(w(30 * scale)), Inches(h(48 * scale))) - set_fill(body, shirt) - set_line(body, "#333333", 0.6) - add_line(slide, left_px + 17 * scale, top_px + 68 * scale, left_px + 10 * scale, top_px + 95 * scale, color="#222222", width_pt=1.4) - add_line(slide, left_px + 35 * scale, top_px + 68 * scale, left_px + 43 * scale, top_px + 95 * scale, color="#222222", width_pt=1.4) - - -def icon_pose(slide, left_px, top_px, scale=1.0, color="#111111"): - pts = { - "head": (left_px + 34 * scale, top_px + 8 * scale), - "neck": (left_px + 34 * scale, top_px + 25 * scale), - "hip": (left_px + 34 * scale, top_px + 58 * scale), - "lh": (left_px + 10 * scale, top_px + 42 * scale), - "rh": (left_px + 58 * scale, top_px + 42 * scale), - "lf": (left_px + 17 * scale, top_px + 88 * scale), - "rf": (left_px + 52 * scale, top_px + 88 * scale), - } - head = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(pts["head"][0] - 5 * scale)), Inches(y(pts["head"][1] - 5 * scale)), Inches(w(10 * scale)), Inches(h(10 * scale))) - set_fill(head, color) - set_line(head, color, 0.5) - for a, b in [("neck", "hip"), ("neck", "lh"), ("neck", "rh"), ("hip", "lf"), ("hip", "rf")]: - add_line(slide, pts[a][0], pts[a][1], pts[b][0], pts[b][1], color=color, width_pt=1.5) - for key, c in [("lh", "#E17721"), ("rh", "#7A55C7"), ("lf", "#E17721"), ("rf", "#0A7F9C")]: - dot = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(pts[key][0] - 3 * scale)), Inches(y(pts[key][1] - 3 * scale)), Inches(w(6 * scale)), Inches(h(6 * scale))) - set_fill(dot, c) - set_line(dot, c, 0.3) - - -def icon_frames(slide, left_px, top_px): - for i in range(4): - rect = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(left_px + i * 7)), Inches(y(top_px + i * 5)), Inches(w(44)), Inches(h(54))) - set_fill(rect, "#F6F6F6") - set_line(rect, "#333333", 0.8) - icon_person(slide, left_px + 28, top_px + 26, scale=0.33) - - -def icon_checklist(slide, left_px, top_px): - board = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(left_px)), Inches(y(top_px)), Inches(w(70)), Inches(h(96))) - set_fill(board, "#EAF2FB") - set_line(board, "#0B3A91", 1.2) - clip = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(left_px + 22)), Inches(y(top_px - 8)), Inches(w(28)), Inches(h(16))) - set_fill(clip, "#2F5A88") - set_line(clip, "#2F5A88", 0.6) - for i in range(4): - cy = top_px + 24 + i * 16 - add_line(slide, left_px + 16, cy, left_px + 20, cy + 5, color="#173B5D", width_pt=1.5) - add_line(slide, left_px + 20, cy + 5, left_px + 28, cy - 4, color="#173B5D", width_pt=1.5) - add_line(slide, left_px + 36, cy, left_px + 58, cy, color="#111111", width_pt=1.0) - - -def icon_database_people(slide, left_px, top_px): - top = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left_px)), Inches(y(top_px)), Inches(w(92)), Inches(h(28))) - set_fill(top, "#008C91") - set_line(top, "#00737B", 1.0) - body = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(left_px)), Inches(y(top_px + 14)), Inches(w(92)), Inches(h(58))) - set_fill(body, "#008C91") - set_line(body, "#00737B", 1.0) - bottom = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left_px)), Inches(y(top_px + 58)), Inches(w(92)), Inches(h(28))) - set_fill(bottom, "#008C91") - set_line(bottom, "#00737B", 1.0) - for yy in (top_px + 28, top_px + 50): - add_line(slide, left_px + 6, yy, left_px + 86, yy, color="#FFFFFF", width_pt=1.4) - for i, (dx, dy, s) in enumerate([(80, 48, 1.0), (105, 50, 0.8), (66, 63, 0.8)]): - head = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left_px + dx)), Inches(y(top_px + dy)), Inches(w(20 * s)), Inches(h(20 * s))) - set_fill(head, "#0A6973") - set_line(head, "#0A6973", 0.8) - body2 = slide.shapes.add_shape(MSO_SHAPE.ARC, Inches(x(left_px + dx - 8 * s)), Inches(y(top_px + dy + 16 * s)), Inches(w(36 * s)), Inches(h(26 * s))) - no_fill(body2) - set_line(body2, "#0A6973", 4.0) - - -def icon_camera(slide, left_px, top_px): - cam = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(left_px)), Inches(y(top_px)), Inches(w(50)), Inches(h(32))) - set_fill(cam, "#2B2B2B") - set_line(cam, "#111111", 1) - lens = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(left_px + 13)), Inches(y(top_px + 6)), Inches(w(18)), Inches(h(18))) - set_fill(lens, "#111111") - set_line(lens, "#444444", 0.8) - tri = slide.shapes.add_shape(MSO_SHAPE.ISOSCELES_TRIANGLE, Inches(x(left_px + 48)), Inches(y(top_px + 7)), Inches(w(26)), Inches(h(18))) - tri.rotation = 90 - set_fill(tri, "#2B2B2B") - set_line(tri, "#111111", 0.8) - add_line(slide, left_px + 25, top_px + 32, left_px + 25, top_px + 78, color="#111111", width_pt=2) - add_line(slide, left_px + 25, top_px + 50, left_px + 5, top_px + 96, color="#111111", width_pt=1.8) - add_line(slide, left_px + 25, top_px + 50, left_px + 50, top_px + 96, color="#111111", width_pt=1.8) - add_line(slide, left_px + 25, top_px + 50, left_px + 25, top_px + 96, color="#111111", width_pt=1.8) - - -def draw_rebuild() -> None: - OUT.mkdir(parents=True, exist_ok=True) - prs = Presentation() - prs.slide_width = Inches(SLIDE_W) - prs.slide_height = Inches(SLIDE_H) - slide = prs.slides.add_slide(prs.slide_layouts[6]) - bg = slide.background.fill - bg.solid() - bg.fore_color.rgb = rgb("#FFFFFF") - - blue = "#0B3A91" - cyan = "#007E86" - orange = "#C95100" - purple = "#6B57C8" - green = "#2D7B36" - - # Main panels. - add_panel(slide, 10, 44, 245, 660, "1. Virtual Interview\nSetup", blue, 12) - add_panel(slide, 303, 172, 213, 438, "2. Raw Video:\n287 participants\n× 36 questions", blue, 12) - add_panel(slide, 585, 44, 392, 690, "3. Preprocessing Modules", blue, 12) - add_panel(slide, 1022, 190, 155, 445, "4. Timestamp\nAlignment", blue, 11.5) - add_panel(slide, 1206, 136, 192, 571, "5. Five Modalities", cyan, 11.5) - add_panel(slide, 1430, 136, 188, 571, "6. NEO-FFI-3\nSelf-report Labels", orange, 11.2) - add_panel(slide, 1654, 240, 158, 447, "7. Multimodal\nPersonality\nDataset", cyan, 11.2) - - # Panel 1. - icon_person(slide, 18, 140, scale=1.25, shirt="#5D6C76") - monitor = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(88)), Inches(y(128)), Inches(w(150)), Inches(h(96))) - set_fill(monitor, "#9EAAB5") - set_line(monitor, "#111111", 1.5) - add_text(slide, "00:15", x(94), y(132), w(42), h(18), size=5.5, color="#FFFFFF", bold=True) - icon_face(slide, 151, 140, size_px=42) - add_text(slide, "Participant in front of\n49-inch screen", x(47), y(292), w(165), h(42), style="body") - screen = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, Inches(x(31)), Inches(y(352)), Inches(w(197)), Inches(h(104))) - set_fill(screen, "#9EAAB5") - set_line(screen, "#111111", 1.4) - icon_face(slide, 105, 368, size_px=58) - add_text(slide, "3D virtual interviewer\non screen", x(46), y(465), w(150), h(42), style="body") - icon_camera(slide, 31, 530) - add_line(slide, 83, 552, 207, 520, color=blue, width_pt=0.9, dash=True) - add_line(slide, 83, 552, 208, 630, color=blue, width_pt=0.9, dash=True) - icon_person(slide, 205, 518, scale=0.55, shirt="#6E7F87") - add_text(slide, "Wide-angle camera\ncapturing full body", x(44), y(648), w(165), h(45), style="body") - - # Panel 2 raw video. - icon_video(slide, 335, 300, 152, 110) - for row in range(2): - for col in range(3 if row == 0 else 2): - px = 323 + col * 58 - py = 440 + row * 78 - thumb = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(px)), Inches(y(py)), Inches(w(52)), Inches(h(62))) - set_fill(thumb, "#D3D0C8") - set_line(thumb, "#555555", 0.6) - icon_person(slide, px + 16, py + 8, scale=0.36, shirt="#4C6C78") - add_text(slide, "...", x(447), y(520), w(42), h(30), size=17, bold=True) - - # Panel 3 preprocessing cards. - cards = [ - (596, 98, 372, 102, "#2E76CC", "#F8FBFF", "FFmpeg", "Audio", blue), - (596, 224, 372, 102, "#2D8A45", "#F7FFF7", "FunASR", "Spoken Text\n+ Timestamps", green), - (596, 352, 372, 102, purple, "#FCFAFF", "MTCNN", "Face Clips", purple), - (596, 478, 372, 102, cyan, "#F7FFFF", "AlphaPose", "Full-body\nPose\nSkeletons", cyan), - (596, 604, 372, 102, orange, "#FFF8F2", "Frame\nSampling", "Video\nFrames", orange), - ] - for idx, (lx, ty, ww, hh, border, fill, label, out_label, label_color) in enumerate(cards): - add_card(slide, lx, ty, ww, hh, border, fill) - cy = ty + hh / 2 - if idx == 0: - icon_video(slide, lx + 18, ty + 22, 66, 52) - icon_wave(slide, lx + 224, ty + 32, 70, 36) - add_text(slide, label, x(lx + 92), y(ty + 34), w(96), h(34), color=blue, style="module_label") - add_arrow(slide, lx + 180, cy, lx + 216, cy, color="#111111") - add_text(slide, out_label, x(lx + 300), y(ty + 35), w(68), h(32), style="module_output") - elif idx == 1: - icon_mic(slide, lx + 18, ty + 21, color="#173B5D") - icon_doc(slide, lx + 210, ty + 23) - add_text(slide, label, x(lx + 74), y(ty + 35), w(82), h(30), color=green, style="module_label") - add_arrow(slide, lx + 154, cy, lx + 202, cy, color="#111111") - clock = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(lx + 236)), Inches(y(ty + 64)), Inches(w(24)), Inches(h(24))) - no_fill(clock) - set_line(clock, "#111111", 1.0) - add_text(slide, out_label, x(lx + 270), y(ty + 37), w(94), h(38), size=6.9, style="module_output") - elif idx == 2: - icon_face(slide, lx + 20, ty + 30, size_px=42) - icon_face(slide, lx + 206, ty + 21, size_px=70) - add_text(slide, label, x(lx + 74), y(ty + 38), w(84), h(28), color=purple, style="module_label") - add_arrow(slide, lx + 154, cy, lx + 200, cy, color="#111111") - add_text(slide, out_label, x(lx + 282), y(ty + 38), w(82), h(28), style="module_output") - elif idx == 3: - icon_pose(slide, lx + 16, ty + 19, scale=0.82) - icon_pose(slide, lx + 205, ty + 18, scale=0.82, color="#254D8E") - add_text(slide, label, x(lx + 70), y(ty + 36), w(94), h(30), color=cyan, style="module_label") - add_arrow(slide, lx + 154, cy, lx + 200, cy, color="#111111") - add_text(slide, out_label, x(lx + 278), y(ty + 28), w(88), h(58), size=7.0, style="module_output") - else: - icon_frames(slide, lx + 18, ty + 20) - person_box = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(lx + 210)), Inches(y(ty + 22)), Inches(w(62)), Inches(h(72))) - set_fill(person_box, "#D3D0C8") - set_line(person_box, "#555555", 0.7) - icon_person(slide, lx + 230, ty + 32, scale=0.38) - add_text(slide, label, x(lx + 100), y(ty + 24), w(82), h(56), color=orange, style="module_label") - add_arrow(slide, lx + 172, cy, lx + 204, cy, color="#111111") - add_text(slide, out_label, x(lx + 292), y(ty + 34), w(70), h(38), style="module_output") - - # Main flow arrows around preprocessing. - add_arrow(slide, 255, 392, 297, 392, color="#111111") - add_arrow(slide, 516, 268, 582, 268, color="#111111") - add_arrow(slide, 516, 392, 582, 392, color="#111111") - add_arrow(slide, 516, 522, 582, 522, color="#111111") - add_polyline(slide, [(516, 392), (550, 392), (550, 145), (582, 145)], color="#111111", width_pt=1.2, arrow=True) - add_polyline(slide, [(516, 392), (550, 392), (550, 650), (582, 650)], color="#111111", width_pt=1.2, arrow=True) - add_polyline(slide, [(977, 145), (1000, 145), (1000, 270), (1018, 270)], color="#111111", width_pt=1.2, arrow=True) - add_polyline(slide, [(977, 270), (1000, 270), (1000, 270), (1018, 270)], color="#111111", width_pt=1.2, arrow=True) - add_polyline(slide, [(977, 397), (1000, 397), (1000, 270), (1018, 270)], color="#111111", width_pt=1.2, arrow=True) - add_polyline(slide, [(977, 522), (1000, 522), (1000, 270), (1018, 270)], color="#111111", width_pt=1.2, arrow=True) - add_polyline(slide, [(977, 650), (1000, 650), (1000, 270), (1018, 270)], color="#111111", width_pt=1.2, arrow=True) - - # Timestamp alignment. - clock = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(1056)), Inches(y(312)), Inches(w(88)), Inches(h(88))) - no_fill(clock) - set_line(clock, blue, 3.0) - add_line(slide, 1100, 356, 1100, 326, color=blue, width_pt=3) - add_line(slide, 1100, 356, 1124, 378, color=blue, width_pt=3) - dot = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(1094)), Inches(y(350)), Inches(w(12)), Inches(h(12))) - set_fill(dot, blue) - set_line(dot, blue, 0.4) - add_text(slide, "Align audio,\ntext, face,\nframes, and\npose streams\nto a unified\ntimeline", x(1048), y(438), w(103), h(135), style="body_small") - add_arrow(slide, 1177, 410, 1202, 410, color="#111111") - - # Five modalities. - mod_cards = [ - (1216, 200, "#6B57C8", "#FCFAFF", "Face"), - (1216, 306, "#E17721", "#FFF8F2", "Frame"), - (1216, 410, cyan, "#F7FFFF", "Pose"), - (1216, 518, "#2E76CC", "#F8FBFF", "Audio"), - (1216, 624, "#2D8A45", "#F7FFF7", "Text"), - ] - for i, (lx, ty, border, fill, label) in enumerate(mod_cards): - add_card(slide, lx, ty, 172, 88, border, fill) - if label == "Face": - icon_face(slide, lx + 20, ty + 14, size_px=62) - elif label == "Frame": - icon_video(slide, lx + 18, ty + 16, 82, 56) - icon_person(slide, lx + 50, ty + 24, scale=0.32) - elif label == "Pose": - icon_pose(slide, lx + 35, ty + 13, scale=0.70, color="#254D8E") - elif label == "Audio": - icon_wave(slide, lx + 22, ty + 25, 88, 40) - else: - icon_doc(slide, lx + 30, ty + 17, 45, 56) - add_text(slide, label, x(lx + 102), y(ty + 27), w(72), h(34), color=border, style="module_label") - add_arrow(slide, 1398, ty + 44, 1425, ty + 44, color="#111111") - - # NEO labels. - icon_checklist(slide, 1484, 222) - add_text(slide, "OCEAN Scores", x(1464), y(348), w(122), h(32), size=9.2, bold=True) - traits = [ - ("O", "Openness", "#775DCC"), - ("C", "Conscientiousness", "#D76B12"), - ("E", "Extraversion", "#178B95"), - ("A", "Agreeableness", "#4E9D56"), - ("N", "Neuroticism", "#3B7FC6"), - ] - for idx, (letter, name, color) in enumerate(traits): - yy = 403 + idx * 56 - circ = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(1448)), Inches(y(yy)), Inches(w(34)), Inches(h(34))) - set_fill(circ, color) - set_line(circ, color, 0.6) - add_text(slide, letter, x(1448), y(yy + 1), w(34), h(30), color="#FFFFFF", style="badge") - add_text(slide, name, x(1488), y(yy + 2), w(128), h(30), align=PP_ALIGN.LEFT, style="trait") - - add_arrow(slide, 1618, 412, 1650, 412, color="#111111") - - # Dataset panel. - icon_database_people(slide, 1684, 352) - add_text(slide, "Multimodal\nvideos aligned\nacross five\nmodalities with\nNEO-FFI-3\n(OCEAN) labels", x(1668), y(475), w(130), h(152), style="body_small") - - # Legend. - legend = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, Inches(x(20)), Inches(y(780)), Inches(w(940)), Inches(h(88))) - no_fill(legend) - set_line(legend, "#173B5D", 1.0, dash=True) - icon_video(slide, 48, 802, 48, 42) - add_text(slide, "Video", x(110), y(809), w(70), h(28), align=PP_ALIGN.LEFT, style="legend") - icon_wave(slide, 200, 806, 70, 34) - add_text(slide, "Audio", x(278), y(809), w(70), h(28), align=PP_ALIGN.LEFT, style="legend") - icon_doc(slide, 376, 799, 34, 46) - add_text(slide, "Text", x(422), y(809), w(62), h(28), align=PP_ALIGN.LEFT, style="legend") - icon_face(slide, 506, 798, size_px=48) - add_text(slide, "Face", x(566), y(809), w(58), h(28), align=PP_ALIGN.LEFT, style="legend") - icon_pose(slide, 642, 794, scale=0.58) - add_text(slide, "Pose", x(700), y(809), w(70), h(28), align=PP_ALIGN.LEFT, style="legend") - icon_frames(slide, 800, 798) - add_text(slide, "Frames", x(862), y(809), w(82), h(28), align=PP_ALIGN.LEFT, style="legend") - - # Minimal reproducibility metadata. - metadata = { - "summary": "Editable PPTX reconstruction of the provided reference image using PowerPoint text, shapes, connectors, and vector-like icons.", - "reference_image": str(REFERENCE), - "output_pptx": str(OUT / "editable_composition.pptx"), - "editable_layers": ["all visible text", "main containers", "module cards", "arrows", "legend", "simple icons"], - "approximated_layers": ["human figures", "interviewer portrait", "camera perspective", "database people icon"], - "text_style_tokens": TEXT_STYLES, - "canvas": {"reference_px": [REF_W, REF_H], "slide_inches": [SLIDE_W, SLIDE_H]}, - } - (OUT / "figure_program.json").write_text(json.dumps(metadata, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "rebuild_notes.md").write_text( - "# Summary\n" - "This is a PowerPoint-first editable reconstruction of the provided image.\n\n" - "## Editable\n" - "- Titles, labels, captions, OCEAN trait text, and legend labels are PPT text boxes.\n" - "- Section containers, module cards, arrows, connectors, checklist, waves, document, video, pose, database, and camera icons are PPT shapes/connectors.\n\n" - "## Approximate\n" - "- Human figures, face portraits, screen views, and database people are editable shape approximations rather than exact traced artwork.\n" - "- The goal is functional editability and layout fidelity, not pixel-perfect illustration tracing.\n" - "\n## Reusable Typography Pass\n" - "- Text is routed through role-based style tokens in `scripts/rebuild_c60_editable.py`.\n" - "- Reuse `panel_title`, `module_label`, `module_output`, `body`, `trait`, and `legend` for future image-to-PPT rebuilds instead of hand-tuning every text box.\n" - "- Font weight is controlled by role, with smaller bold module labels and regular body/trait labels to avoid over-heavy text.\n", - encoding="utf-8", - ) - - pptx = OUT / "editable_composition.pptx" - prs.save(pptx) - - -def export_preview() -> None: - try: - import win32com.client # type: ignore - except Exception as exc: - (OUT / "preview_export_error.txt").write_text(str(exc), encoding="utf-8") - return - app = win32com.client.Dispatch("PowerPoint.Application") - app.Visible = 1 - presentation = app.Presentations.Open(str(OUT / "editable_composition.pptx"), WithWindow=False) - try: - presentation.Slides(1).Export(str(OUT / "rebuild_preview.png"), "PNG", 1806, 920) - finally: - presentation.Close() - app.Quit() - - -if __name__ == "__main__": - draw_rebuild() - export_preview() diff --git a/experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py b/experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py deleted file mode 100644 index f93002e..0000000 --- a/experiments/legacy_reference_rebuilds/rebuild_virtual_interview_pipeline_editable.py +++ /dev/null @@ -1,550 +0,0 @@ -from __future__ import annotations - -import argparse -import base64 -import json -import mimetypes -import os -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path - -from PIL import Image -import requests -from pptx import Presentation -from pptx.dml.color import RGBColor -from pptx.enum.dml import MSO_LINE_DASH_STYLE -from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE -from pptx.enum.text import MSO_ANCHOR, PP_ALIGN -from pptx.util import Inches, Pt - - -ROOT = Path(__file__).resolve().parents[1] -REFERENCE = Path(r"C:\Users\zhang\AppData\Local\Temp\codex-clipboard-ff76bc61-99eb-43af-a58c-c3179570fd8e.png") -OUT = ROOT / "output" / "virtual_interview_pipeline_editable_rebuild" -ASSET_DIR = OUT / "assets" -REFERENCE_CROP_DIR = OUT / "reference_slot_crops" -AI_CANDIDATE_DIR = OUT / "asset_candidates" - -REF_W = 1774 -REF_H = 887 -SLIDE_W = 15.60 -SLIDE_H = SLIDE_W * REF_H / REF_W -FONT = "Arial" - - -ASSET_SPECS = [ - { - "id": "setup_top", - "bbox": (22, 120, 238, 280), - "prompt": "participant sitting at a desk facing a large monitor with a virtual female interviewer on screen", - "background": "#FFFFFF", - }, - { - "id": "setup_mid", - "bbox": (31, 342, 238, 428), - "prompt": "large flat monitor showing a 3D female virtual interviewer portrait, clean lab illustration", - "background": "#FFFFFF", - }, - { - "id": "setup_cam", - "bbox": (22, 505, 235, 625), - "prompt": "wide-angle camera on tripod capturing a standing participant, blue dotted field-of-view lines", - "background": "#FFFFFF", - }, - { - "id": "raw_people", - "bbox": (318, 430, 490, 530), - "prompt": "small video thumbnails showing a standing participant in front of a neutral background", - "background": "#FFFFFF", - }, - { - "id": "face", - "bbox": (1205, 199, 1280, 267), - "prompt": "front-facing male face portrait inside a face-detection crop frame", - "background": "#F8FBFF", - }, - { - "id": "person_frame", - "bbox": (1200, 305, 1280, 372), - "prompt": "film-strip frame containing a full-body standing participant", - "background": "#F8FBFF", - }, - { - "id": "pose", - "bbox": (1212, 402, 1278, 470), - "prompt": "full-body human pose skeleton with colored joints and limbs", - "background": "#F8FBFF", - }, - { - "id": "dataset", - "bbox": (1640, 345, 1745, 445), - "prompt": "teal database cylinder with grouped user silhouettes in front", - "background": "#FFFFFF", - }, -] - - -def x(px: float) -> float: - return px / REF_W * SLIDE_W - - -def y(px: float) -> float: - return px / REF_H * SLIDE_H - - -def w(px: float) -> float: - return px / REF_W * SLIDE_W - - -def h(px: float) -> float: - return px / REF_H * SLIDE_H - - -def rgb(hex_color: str) -> RGBColor: - value = hex_color.strip("#") - return RGBColor(int(value[0:2], 16), int(value[2:4], 16), int(value[4:6], 16)) - - -def crop_asset(slot_id: str, bbox: tuple[int, int, int, int]) -> Path: - ASSET_DIR.mkdir(parents=True, exist_ok=True) - out = ASSET_DIR / f"{slot_id}.png" - with Image.open(REFERENCE).convert("RGB") as image: - image.crop(bbox).save(out) - return out - - -def crop_reference_slot(slot_id: str, bbox: tuple[int, int, int, int]) -> Path: - REFERENCE_CROP_DIR.mkdir(parents=True, exist_ok=True) - out = REFERENCE_CROP_DIR / f"{slot_id}.png" - with Image.open(REFERENCE).convert("RGB") as image: - image.crop(bbox).save(out) - return out - - -def supported_aspect_ratio(width: int, height: int) -> str: - ratio = width / max(height, 1) - if ratio >= 1.55: - return "16:9" - if ratio >= 1.18: - return "4:3" - if ratio >= 0.85: - return "1:1" - if ratio >= 0.62: - return "3:4" - return "9:16" - - -def generate_ai_asset(spec: dict, crop_path: Path, out_path: Path) -> dict: - api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") - url = os.getenv("GEMINI_GEN_IMG_URL") - if not api_key or not url: - raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for API asset generation") - - left, top, right, bottom = spec["bbox"] - aspect_ratio = supported_aspect_ratio(right - left, bottom - top) - mime = mimetypes.guess_type(crop_path.name)[0] or "image/png" - image_b64 = base64.b64encode(crop_path.read_bytes()).decode("ascii") - prompt = ( - "Use the provided crop as a reference for object identity, pose, and scientific-diagram style. " - "Create a clean standalone raster asset for an editable PowerPoint research pipeline figure. " - f"Main subject: {spec['prompt']}. " - f"Canvas aspect ratio: {aspect_ratio}. " - "Keep the subject large and centered, with only narrow background margins. " - "Preserve the reference's clean academic illustration style, subtle shadows, dark outline, and muted colors. " - f"Use a flat background matching {spec['background']}. " - "Do not include any readable text, labels, numbers, captions, watermarks, panel borders, or arrows. " - "Do not generate a full diagram; generate only this local visual asset." - ) - payload = { - "contents": [{ - "role": "user", - "parts": [ - {"text": prompt}, - {"inlineData": {"mimeType": mime, "data": image_b64}}, - ], - }], - "generationConfig": { - "responseModalities": ["IMAGE"], - "imageConfig": {"imageSize": "1K", "aspectRatio": aspect_ratio}, - }, - } - out_path.parent.mkdir(parents=True, exist_ok=True) - response = requests.post( - url, - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=240, - ) - response.raise_for_status() - data = response.json() - for candidate in data.get("candidates", []): - for part in candidate.get("content", {}).get("parts", []): - inline = part.get("inlineData") or part.get("inline_data") - if inline and inline.get("data"): - out_path.write_bytes(base64.b64decode(inline["data"])) - return { - "slot_id": spec["id"], - "status": "generated", - "asset_path": str(out_path), - "reference_crop_path": str(crop_path), - "aspect_ratio": aspect_ratio, - "prompt": prompt, - } - raise RuntimeError(f"No image returned for {spec['id']}") - - -def prepare_visual_assets(asset_mode: str = "api", workers: int = 4, force_generate: bool = False) -> tuple[dict[str, Path], dict]: - ASSET_DIR.mkdir(parents=True, exist_ok=True) - assets: dict[str, Path] = {} - reports: list[dict] = [] - crops = {spec["id"]: crop_reference_slot(spec["id"], spec["bbox"]) for spec in ASSET_SPECS} - - if asset_mode == "crop": - for spec in ASSET_SPECS: - out_path = ASSET_DIR / f"{spec['id']}.png" - with Image.open(crops[spec["id"]]).convert("RGB") as image: - image.save(out_path) - assets[spec["id"]] = out_path - reports.append({"slot_id": spec["id"], "status": "reference_crop", "asset_path": str(out_path), "api_requests_attempted": 0}) - return assets, {"summary": "Visual asset preparation report.", "asset_mode": asset_mode, "api_requests_attempted": 0, "assets": reports} - - def run_one(spec: dict) -> dict: - out_path = ASSET_DIR / f"{spec['id']}.png" - if out_path.exists() and not force_generate: - return {"slot_id": spec["id"], "status": "reused_existing_ai_asset", "asset_path": str(out_path), "api_requests_attempted": 0} - try: - report = generate_ai_asset(spec, crops[spec["id"]], out_path) - return {**report, "api_requests_attempted": 1} - except Exception as exc: - with Image.open(crops[spec["id"]]).convert("RGB") as image: - image.save(out_path) - return { - "slot_id": spec["id"], - "status": "fallback_reference_crop_after_api_failure", - "asset_path": str(out_path), - "reference_crop_path": str(crops[spec["id"]]), - "api_requests_attempted": 1, - "error": str(exc), - } - - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = {executor.submit(run_one, spec): spec for spec in ASSET_SPECS} - for future in as_completed(futures): - reports.append(future.result()) - reports.sort(key=lambda item: item["slot_id"]) - for item in reports: - assets[item["slot_id"]] = Path(item["asset_path"]) - return assets, { - "summary": "Visual asset preparation report.", - "asset_mode": asset_mode, - "api_requests_attempted": sum(int(item.get("api_requests_attempted") or 0) for item in reports), - "assets": reports, - } - - -def add_text(slide, text: str, left, top, width, height, *, size=10, color="#111111", bold=False, align=PP_ALIGN.CENTER): - shape = slide.shapes.add_textbox(Inches(x(left)), Inches(y(top)), Inches(w(width)), Inches(h(height))) - tf = shape.text_frame - tf.clear() - tf.margin_left = 0 - tf.margin_right = 0 - tf.margin_top = 0 - tf.margin_bottom = 0 - tf.vertical_anchor = MSO_ANCHOR.MIDDLE - p = tf.paragraphs[0] - p.alignment = align - run = p.add_run() - run.text = text - run.font.name = FONT - run.font.size = Pt(size) - run.font.bold = bold - run.font.color.rgb = rgb(color) - return shape - - -def add_box(slide, left, top, width, height, *, line="#00328C", fill="#FFFFFF", radius=True, lw=1.2, dash=False): - shape_type = MSO_SHAPE.ROUNDED_RECTANGLE if radius else MSO_SHAPE.RECTANGLE - shape = slide.shapes.add_shape(shape_type, Inches(x(left)), Inches(y(top)), Inches(w(width)), Inches(h(height))) - shape.fill.solid() - shape.fill.fore_color.rgb = rgb(fill) - shape.line.color.rgb = rgb(line) - shape.line.width = Pt(lw) - if dash: - shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH - return shape - - -def add_picture_contain(slide, image_path: Path, left, top, width, height): - with Image.open(image_path) as image: - ratio = image.width / max(image.height, 1) - box_ratio = width / max(height, 1) - if ratio > box_ratio: - fit_w = width - fit_h = width / ratio - else: - fit_h = height - fit_w = height * ratio - return slide.shapes.add_picture( - str(image_path), - Inches(x(left + (width - fit_w) / 2)), - Inches(y(top + (height - fit_h) / 2)), - width=Inches(w(fit_w)), - height=Inches(h(fit_h)), - ) - - -def add_arrow(slide, x1, y1, x2, y2, *, color="#000000", width=2.0): - conn = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(x(x1)), Inches(y(y1)), Inches(x(x2)), Inches(y(y2))) - conn.line.color.rgb = rgb(color) - conn.line.width = Pt(width) - conn.line.end_arrowhead = True - return conn - - -def add_line(slide, x1, y1, x2, y2, *, color="#000000", width=1.5): - conn = slide.shapes.add_connector(MSO_CONNECTOR.STRAIGHT, Inches(x(x1)), Inches(y(y1)), Inches(x(x2)), Inches(y(y2))) - conn.line.color.rgb = rgb(color) - conn.line.width = Pt(width) - return conn - - -def add_panel(slide, left, top, width, height, title, color="#00328C"): - add_box(slide, left, top, width, height, line=color, fill="#FFFFFF", radius=True, lw=1.5) - add_text(slide, title, left + 8, top + 10, width - 16, 54, size=12.8, color=color, bold=True) - - -def add_film_icon(slide, left, top, width, height, *, fill="#3A6FA5"): - box = add_box(slide, left, top, width, height, line="#1B2F4A", fill=fill, radius=True, lw=1.2) - for i in range(5): - add_box(slide, left + 8 + i * (width - 28) / 4, top + 8, 10, 8, line="#1B2F4A", fill="#BFD3E6", radius=False, lw=0.6) - add_box(slide, left + 8 + i * (width - 28) / 4, top + height - 16, 10, 8, line="#1B2F4A", fill="#BFD3E6", radius=False, lw=0.6) - tri = slide.shapes.add_shape(MSO_SHAPE.ISOSCELES_TRIANGLE, Inches(x(left + width * 0.42)), Inches(y(top + height * 0.30)), Inches(w(width * 0.22)), Inches(h(height * 0.40))) - tri.rotation = 90 - tri.fill.solid() - tri.fill.fore_color.rgb = rgb("#EAF2FF") - tri.line.color.rgb = rgb("#1B2F4A") - return box - - -def add_audio_wave(slide, cx, cy, width, height, *, color="#003E9C"): - bars = 15 - for i in range(bars): - frac = i / (bars - 1) - amp = (0.25 + 0.75 * abs(0.5 - abs(frac - 0.5)) * 2) * height - x0 = cx - width / 2 + frac * width - add_line(slide, x0, cy - amp / 2, x0, cy + amp / 2, color=color, width=1.1) - - -def add_doc_icon(slide, left, top, width, height): - add_box(slide, left, top, width, height, line="#1B1B1B", fill="#FFFFFF", radius=False, lw=1.1) - for i in range(4): - add_line(slide, left + 10, top + 15 + i * 10, left + width - 8, top + 15 + i * 10, color="#222222", width=0.8) - - -def add_clock(slide, cx, cy, r): - oval = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(cx - r)), Inches(y(cy - r)), Inches(w(r * 2)), Inches(h(r * 2))) - oval.fill.solid() - oval.fill.fore_color.rgb = rgb("#F7FBFF") - oval.line.color.rgb = rgb("#003A89") - oval.line.width = Pt(3) - add_line(slide, cx, cy, cx, cy - r * 0.58, color="#003A89", width=3) - add_line(slide, cx, cy, cx + r * 0.42, cy + r * 0.35, color="#003A89", width=3) - - -def build_deck(asset_mode: str = "api", asset_workers: int = 4, force_generate_assets: bool = False) -> tuple[Path, dict]: - OUT.mkdir(parents=True, exist_ok=True) - prs = Presentation() - prs.slide_width = Inches(SLIDE_W) - prs.slide_height = Inches(SLIDE_H) - slide = prs.slides.add_slide(prs.slide_layouts[6]) - bg = slide.background.fill - bg.solid() - bg.fore_color.rgb = rgb("#FFFFFF") - - assets, asset_report = prepare_visual_assets(asset_mode, asset_workers, force_generate_assets) - - blue = "#00338D" - teal = "#007A83" - orange = "#C65A00" - purple = "#6E55B8" - green = "#2D7D3A" - - add_panel(slide, 8, 43, 240, 646, "1. Virtual Interview\nSetup", blue) - add_picture_contain(slide, assets["setup_top"], 28, 118, 205, 162) - add_text(slide, "Participant in front of\n49-inch screen", 42, 286, 170, 48, size=8.8, bold=True) - add_picture_contain(slide, assets["setup_mid"], 30, 342, 210, 93) - add_text(slide, "3D virtual interviewer\non screen", 43, 451, 165, 44, size=8.8, bold=True) - add_picture_contain(slide, assets["setup_cam"], 22, 500, 205, 128) - add_text(slide, "Wide-angle camera\ncapturing full body", 44, 634, 170, 38, size=8.4, bold=True) - - add_panel(slide, 295, 168, 208, 429, "2. Raw Video:\n287 participants\n× 36 questions", blue) - add_film_icon(slide, 327, 292, 149, 112, fill="#2E6FA7") - for row, yy in enumerate([430, 506]): - for col, xx in enumerate([316, 369, 424]): - if row == 1 and col == 2: - continue - add_picture_contain(slide, assets["raw_people"], xx, yy, 49, 61) - add_text(slide, "…", 446, 500, 40, 50, size=20, bold=True) - - add_panel(slide, 570, 43, 382, 672, "3. Preprocessing Modules", blue) - module_specs = [ - (580, 96, 360, 100, "#0E60B8", "FFmpeg", "Audio"), - (580, 219, 360, 100, "#2D7D3A", "FunASR", "Spoken Text\n+ Timestamps"), - (580, 343, 360, 100, "#6E55B8", "MTCNN", "Face Clips"), - (580, 467, 360, 100, "#00879C", "AlphaPose", "Full-body\nPose\nSkeletons"), - (580, 590, 360, 100, "#C65A00", "Frame\nSampling", "Video\nFrames"), - ] - for left, top, width, height, color, method, out_label in module_specs: - add_box(slide, left, top, width, height, line=color, fill="#F9FCFF", radius=True, lw=1.2) - add_text(slide, method, left + 78, top + 30, 95, 42, size=10.4, color=color, bold=True) - add_arrow(slide, left + 172, top + 50, left + 215, top + 50, color="#000000", width=1.5) - out_left = left + 296 if method == "FFmpeg" else left + 260 - out_width = 55 if method == "FFmpeg" else 92 - add_text(slide, out_label, out_left, top + 28, out_width, 45, size=9.2 if method == "FFmpeg" else 9.5, bold=True) - add_film_icon(slide, 596, 116, 63, 53, fill="#294C72") - add_audio_wave(slide, 835, 146, 72, 42, color="#003E9C") - mic = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(606)), Inches(y(239)), Inches(w(21)), Inches(h(48))) - mic.fill.solid(); mic.fill.fore_color.rgb = rgb("#1B4F82"); mic.line.color.rgb = rgb("#1B1B1B") - add_line(slide, 616, 286, 616, 303, color="#1B1B1B", width=2) - add_line(slide, 600, 303, 632, 303, color="#1B1B1B", width=2) - add_doc_icon(slide, 780, 240, 45, 58) - add_picture_contain(slide, assets["face"], 599, 369, 53, 51) - add_picture_contain(slide, assets["face"], 782, 359, 59, 61) - add_picture_contain(slide, assets["pose"], 596, 481, 55, 70) - add_picture_contain(slide, assets["pose"], 788, 481, 55, 70) - for i in range(3): - add_box(slide, 594 + i * 8, 613 - i * 5, 47, 52, line="#333333", fill="#FFFFFF", radius=False, lw=0.7) - add_picture_contain(slide, assets["person_frame"], 782, 604, 65, 70) - - add_panel(slide, 996, 184, 149, 436, "4. Timestamp\nAlignment", blue) - add_clock(slide, 1071, 350, 45) - add_text(slide, "Align audio,\ntext, face,\nframes, and\npose streams\nto a unified\ntimeline", 1020, 433, 104, 150, size=9.2, bold=True) - - add_panel(slide, 1176, 132, 186, 560, "5. Five Modalities", teal) - modality_rows = [ - (1186, 186, purple, "Face", assets["face"]), - (1186, 293, orange, "Frame", assets["person_frame"]), - (1186, 395, teal, "Pose", assets["pose"]), - (1186, 497, "#2076D0", "Audio", None), - (1186, 600, green, "Text", None), - ] - for left, top, color, label, asset in modality_rows: - add_box(slide, left, top, 168, 87, line=color, fill="#F8FBFF", radius=True, lw=1.1) - if asset: - add_picture_contain(slide, asset, left + 18, top + 12, 70, 58) - elif label == "Audio": - add_audio_wave(slide, left + 48, top + 45, 78, 42, color="#003E9C") - else: - add_doc_icon(slide, left + 29, top + 15, 43, 57) - add_text(slide, label, left + 105, top + 27, 55, 36, size=10.5, color=color, bold=True) - - add_panel(slide, 1394, 132, 184, 560, "6. NEO-FFI-3\nSelf-report Labels", orange) - clip = add_box(slide, 1445, 219, 75, 99, line="#0B3264", fill="#F7FBFF", radius=True, lw=1.2) - add_box(slide, 1470, 217, 25, 10, line="#0B3264", fill="#0B3264", radius=True, lw=1.0) - for i in range(4): - add_text(slide, "✓", 1456, 245 + i * 19, 16, 14, size=10, color="#0B3264", bold=True) - add_line(slide, 1476, 253 + i * 19, 1513, 253 + i * 19, color="#222222", width=1.2) - add_text(slide, "OCEAN Scores", 1424, 345, 115, 24, size=10, bold=True) - ocean = [("O", "Openness", purple), ("C", "Conscientiousness", "#D66300"), ("E", "Extraversion", "#00919B"), ("A", "Agreeableness", "#4B934A"), ("N", "Neuroticism", "#3C82C8")] - for i, (letter, name, color) in enumerate(ocean): - cy = 403 + i * 55 - oval = slide.shapes.add_shape(MSO_SHAPE.OVAL, Inches(x(1410)), Inches(y(cy - 17)), Inches(w(30)), Inches(h(35))) - oval.fill.solid(); oval.fill.fore_color.rgb = rgb(color); oval.line.color.rgb = rgb(color) - add_text(slide, letter, 1413, cy - 12, 24, 24, size=11, color="#FFFFFF", bold=True) - add_text(slide, name, 1448, cy - 12, 120, 26, size=8.8, bold=True, align=PP_ALIGN.LEFT) - - add_panel(slide, 1612, 235, 155, 397, "7. Multimodal\nPersonality\nDataset", teal) - add_picture_contain(slide, assets["dataset"], 1635, 344, 105, 95) - add_text(slide, "Multimodal\nvideos aligned\nacross five\nmodalities with\nNEO-FFI-3\n(OCEAN) labels", 1630, 470, 120, 132, size=8.7, bold=True) - - # Macro arrows. - for args in [ - (248, 384, 294, 384), (503, 384, 570, 384), (940, 268, 996, 268), (940, 389, 996, 389), - (940, 545, 996, 545), (1145, 386, 1176, 386), (1362, 233, 1394, 233), (1362, 335, 1394, 335), - (1362, 441, 1394, 441), (1362, 543, 1394, 543), (1362, 645, 1394, 645), (1578, 386, 1612, 386) - ]: - add_arrow(slide, *args, color="#000000", width=1.8) - # Bracket-like incoming fan from raw video to preprocessing rows. - add_line(slide, 536, 140, 536, 685, color="#000000", width=1.3) - for yy in [140, 260, 384, 512, 685]: - add_arrow(slide, 536, yy, 570, yy, color="#000000", width=1.3) - - # Legend. - add_box(slide, 19, 762, 918, 84, line="#18345C", fill="#FFFFFF", radius=True, lw=1.0, dash=True) - legend = [ - ("Video", "film"), ("Audio", "audio"), ("Text", "doc"), ("Face", "face"), ("Pose", "pose"), ("Frames", "frames") - ] - lx = 44 - for label, kind in legend: - if kind == "film": - add_film_icon(slide, lx, 784, 48, 43, fill="#294C72") - elif kind == "audio": - add_audio_wave(slide, lx + 23, 805, 48, 34, color="#003E9C") - elif kind == "doc": - add_doc_icon(slide, lx, 781, 32, 47) - elif kind == "face": - add_picture_contain(slide, assets["face"], lx, 780, 48, 49) - elif kind == "pose": - add_picture_contain(slide, assets["pose"], lx, 777, 48, 55) - else: - for i in range(3): - add_box(slide, lx + i * 6, 781 - i * 3, 40, 47, line="#333333", fill="#FFFFFF", radius=False, lw=0.7) - add_picture_contain(slide, assets["person_frame"], lx + 12, 786, 35, 39) - add_text(slide, label, lx + 62, 790, 70, 32, size=9.4, bold=True, align=PP_ALIGN.LEFT) - lx += 145 - - pptx = OUT / "editable_composition.pptx" - prs.save(pptx) - return pptx, asset_report - - -def export_preview(pptx_path: Path) -> None: - try: - import win32com.client # type: ignore - except Exception: - return - app = win32com.client.Dispatch("PowerPoint.Application") - presentation = app.Presentations.Open(str(pptx_path), WithWindow=False) - try: - presentation.Slides(1).Export(str(OUT / "rebuild_preview.png"), "PNG", REF_W, REF_H) - finally: - presentation.Close() - app.Quit() - - -def write_reports(pptx_path: Path, asset_report: dict) -> None: - report = { - "summary": "Editable PPT reconstruction of the virtual interview multimodal personality dataset pipeline.", - "reference": str(REFERENCE), - "reference_size_px": [REF_W, REF_H], - "slide_size_in": [SLIDE_W, SLIDE_H], - "editable_layers": ["panel frames", "labels", "method names", "arrows", "legend", "simple icons"], - "raster_assets": sorted(path.name for path in ASSET_DIR.glob("*.png")), - "asset_policy": asset_report.get("asset_mode"), - "api_requests_attempted": asset_report.get("api_requests_attempted"), - "output_pptx": str(pptx_path), - } - (OUT / "figure_program.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "asset_generation_report.json").write_text(json.dumps(asset_report, indent=2, ensure_ascii=False), encoding="utf-8") - (OUT / "rebuild_notes.md").write_text( - "# Summary\n" - "This rebuild uses editable PowerPoint shapes for the pipeline structure, text, arrows, labels, and legend. " - "Pictorial interview and modality examples are inserted as slot-level raster assets generated from local reference crops when API mode is enabled.\n", - encoding="utf-8", - ) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Rebuild the virtual interview pipeline as an editable PPTX.") - parser.add_argument("--asset-mode", choices=["api", "crop"], default="api", help="Use API-generated slot assets or direct reference crops.") - parser.add_argument("--asset-workers", type=int, default=4, help="Parallel workers for API asset generation.") - parser.add_argument("--force-generate-assets", action="store_true", help="Regenerate API assets even if existing assets are present.") - args = parser.parse_args() - pptx, asset_report = build_deck(args.asset_mode, args.asset_workers, args.force_generate_assets) - write_reports(pptx, asset_report) - export_preview(pptx) - - -if __name__ == "__main__": - main() diff --git a/experiments/literature_review/README.md b/experiments/literature_review/README.md index a0f8eb0..c48bc32 100644 --- a/experiments/literature_review/README.md +++ b/experiments/literature_review/README.md @@ -1,8 +1,8 @@ # Literature review archive -This directory contains early research notes and extracted paper text used to understand related diagram-generation systems. +This directory contains early research notes used to understand related diagram-generation systems. -- `raw_text/` contains archived text extracts for internal literature analysis. -- `scripts/` contains historical extraction and translation helpers. Some scripts preserve machine-specific paths and are not production utilities. +- Raw paper text extracts are intentionally not tracked in Git to avoid shipping large or copyrighted source material. +- Historical extraction and translation helpers were removed from the public repository because they depended on machine-specific local paths and were not production utilities. Nothing under this directory may be imported by `rfs/`, the Codex plugin skill, or automated production workflows. diff --git a/experiments/literature_review/raw_text/diagrammergpt.txt b/experiments/literature_review/raw_text/diagrammergpt.txt deleted file mode 100644 index a735262..0000000 --- a/experiments/literature_review/raw_text/diagrammergpt.txt +++ /dev/null @@ -1,1644 +0,0 @@ - - ---- PAGE 1 --- - -DiagrammerGPT: Generating Open-Domain, Open-Platform Diagrams via LLM Planning - -Over the past few years, text-to-image (T2I) generation models (Rombach et al., 2021; -Ramesh et al., 2022; Yu et al., 2022; Chang et al., 2023; Dai et al., 2023) have shown impressive advancements in image generation quality. Large language models (LLMs) have also recently shown strong capabilities and usefulness in broad language understanding and generation tasks (Touvron et al., 2023a;b; OpenAI, 2023b; Chung et al., 2022; Brown et al., 2020; Chowdhery et al., 2022). Recent works also have demonstrated that it is possible to leverage LLMs to control layouts for the downstream T2I models, for better semantic alignment with input text prompts (Cho et al., 2023b; Feng et al., 2023; Lian et al., 2023). - -However, it has not been explored to use the combination of LLM and T2I generation frameworks for creating diagrams. A diagram is a symbolic/schematic representation that explains information using rich and spatially complex visualizations (e.g., a dense combination of objects, text labels, arrows, lines, etc.). A system that helps to create accurate diagrams would be useful for preparing many educational and academic resources (e.g., creating a diagram that explains new concepts in books, presentations, and papers). While the existing T2I generation models are good at generating realistic images, they often fail at diagram generation because they lack fine-grained object layout control when many objects are densely connected via complex relations such as arrows/lines and also often fail to render comprehensible text labels. In diagram generation, it is more important to - -Published as a conference paper at COLM 2024 - -Abhay Zala Han Lin Jaemin Cho Mohit Bansal UNC Chapel Hill {aszala, hanlincs, jmincho, mbansal}@cs.unc.edu - -https://diagrammerGPT.github.io - -arXiv:2310.12128v2 [cs.CV] 15 Jul 2024 - -1 Introduction - -1 - -Abstract - -Text-to-image (T2I) generation has seen significant growth over the past few years. Despite this, there has been little work on generating diagrams with T2I models. A diagram is a symbolic/schematic representation that explains information using structurally rich and spatially complex visualizations (e.g., a dense combination of related objects, text labels, directional arrows/lines, etc.). Existing state-of-the-art T2I models often fail at diagram generation because they lack fine-grained object layout control when many objects are densely connected via complex relations such as arrows/lines, and also often fail to render comprehensible text labels. To address this gap, we present DiagrammerGPT, a novel two-stage text-to-diagram generation framework leveraging the layout guidance capabilities of LLMs to generate more accurate diagrams. In the first stage, we use LLMs to generate and iteratively refine ‘diagram plans’ (in a planner-auditor feedback loop). In the second stage, we use a diagram generator, DiagramGLIGEN, and a text label rendering module to generate diagrams (with clear text labels) following the diagram plans. To benchmark the text-to-diagram generation task, we introduce AI2D-Caption, a densely annotated diagram dataset built on top of the AI2D dataset. We show that our DiagrammerGPT framework produces more accurate diagrams, outperforming existing T2I models. We also provide comprehensive analysis, including open-domain diagram generation, multi-platform vector graphic diagram generation, human-in-the-loop editing, and multimodal planner/auditor LLMs. - ---- PAGE 2 --- - -Figure 1: An overview of DiagrammerGPT, our two-stage framework for open-domain diagram generation. In the first diagram planning stage (Sec. 3.1), given a prompt, our LLM (GPT-4 (OpenAI, 2023b)) generates a diagram plan, which consists of dense entities, fine-grained relationships, and precise layouts. Then, the LLM iteratively refines the plan to correct mistakes. In the second diagram generation stage (Sec. 3.2), our DiagramGLIGEN outputs the diagram given the diagram plan, then, we render the text labels on the diagram. - -convey correct information (with correct object relationships) than to generate photorealistic objects. In our experiments, existing T2I generation models usually generate diagrams where objects and arrows/lines have incorrect relationships and rendered text labels are incomprehensible. Even the recent state-of-the-art (SOTA) model DALL-E 3 (OpenAI, 2023a) struggles to render accurate diagrams (mention in their system card and shown in Fig. 6). - -To address the issues in text-to-diagram generation, we introduce DiagrammerGPT, a novel two-stage framework capable of generating more accurate open-domain diagrams for multiple platforms by following LLM-generated diagram plans. As shown in Fig. 1, our DiagrammerGPT splits the text-to-diagram generation task into two stages: diagram planning and diagram generation. For the first stage (Fig. 1 left), we employ an LLM to create and refine ‘diagram plans’. In the second stage (Fig. 1 right), we introduce DiagramGLIGEN, a layout-guided diagram generation module, to generate diagrams based on diagram plans, then explicitly render text labels, ensuring their readability. - -In the first stage, diagram planning (Sec. 3.1), we employ an LLM (e.g., GPT-4 (OpenAI, 2023b)) to act as a planner and generate diagram plans given text prompts. The diagram plans consist of (1) a list of entities (i.e., objects and text labels), (2) relationships between the entities (i.e., arrows or lines between entities), and (3) entity layouts (i.e., 2D bounding box coordinates). After the initial generation of diagram plans, inspired by recent works on LLMs that self-refine previously generated contents (Chen et al., 2023c; Miao et al., 2023; -Madaan et al., 2023), we use another LLM to act as an auditor and find potential errors such as incorrect object positions or relationships between objects. Then, the planner LLM takes feedback from the auditor LLM to update the diagram plan (e.g., in Fig. 1, the sun is smaller than the earths, so the plan is adjusted to fix the relative scales). We find that our LLM-generated diagram plans are quite accurate and that refinement can effectively help correct some small errors (see Sec. 5.2 and feedback example in appendix for more details). - -In the second stage, diagram generation (Sec. 3.2), we introduce DiagramGLIGEN, a layoutguided diagram generation module, to generate diagrams based on diagram plans, then explicitly render text labels, ensuring their readability. We implement DiagramGLIGEN based on GLIGEN (Li et al., 2023b) architecture, which adds gated self-attention layers to the Stable Diffusion v1.4 (Rombach et al., 2021) model. While the original GLIGEN model is only trained on natural images and take only objects for layout grounding, DiagramGLIGEN is more specialized in the diagram domain, by being trained on our new AI2D-Caption diagram dataset (see the following paragraph and Sec. 4.1 for more details) and taking the text labels and arrows as additional layout grounding inputs. As the SOTA diffusion models still struggle in rendering text (Liu et al., 2023b; Chen et al., 2023a; Cho et al., 2023b), we explicitly render text labels on top of the generated diagrams to ensure they are readable. -Compared to the T2I baselines, our diagram generation stage allows more accurate object layouts and relationships between the objects (e.g., arrows/lines), and clear text labels. - -Published as a conference paper at COLM 2024 - -A diagram showing the Earth revolve around the sun four times, one of each solstice and equinox. It also … - -Diagram Plan from GPT-4 DiagramGLIGEN Initial Plan Visualization - -Vernal Equinox - -March 22-23 - -Entities: -images [earth (I0), earth (I1), ...] text labels [“Vernal...” (T0), ...] ... -Entity Locations: -I0: [39, 11, 17, 21], ... -Entity Relations: -I0 has an arrow to I1; -... - -Sun - -Summer Solstice - -June 21-22 - -Summer Solstice - -Winter Solstice December 21-22 - -June 21-22 - -Autumnal Equinox September 22-23 - -2 - -Diagram Planning Diagram Generation - -with Text Label - -Rendering Refined Plan After Feedback - -Vernal Equinox - -March 22-23 - -Winter Solstice December 21-22 - -Autumnal Equinox September 22-23 - ---- PAGE 3 --- - -Figure 2: Illustration of the first stage of DiagrammerGPT: diagram planning (Sec. 3.1). -We use a planner LLM (e.g., GPT-4 (OpenAI, 2023b)) to create the fine-grained layouts of diagrams, which we call diagram plans. We first generate an initial diagram from the input text prompt with an LLM (left). Then we iteratively refine diagram plans in a feedback loop of the planner and auditor LLMs. - -Since there are no diagram datasets with detailed captions and fine-grained layout annotations, we construct AI2D-Caption, a new dataset for the text-to-diagram generation task built on top of the AI2D (Kembhavi et al., 2016) dataset (Sec. 4.1). We employ LLaVA 1.5 (Liu et al., 2023a), a SOTA multimodal LLM, to create annotations of captions and object descriptions on AI2D diagrams. - -We comprehensively compare our DiagrammerGPT to recent text-to-image/diagram generation methods, including Stable Diffusion v1.4 (Rombach et al., 2021), VPGen (Cho et al., 2023b), and AutomaTikZ (Belouadi et al., 2023), in both zeroshot and fine-tuned settings (see Sec. 4.2 and Sec. 5.1). In our quantitative and qualitative analysis, our DiagrammerGPT demonstrates more accurate diagram generation performance than the baseline models. -Our method also outperforms Stable Diffusion v1.4, the closest and strongest baseline, in a human preference study on both image-text alignment and object relationship criteria. In our error analysis, we show that diagram plans generated by our LLM (GPT-4) are quite accurate (before and after refinement), while DiagramGLIGEN sometimes makes mistakes when generating diagrams. This indicates that our DiagrammerGPT can benefit from future layout-guided image generation backbones stronger than GLIGEN (see Sec. 5.2). - -We also conduct additional analysis, including open-domain diagram generation, multiple platforms vector graphic diagram generation, human-in-the-loop editing, and text-only vs. multimodal LLM (e.g., GPT-4Vision (OpenAI, 2023c); see Sec. 5.4 and appendix). First, we experiment with generating diagrams in unseen domains (e.g., geology and plants) that are not covered in the LLM in-context learning domains (astronomy, biology, engineering), where our DiagrammerGPT could often generate semantically accurate diagram plans and diagrams from these unseen prompts. Second, we experiment with rendering our diagram plans in multiple platforms: Microsoft PowerPoint, Inkscape, and Adobe Illustrator (see Fig. 7). Third, we show that once a diagram plan is exported to another platform, end-users can also manually edit plans to their liking and send them back to our DiagramGLIGEN to generate images using their manually refined plan (i.e., human-in-the-loop refinement of diagram plan). Lastly, we experiment with using the recently released GPT-4Vision (OpenAI, 2023c) instead of text-only GPT-4 for our planner and auditor LLMs (in appendix). - -In the text-to-image (T2I) generation task, models generate an image from a given text prompt. Recently multimodal language models (e.g., Parti (Yu et al., 2022) and MUSE (Chang et al., 2023)) and diffusion models (e.g., Stable Diffusion (Rombach et al., 2021), DALL-E 2 (Ramesh et al., 2022), and Imagen (Saharia et al., 2022)) have gained popularity for this task. -These recent T2I generation models have demonstrated impressive photorealism in their zeroshot image generation capabilities. However, using these models directly for diagram - -Published as a conference paper at COLM 2024 - -Initial Diagram Plan Generation - -Generate a diagram plan for the caption: “A diagram showing the life cycle of a flea, illustrating the different stages from egg to adult.” - -flea life cycle - -Entities: -images [egg (I0), larva (I1), ...] text labels [“egg” (T0), ...] Entity Locations: -I0: [20, 30, 14, 14], ... -Entity Relations: -T0 labels I0; ... - -Planner (GPT-4) - -egg larva pupa adult - -2 Related Works - -2.1 Text-to-Image Generation - -3 - -Diagram Plan Refinement via Iterative Feedback - -flea life cycle - -Planner (GPT-4) - -What’s wrong with the current diagram? - -egg larva pupa adult - -Auditor (GPT-4) - -The diagram should be circular ... should be arrows from one stage to the next ... - -egg - -larva - -adult - -Planner (GPT-4) - -Updated the diagram plan. -What’s wrong with the current diagram? - -flea life cycle - -Update the diagram plan up to Nfeedback iterations … - -pupa - ---- PAGE 4 --- - -Figure 3: Illustration of the second stage of DiagrammerGPT: diagram generation (Sec. 3.2). -We first generate the objects from the diagram plan with DiagramGLIGEN, our layoutguided diagram generation model. Then, we use Pillow to render clear text labels. - -generation is challenging due to the significant distribution gap between diagrams and their training images. Additionally, these models often lack precise control of fine-grained layouts when many objects are densely connected via complex relations and frequently produce illegible text labels, which are essential for the diagram generation task. - -Large language models (LLMs) have demonstrated their usefulness in various language generation tasks (Touvron et al., 2023a;b; OpenAI, 2023b; Chung et al., 2022; Brown et al., 2020; Chowdhery et al., 2022). Recent works also leverage LLMs to control layouts for the downstream image generation models, for better semantic alignment with input text prompts (Cho et al., 2023b; Feng et al., 2023; Lian et al., 2023). However, these works focus on generating natural images and do not have capabilities that are crucial in diagram generations, e.g., rendering clear text labels or the ability to precisely control fine-grained layouts of many objects that are densely connected via complex relations such as arrows/lines, as shown in our experiments (Sec. 5). A concurrent work, AutomaTikZ (Belouadi et al., 2023) uses LLMs to generate TikZ (Tantau, 2007)1 code to produce scientific vector graphics. While TikZ can be used to draw specific types of diagrams, such as 2D bar plots or directional acyclic graphs, it is difficult to generate diagrams including entities not supported by the TikZ primitives, such as animals. In contrast, our DiagrammerGPT allows for generating diagrams including diverse entities (e.g., different animals in life cycle diagrams or different planets/stars in astronomy diagrams). - -We introduce DiagrammerGPT, a novel two-stage framework for generating open-domain diagrams from text prompts, where an LLM first generates the overall plan, and a visual generator renders an actual diagram following the plan. In the first stage, Diagram Planning (Sec. 3.1), a planner LLM takes a text description of a diagram as input and generates a diagram plan, an overall diagram layout that guides the downstream diagram generation module. The planner LLM generates the initial diagram plan and iteratively refines the diagram plan via feedback from an auditor LLM. In the second Diagram Generation stage (Sec. 3.2), DiagramGLIGEN, our new layout-guided diagram generation module, takes the diagram plan and generates the diagram. Finally, we render the text labels (which are from the diagram plan) onto the diagram to ensure clear and easy-to-read labels for each entity. - -As illustrated in the left part of Fig. 2, we use an LLM (e.g., GPT-4 (OpenAI, 2023b)) to create the overall layouts for diagrams, which we call diagram plans (see appendix for detailed plan configuration and example). - -1A TeX package for generating graphics by composing primitive objects with basic polygons. - -Published as a conference paper at COLM 2024 - -Diagram plan DiagramGLIGEN - -Visual - -Self-Attn - -Object Descriptions - -Sun MercuryVenusEarth Mars Jupiter Saturn Uranus Neptune - -Gated Self-Attn - -Diagram - -Cross-Attn - -Caption - -2.2 Text-to-Image Generation with LLM-Guided Layouts - -3 DiagrammerGPT: Method Details - -3.1 Stage 1: Diagram Planning - -4 - -Generated diagram Generated diagram - -with text labels - -Text label rendering (with Pillow) - -Sun Mercury Venus Earth Mars Jupiter Saturn Uranus Neptune - ---- PAGE 5 --- - -Initial diagram plan generation. We generate diagram plans with GPT-4 via 10 in-context learning examples. Plans consist of three components: (1) entities - a dense list of objects; (2) relationships - complex relationships between entities; and (3) layouts - 2D bounding boxes of the entities. See appendix for diagram plan generation prompts/extra details. - -Diagram plan refinement via iterative feedback. Although the planner (GPT-4) generates fairly accurate initial diagram plans, we can further refine it to account for potentially missing or improperly arranged entities. To address this issue, we introduce an auditor LLM that checks for any mismatch between the current diagram plan and the input prompt. It then provides feedback, enabling the planner LLM to refine the diagram plans. Our auditor and planner LLMs form a feedback loop to iteratively refine the diagram plans. For this auditor LLM, we employ GPT-4 again but with a different preamble and in-context examples designed to give useful feedback (see appendix for the prompts to initialize the auditor LLM). We repeat the feedback loop for up to N iterations, using N= 4 in our experiments. -The right part of Fig. 2 exemplifies the feedback process. - -As shown in Fig. 3, we first generate the diagram images following the diagram plan with DiagramGLIGEN, then render text labels on the diagram. - -DiagramGLIGEN: Layout-guided diagram generation. Conveying factual information is more crucial than drawing photorealistic objects in diagram generation. In our experiments, we observe that existing T2I models often omit important objects, and generate incorrect object relationships and unreadable text labels (see Sec. 5 and Fig. 5). Therefore, we introduce DiagramGLIGEN, a layout-guided text-to-diagram generation model to tackle these issues. -Inspired by GLIGEN (Li et al., 2023b), we implement DiagramGLIGEN by incorporating gated self-attention layers, which take layout grounding inputs, into Stable Diffusion v1.4. -Furthermore, we enhance layout control by incorporating text labels and relationships as part of the layout grounding inputs during training, which can reduce the generation of unreadable text and redundant arrows/lines during inference (by not giving text labels during inference). We employ CLIP (Radford et al., 2021) text encoder to encode object descriptions and their relationships. We use the CLIP image encoder to represent bounding box regions of the text labels in the ground truth diagrams. See the appendix for additional GLIGEN details and training setup. - -Text label rendering. Text labels (e.g., “Sun” labeling the sun object in Fig. 3) in diagrams can effectively assist readers in understanding new concepts (Johnson et al., 2014). However, as shown in Cho et al. (Cho et al., 2023b), existing T2I generation models (including Stable Diffusion v1.4) still struggle to generate high-quality text labels. Therefore, we explicitly render clear text labels on the diagrams with the Pillow Python package (Clark, 2015). - -In the following subsections, we introduce our AI2D-Caption dataset for the text-to-diagram generation task (Sec. 4.1), baseline models (Sec. 4.2), evaluation metrics (Sec. 4.3), and human evaluation setups (Sec. 4.4). - -We introduce the AI2D-Caption dataset for the text-to-diagram generation task. AI2D- Caption is built on top of AI2 Diagrams (AI2D) dataset (Kembhavi et al., 2016), which provides annotations of around 4.9K diagrams covering diverse scientific domains, from Grade 1-6 science textbooks. The original AI2D dataset’s annotations are very short / are missing some aspects (i.e., each object bounding box is labeled simply as ‘blob’), so we employ LLaVA 1.5 (Liu et al., 2023a), a SOTA multimodal language model, to generate detailed captions for the diagrams and region descriptions of each bounding box (see Fig. 4 for comparison of AI2D and AI2D-Caption and the appendix for implementation details). - -Published as a conference paper at COLM 2024 - -3.2 Stage 2: Diagram Generation - -4 Experimental Setup - -4.1 AI2D-Caption Dataset - -5 - ---- PAGE 6 --- - -Figure 4: Example diagram annotation from the AI2D dataset (Kembhavi et al., 2016) (left) and our AI2D-Caption (right). AI2D-Caption additionally provides annotations of the diagram caption and bounding box region descriptions. - -We use all of the LLaVA 1.5-aided caption and region annotations for DiagramGLIGEN training and baseline fine-tuning. - -To ensure that the LLM in-context examples and evaluation are accurate, we manually annotate diagram plans (i.e., captions, object/text label bounding boxes, object descriptions, arrows) for randomly selected 105 diagrams (held out from the training set described above). -Among the 105 diagrams, we use 30 diagrams that cover diverse scientific domains (10 for astronomy, 10 for biology, and 10 for engineering) as in-context examples, and 75 diagrams (25 for astronomy, 25 for biology, and 25 for engineering) as a test split. - -We compare our DiagrammerGPT to several baseline models, including Stable Diffusion v1.4 (Rombach et al., 2021), VPGen (Vicuna13B (Chiang et al., 2023) + GLIGEN) (Cho et al., 2023b), and AutomaTikZ (Belouadi et al., 2023) (CLIMA-13B). For the baselines, we experiment with zero-shot and fine-tuned (on our AI2D-Caption dataset where applicable) diagram generation. For VPGen, we fine-tune both the Vicuna13B and GLIGEN. - -VPEval (objects, counts, relationships, texts). The VPEval metric (Cho et al., 2023b) evaluates diagrams in terms of the presence of objects (object evaluation), the number of objects (count evaluation), the correctness of spatial and connection relationships (relationship evaluation), and the presence of correct text labels (text evaluation). While the original VPEval uses BLIP-2 (Li et al., 2023a) as its visual reasoning model, we employ the recently released LLaVA 1.5 (Liu et al., 2023a) model as our visual reasoning model, as we find it is more faithful in diagram question answering in our initial experiments and is an overall stronger model than BLIP-2 (Liu et al., 2023a). See appendix for setup details. - -Captioning. In line with previous works (Hong et al., 2018; Hinz et al., 2020; Cho et al., 2023a), we also use captioning as a way to determine the accuracy of the generated diagrams. -We use LLaVA 1.5 (Liu et al., 2023a) to caption each generated diagram and then compare this generated caption with the ground-truth caption using CIDEr (Vedantam et al., 2014) and BERTScore (Zhang et al., 2020). - -CLIPScore. Following previous works (Cho et al., 2023a; Saharia et al., 2022; Belouadi et al., 2023), we use CLIPScore (Hessel et al., 2021) to measure the similarity between the generated diagram and the original caption/ground-truth diagram. Concretely, we calculate two types of CLIPScore: (1) CLIPScoreImg-Txt: similarity between the generated diagram and - -Published as a conference paper at COLM 2024 - -electrical circuit!!! -electrical circuit!!! - -B0 - -B2 - -B1 - -4.2 Baseline Models - -4.3 Evaluation Metrics - -6 - -AI2D AI2D-Caption - -Light Bulb - -Switch - -Battery - -A diagram showing an electrical circuit with a light bulb, a battery, and a switch. -(No Caption) - ---- PAGE 7 --- - -Methods VPEval (%) ↑ Captioning ↑ CLIPScore ↑ - -Zeroshot Stable Diffusion v1.4 63.1 27.8 0.0 79.3 39.0 7.7 87.5 27.3 65.3 VPGen 55.8 32.9 0.0 72.8 39.3 6.1 87.2 25.6 61.7 AutomaTikZ 24.9 34.2 5.5 67.7 31.0 12.2 86.9 24.7 64.5 - -Fine-tuned Stable Diffusion v1.4 69.8 35.4 0.0 81.9 45.1 18.2 88.5 30.1 68.1 VPGen 62.8 27.8 0.0 76.3 39.7 4.2 86.9 26.4 61.9 - -DiagrammerGPT (Ours) 86.4 57.0 47.5 87.9 71.2 26.4 89.4 32.1 73.9 - -Table 1: Comparison of DiagrammerGPT to existing text-to-image generation baseline models. On all metrics, DiagrammerGPT outperforms the baselines, indicating that our method is more effective for generating accurate diagrams. - -the ground-truth caption. (2) CLIPScoreImg-Img: similarity between the generated diagram and the ground-truth diagram. We use OpenAI CLIP ViT-L/14 (Radford et al., 2021). - -Human error analysis of the two stages. As our DiagrammerGPT pipeline consists of two stages, it is important to understand where any errors in our pipeline come from. In both stages, we assess (1) Object Presence: whether all required objects for the diagrams are present, and (2) Object Relationships: if the objects have proper relationships to each other (e.g., for a lunar eclipse, the earth should be between the sun and the moon). We have an expert rate both stages on a Likert scale from 1 to 5 for 25 layouts/images. - -Pairwise preference: Image-text alignment & object relationships. We conduct a human analysis comparing our DiagrammerGPT framework to Stable Diffusion v1.4 on 50 diagrams. We choose Stable Diffusion v1.4 fine-tuned on AI2D-Caption for comparison because it is the closest baseline to our DiagrammerGPT and also shows the strongest results among the baselines (see Sec. 5.1). We ask crowd-sourced annotators (20 unique annotators) from Amazon Mechanical Turk2 to evaluate the diagrams generated for each prompt (see appendix for setup details). - -We show our primary quantitative results (Sec. 5.1), human evaluation on pairwise preference study and error analysis (Sec. 5.2), qualitative analysis (Sec. 5.3), additional analysis about open-domain generation, multi-platform vector graphic diagram generation, humanin-the-loop diagram plan editing, (Sec. 5.4), and an ablation of different ablations for diagram plan generation (Sec. 5.5). See appendix for more analysis on DiagramGLIGEN setups and multimodal planner/auditor LLMs. - -VPEval. Table 1 left block shows the VPEval results. For both Stable Diffusion v1.4 and VPGen baselines, fine-tuning improves the score for object skill (e.g., 63.1 →69.8 for Stable Diffusion v1.4, and 55.8 →62.8 for VPGen), but for VPGen count, it decreases scores (32.9 →27.8). For relationships, both models improve (79.3 →81.9 for Stable Diffusion v1.4, and 72.8 →76.3 for VPGen). For text, both models achieve 0 scores before and after fine-tuning. -Our DiagrammerGPT outperforms both zeroshot and fine-tuned baselines on both overall and skill-specific VPEval scores, showcasing the strong layout control, object relationship representation, and accurate text rendering capability of our diagram generation framework. - -Captioning and CLIPScore. Table 1 middle block shows captioning scores (with LLaVA 1.5), and the right block shows CLIPScore (with CLIP-ViT L/14). Our DiagrammerGPT - -Published as a conference paper at COLM 2024 - -4.4 Human Evaluation - -5 Results and Discussion - -5.1 Quantitative Results - -2https://www.mturk.com - -7 - -Object Count Text Relationships Overall CIDEr BERTScore Img-Txt Img-Img - ---- PAGE 8 --- - -Objects Presence (↑) Object Relations (↑) Objects Presence (↑) Object Relations (↑) Objects Presence (↑) Object Relations (↑) - -Table 2: Step-wise error analysis of DiagrammerGPT on 25 AI2D-Caption test prompts. We use a Likert scale (1-5) to evaluate object presence and object relations of the diagram plan (before and after refinement) and the final generated diagram. - -Figure 5: Example diagram generation results from baselines (fine-tuned Stable Diffusion v1.4 and AutomaTikZ) and our DiagrammerGPT on the AI2D-Caption test split. Our DiagrammerGPT correctly follows the caption while the baselines make several errors. - -outperforms both the zeroshot and fine-tuned baselines, indicating our generated diagrams have more relevant information to the input prompt (which is a critical aspect of diagrams), have better image-text alignment, and more closely resemble the ground-truth diagrams (image-image) than the baselines. Our DiagrammerGPT significantly outperforms both fine-tuned VPGen (26.4 vs. 4.2) and fine-tuned Stable Diffusion v1.4 (26.4 vs. 18.2) for CIDEr and also achieves a few higher points on BERTScore. For both CLIPScore’s, DiagrammerGPT has improvement over fine-tuned Stable Diffusion v1.4 (32.1 vs. 30.1 and 73.9 vs. 68.1). - -Step-by-step error analysis. Table 2 shows that our diagram plans exhibit high scores on both object presence (4.96) and object relationship (4.56) even before the refinement, and refinement increases the object relationship scores even further (4.56 →4.72) by adjusting the entity layouts from the initial diagram plan (see appendix for refinement examples). -During the diagram generation stage, the object presence and relationship scores decrease. -This indicates that our DiagrammerGPT could generate more accurate diagrams, once we have access to a layout-guided image generation backbone model stronger than Stable Diffusion v1.4 architecture. - -Human preference. We conduct a human preference study, comparing our DiagrammerGPT and fine-tuned Stable Diffusion v1.4 (SD v1.4) in imagetext alignment and object relationships. As shown in Table 3, our DiagrammerGPT achieves a higher preference than Stable Diffusion v1.4 in both image-text alignment (68% vs 24%) and object relationships (62% vs 34%) criteria. - -Comparison with baselines. Fig. 5 shows example diagrams generated by the baselines (Stable Diffusion v1.4 and AutomaTikZ) and our DiagrammerGPT (both diagram plan and final generation diagram) on the AI2D-Caption test split (see appendix for additional example). Our diagram plan strongly reflects the prompt, and the final diagram is more aligned with the input prompt than the baselines. In the Fig. 5 example, our diagram - -Published as a conference paper at COLM 2024 - -Initial diagram plan Diagram plan after refinement Final Diagram - -4.96 4.56 4.96 4.72 2.96 3.36 - -Fine-tuned SD v1.4 AutomaTikZ - -Input Prompt - -A diagram showing the Earth's position in four - -phases as it revolves - -around the sun. - -5.2 Human Evaluation - -5.3 Qualitative Analysis - -8 - -Stage 1: Diagram Planning (with GPT-4) Stage 2: Diagram Generation (with DiagramGLIGEN) - -Diagram Plan (Ours) DiagrammerGPT (Ours) - -Summer - -earth - -Spring - -Autumn - -earth - -earth - -sun - -earth - -Winter - -Evaluation category Human Preference (%) ↑ - -DiagrammerGPT SD v1.4 Tie - -Image-Text Alignment 68 24 8 Object Relationships 62 34 4 - -Table 3: -Human preference study on generated diagrams: DiagrammerGPT vs. SD v1.4. - ---- PAGE 9 --- - -Input Prompt DiagrammerGPT (Ours) - -Figure 6: Open-domain diagram generation examples with DALL-E 3 and DiagrammerGPT. - -Input Prompt Rendered with Adobe Illustrator - -Figure 7: Examples of vector graphic diagrams generated from diagram plans. - -correctly shows the earth in four phases revolving around the sun. Stable Diffusion v1.4 either over-generates objects in the image (e.g., too many earths), and AutomaTikZ fails to generate a proper diagram. Although our generated diagram plans are generally correct, sometimes DiagramGLIGEN can fail to properly follow all aspects. As noted in Sec. 5.2, our DiagramGLIGEN can improve once a better backbone becomes available. - -Diagram plan refinement. We analyze how our diagram refinement step (see Sec. 3.1) improves the diagram plans. The refinement can fix minor mistakes like missing connections in a circuit or objects being placed in the wrong location. We show examples in the appendix. - -Open-domain diagram generation. We demonstrate that our planner LLM can extend its capabilities to unseen domains beyond the three areas (astronomy/biology/engineering) given in the in-context examples (30 total examples, 10 from each of astronomy/biology/engineering). As shown in Fig. 6, from an input prompt in the earth science domain, our planner LLM generates fairly accurate layouts, and our DiagramGLIGEN can generate a diagram following the layouts. We also compare the recently released DALL-E 3 (OpenAI, 2023a) model and find that it generally produces images with good aesthetic style but tends to generate diagrams with redundant objects (e.g., excessive text descriptions or objects). It also struggles with creating diagrams that adhere to a prompt (e.g., generating incorrect layers in the earth example). The DALL-E 3 system card (OpenAI, 2023a) also notes that DALL-E 3 tends to generate inaccurate information in diagrams. See appendix for more examples. - -Vector graphic diagram generation in different platforms. Although our primary focus is on a pixel-level diagram generation pipeline with DiagramGLIGEN, our diagram plans can also be used for vector graphic diagrams, with multiple platforms like Microsoft PowerPoint, Inkscape, and Adobe Illustrator. Fig. 7 presents an example of vector graphic diagrams. -The example delivers promising results by effectively conveying the crucial information and layouts described in the input text prompts (see additional example in the appendix). -However, it also exhibits certain limitations: (1) inconsistency in icon styles; (2) limited icon retrieval capability. As the diagrams are editable via these same platforms, these limitations can be addressed by end-users editing the diagrams to their liking (see below). - -Human-in-the-loop diagram plan editing. With the diagram plans being rendered in vector graphic platforms, as mentioned above, our DiagrammerGPT can provide an editable dia- - -Published as a conference paper at COLM 2024 - -A diagram showing the - -layers of the earth. It includes the inner and outer cores, the mantle, - -and the crust. - -Rendered with Microsoft PowerPoint Rendered with Inkscape - -A diagram showing two food chains. The left food - -chain, starting from the bottom, goes from lichen, to slug, to toad, to snake, - -to eagle. The right food chain, starting from the bottom, goes from algae, to - -snail, to crayfish, to fish, to - -alligator. - -5.4 Additional Analysis: Open-Domain, Open-Platform, and Human-in-the-Loop - -9 - -DALL•E 3 Diagram Plan (Ours) - -crust - -mantle - -outer core - -inner core - -Inner Core - -Outer Core - -Mantle - -Crust - ---- PAGE 10 --- - -Figure 8: Illustration of human-in-the-loop diagram plan editing. DiagrammerGPT first provides an initial diagram plan with the corresponding generated diagram, users can then review the generated layouts/diagrams and make adjustments based on their needs. - -gram plan, allowing for human-in-the-loop editing. As illustrated in Fig. 8, DiagrammerGPT first generates an initial diagram plan along with the rendered image. Users can then review the generated layouts/diagrams and make adjustments based on their needs/wants (e.g., move the objects, add/remove objects, adjust object sizes, etc.). With the human-refined diagram plan, users can either keep it in vector format and use icons (as mentioned in the previous paragraph) or give it back to DiagramGLIGEN and then create pixel-level diagrams, resulting in diagrams/layouts that are better suited to end-users requirements. - -We also experiment with different LLMs such as GPT-3.5 Turbo (OpenAI, 2023d), GPT-4 (OpenAI, 2023b), LLaMA 3 (8B) (AI@Meta, 2024), and LLaMA2-Chat (13B) (Touvron et al., 2023b). We compare them with ‘layout recall’ – how often each ground-truth object/text label can be found in the predicted diagram plan. We first obtain bipartite matching of objects/text labels between prediction and groundtruth via Hungarian matching algorithm, using BLEU-1 (Papineni et al., 2002) as the match- - -N , where N is the total number of ground-truth object/text labels, objn is nth the ground-truth object/text label, and match outputs 1 if the object/text label is matched in the predicted diagram plan and 0 if not. - -Table 4 shows a comparison of four LLMs: LLaMA3 8B, GPT-3.5 Turbo, GPT-4, and Vicuna 13B (VPGen checkpoint). As indicated by GPT-3.5 Turbo’s and GPT-4’s high ‘overall’ performance (74.7 and 78.4 respectively), both models are capable of generating accurate diagrams; however, as GPT-4 is slightly better in both the ‘object’ and ‘text’ metrics, we use GPT-4 as our main LLM. LLaMA3 and fine-tuned Vicuna are not able to perform well for the task with both getting an overall score below 30. - -In this work, we propose DiagrammerGPT, a novel two-stage text-to-diagram generation framework that leverages the knowledge of LLMs for planning and refining the overall diagram plans. We demonstrate that our DiagrammerGPT achieves more semantically accurate layouts in diagram generation than baseline models in both quantitative and qualitative analysis. In addition, we provide comprehensive human error analysis, ablation studies, and analysis about open-domain diagram generation, vector graphic diagram generation, human-in-the-loop diagram plan editing, and multimodal planner/auditor LLMs. We hope our work can inspire future research on diagram generation. - -Published as a conference paper at COLM 2024 - -A diagram showing the earth, moon, and - -LLM Diagram - -earth sun moon - -Planner - -Moon - -sun with text labels. - -Earth Sun - -Input Prompt Initial Plan Generation - -5.5 Different LLMs for diagram plan generation - -ing score. Then we calculate layout recall metric: ∑N - -6 Conclusion - -10 - -Let me: -(1) align the text labels (2) move all the objects up and make them a bit larger - -moon - -earth sun - -Earth Sun Moon - -Initial Diagram New Diagram - -User Refined Plan - -User Edits - -LLM Layout Recall - -Object Text Overall - -LLaMA3 8B 24.9 32.2 29.2 VPGen (Vicuna 13B) 30.6 0 12.9 GPT-3.5 Turbo 82.5 54.6 74.7 GPT-4 (default) 84.1 60.1 78.4 - -Table 4: Ablation of different LLMs for the planner and auditor. - -n=1 match(objn) - ---- PAGE 11 --- - -We thank the reviewers for the thoughtful discussion and feedback. This work was supported by DARPA ECOLE Program No. HR00112390060, NSF-AI Engage Institute DRL- 2112635, DARPA Machine Commonsense (MCS) Grant N66001-19-2-4031, ARO Award W911NF2110220, ONR Grant N00014-23-1-2356, Accelerate Foundation Models Research program, and a Bloomberg Data Science Ph.D. Fellowship. The views contained in this article are those of the authors and not of the funding agency. - -Jaided AI. Easyocr: Ready-to-use ocr with 80+ supported languages and all popular writing scripts - -including latin, chinese, arabic, devanagari, cyrillic and etc., 2023. URL https://github.com/ JaidedAI/EasyOCR. - -AI@Meta. Llama 3 model card, 2024. URL https://github.com/meta-llama/llama3/blob/main/ - -Jonas Belouadi, Anne Lauscher, and Steffen Eger. Automatikz: Text-guided synthesis of scientific - -vector graphics with tikz. ArXiv, abs/2307.01952, 2023. URL https://api.semanticscholar.org/ CorpusID:263334353. - -Tom B. Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, - -Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel M. Ziegler, Jeffrey Wu, Clemens Winter, Christopher Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. Language Models are Few-Shot Learners. In NeurIPS, 2020. URL http: -//arxiv.org/abs/2005.14165. - -Huiwen Chang, Han Zhang, Jarred Barber, AJ Maschinot, Jose Lezama, Lu Jiang, Ming-Hsuan - -Yang, Kevin Murphy, William T. Freeman, Michael Rubinstein, Yuanzhen Li, and Dilip Krishnan. -Muse: Text-To-Image Generation via Masked Generative Transformers. In ICML, 2023. URL http://arxiv.org/abs/2301.00704. - -Jingye Chen, Yupan Huang, Tengchao Lv, Lei Cui, Qifeng Chen, and Furu Wei. Textdiffuser: Diffusion - -models as text painters. ArXiv, abs/2305.10855, 2023a. URL https://api.semanticscholar.org/ CorpusID:258762187. - -Jingye Chen, Yupan Huang, Tengchao Lv, Lei Cui, Qifeng Chen, and Furu Wei. Textdiffuser-2: - -Unleashing the power of language models for text rendering. arXiv preprint arXiv:2311.16465, 2023b. - -Xinyun Chen, Maxwell Lin, Nathanael Sch¨arli, and Denny Zhou. Teaching large language models - -to self-debug. ArXiv, abs/2304.05128, 2023c. URL https://api.semanticscholar.org/CorpusID: -258059885. - -Wei-Lin Chiang, Zhuohan Li, Zi Lin, Ying Sheng, Zhanghao Wu, Hao Zhang, Lianmin Zheng, Siyuan - -Zhuang, Yonghao Zhuang, Joseph E. Gonzalez, Ion Stoica, and Eric P. Xing. Vicuna: An open-source chatbot impressing gpt-4 with 90%* chatgpt quality, March 2023. URL https://lmsys.org/blog/ 2023-03-30-vicuna/. - -Jaemin Cho, Abhay Zala, and Mohit Bansal. Dall-eval: Probing the reasoning skills and social biases - -Jaemin Cho, Abhay Zala, and Mohit Bansal. Visual programming for text-to-image generation and - -Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, - -Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, Parker Schuh, Kensen Shi, Sasha Tsvyashchenko, Joshua Maynez, Abhishek Rao, Parker Barnes, Yi Tay, Noam M. Shazeer, Vinodkumar Prabhakaran, Emily Reif, Nan Du, Benton C. Hutchinson, Reiner Pope, James Bradbury, Jacob Austin, Michael Isard, Guy Gur-Ari, Pengcheng Yin, Toju Duke, Anselm Levskaya, Sanjay Ghemawat, Sunipa Dev, Henryk Michalewski, Xavier Garc´ıa, Vedant Misra, Kevin Robinson, Liam Fedus, Denny Zhou, Daphne Ippolito, David Luan, Hyeontaek Lim, Barret Zoph, Alexander Spiridonov, Ryan Sepassi, David Dohan, Shivani Agrawal, Mark Omernick, Andrew M. Dai, Thanumalayan Sankaranarayana Pillai, Marie Pellat, Aitor Lewkowycz, Erica Moreira, Rewon - -Published as a conference paper at COLM 2024 - -Acknowledgments - -References - -MODEL_CARD.md. - -of text-to-image generation models. In ICCV, 2023a. - -evaluation. In NeurIPS, 2023b. - -11 - ---- PAGE 12 --- - -Child, Oleksandr Polozov, Katherine Lee, Zongwei Zhou, Xuezhi Wang, Brennan Saeta, Mark D´ıaz, Orhan Firat, Michele Catasta, Jason Wei, Kathleen S. Meier-Hellstern, Douglas Eck, Jeff Dean, Slav Petrov, and Noah Fiedel. Palm: Scaling language modeling with pathways. ArXiv, abs/2204.02311, 2022. URL https://api.semanticscholar.org/CorpusID:247951931. - -Hyung Won Chung, Le Hou, S. Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, - -Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Dasha Valter, Sharan Narang, Gaurav Mishra, Adams Wei Yu, Vincent Zhao, Yanping Huang, Andrew M. Dai, Hongkun Yu, Slav Petrov, Ed Huai hsin Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei. Scaling instruction-finetuned language models. ArXiv, abs/2210.11416, 2022. URL https://api.semanticscholar.org/CorpusID:253018554. - -Alex Clark. Pillow (pil fork) documentation, 2015. URL https://buildmedia.readthedocs.org/ - -Xiaoliang Dai, Ji Hou, Chih-Yao Ma, Sam Tsai, Jialiang Wang, Rui Wang, Peizhao Zhang, Simon - -Vandenhende, Xiaofang Wang, Abhimanyu Dubey, Matthew Yu, Abhishek Kadian, Filip Radenovic, Dhruv Mahajan, Kunpeng Li, Yue Zhao, Vladan Petrovic, Mitesh Kumar Singh, Simran Motwani, Yi Wen, Yiwen Song, Roshan Sumbaly, Vignesh Ramanathan, Zijian He, Peter Vajda, and Devi Parikh. Emu: Enhancing image generation models using photogenic needles in a haystack, 2023. - -Weixi Feng, Wanrong Zhu, Tsu-jui Fu, Varun Jampani, Arjun Akula, Xuehai He, Sugato Basu, Xin Eric - -Wang, and William Yang Wang. Layoutgpt: Compositional visual planning and generation with large language models. NeurIPS, 2023. - -Jack Hessel, Ari Holtzman, Maxwell Forbes, Ronan Le Bras, and Yejin Choi. CLIPScore: a reference-free - -Tobias Hinz, Stefan Heinrich, and Stefan Wermter. Semantic Object Accuracy for Generative Text-to- - -Image Synthesis. IEEE Transactions on Pattern Analysis and Machine Intelligence, pp. 1–1, 2020. ISSN 0162-8828. doi: 10.1109/TPAMI.2020.3021209. URL https://ieeexplore.ieee.org/document/ 9184960/. - -Seunghoon Hong, Dingdong Yang, Jongwook Choi, and Honglak Lee. Inferring Semantic Layout for - -Amy Johnson, Kirsten Butcher, Gamze Ozogul, and Martin Reisslein. Introductory circuit analy- - -sis learning from abstract and contextualized circuit representations: Effects of diagram labels. -Education, IEEE Transactions on, 57:160–168, 08 2014. doi: 10.1109/TE.2013.2284258. - -Aniruddha Kembhavi, Mike Salvato, Eric Kolve, Minjoon Seo, Hannaneh Hajishirzi, and Ali Farhadi. - -A diagram is worth a dozen images. In Bastian Leibe, Jiri Matas, Nicu Sebe, and Max Welling (eds.), ECCV, pp. 235–251, 2016. ISBN 978-3-319-46493-0. - -Junnan Li, Dongxu Li, Silvio Savarese, and Steven C. H. Hoi. Blip-2: Bootstrapping language- - -image pre-training with frozen image encoders and large language models. In ICML, volume abs/2301.12597, 2023a. URL https://api.semanticscholar.org/CorpusID:256390509. - -Yuheng Li, Haotian Liu, Qingyang Wu, Fangzhou Mu, Jianwei Yang, Jianfeng Gao, Chunyuan Li, and - -Yong Jae Lee. Gligen: Open-set grounded text-to-image generation. In CVPR, pp. 22511–22521, 2023b. - -Long Lian, Boyi Li, Adam Yala, and Trevor Darrell. Llm-grounded diffusion: Enhancing prompt - -understanding of text-to-image diffusion models with large language models. arXiv preprint arXiv:2305.13655, 2023. - -Han Lin, Abhay Zala, Jaemin Cho, and Mohit Bansal. Videodirectorgpt: Consistent multi-scene video - -Haotian Liu, Chunyuan Li, Yuheng Li, and Yong Jae Lee. Improved baselines with visual instruction - -Rosanne Liu, Dan Garrette, Chitwan Saharia, William Chan, Adam Roberts, Sharan Narang, Irina - -Blok, Rj Mical, Mohammad Norouzi, and Noah Constant. Character-aware models improve visual text rendering. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 16270–16297, Toronto, Canada, July 2023b. Association for Computational Linguistics. doi: 10.18653/v1/2023.acl-long.900. URL https://aclanthology. -org/2023.acl-long.900. - -Published as a conference paper at COLM 2024 - -media/pdf/pillow/latest/pillow.pdf. - -evaluation metric for image captioning. In EMNLP, 2021. - -Hierarchical Text-to-Image Synthesis. In CVPR, 2018. - -generation via llm-guided planning. arXiv preprint arXiv:2309.15091, 2023. - -tuning, 2023a. - -12 - ---- PAGE 13 --- - -Ilya Loshchilov and Frank Hutter. Fixing weight decay regularization in adam. In International - -Aman Madaan, Niket Tandon, Prakhar Gupta, Skyler Hallinan, Luyu Gao, Sarah Wiegreffe, Uri Alon, - -Nouha Dziri, Shrimai Prabhumoye, Yiming Yang, Sean Welleck, Bodhisattwa Prasad Majumder, Shashank Gupta, Amir Yazdanbakhsh, and Peter Clark. Self-refine: Iterative refinement with self-feedback. ArXiv, abs/2303.17651, 2023. URL https://api.semanticscholar.org/CorpusID: -257900871. - -Ning Miao, Yee Whye Teh, and Tom Rainforth. Selfcheck: Using llms to zero-shot check their own - -step-by-step reasoning. ArXiv, abs/2308.00436, 2023. URL https://api.semanticscholar.org/ CorpusID:260350986. - -OpenAI. Dall-e 3 system card, 2023a. URL https://cdn.openai.com/papers/DALL_E_3_System_ - -OpenAI. Gpt-4 technical report. ArXiv, abs/2303.08774, 2023b. URL https://api.semanticscholar. - -OpenAI. Gpt-4v(ision) system card, 2023c. URL https://cdn.openai.com/papers/GPTV_System_ - -Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. Bleu: a method for automatic - -evaluation of machine translation. In Annual Meeting of the Association for Computational Linguistics, 2002. URL https://api.semanticscholar.org/CorpusID:11080756. - -Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish - -Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, and Ilya Sutskever. Learning transferable visual models from natural language supervision. In ICML, 2021. - -Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, and Mark Chen. Hierarchical Text- - -Conditional Image Generation with CLIP Latents, 2022. URL http://arxiv.org/abs/2204.06125. - -Robin Rombach, A. Blattmann, Dominik Lorenz, Patrick Esser, and Bj¨orn Ommer. High-resolution - -image synthesis with latent diffusion models. 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 10674–10685, 2021. URL https://api.semanticscholar.org/ CorpusID:245335280. - -Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kam- - -yar Seyed Ghasemipour, Burcu Karagol Ayan, S. Sara Mahdavi, Rapha Gontijo Lopes, Tim Salimans, Jonathan Ho, David J Fleet, and Mohammad Norouzi. Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding. In NeurIPS, 2022. URL http://arxiv.org/abs/2205.11487. - -Till Tantau. pgf – a portable graphic format for tex, 2007. URL https://github.com/pgf-tikz/pgf. - -Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth´ee - -Lacroix, Baptiste Rozi`ere, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, and Guillaume Lample. Llama: Open and efficient foundation language models. ArXiv, abs/2302.13971, 2023a. URL https://api.semanticscholar.org/CorpusID: -257219404. - -Hugo Touvron, Louis Martin, Kevin R. Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, - -Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Daniel M. Bikel, Lukas Blecher, Cristian Cant´on Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony S. -Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez, Madian Khabsa, Isabel M. Kloumann, A. V. Korenev, Punit Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushkar Mishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric Michael Smith, R. Subramanian, Xia Tan, Binh Tang, Ross Taylor, Adina Williams, Jian Xiang Kuan, Puxin Xu, Zhengxu Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, and Thomas Scialom. Llama 2: Open foundation and fine-tuned chat models. ArXiv, abs/2307.09288, 2023b. URL https://api.semanticscholar.org/CorpusID:259950998. - -Published as a conference paper at COLM 2024 - -Conference on Learning Representations, 2019. - -Card.pdf. - -org/CorpusID:257532815. - -Card.pdf. - -OpenAI. Chatgpt. https://openai.com/chatgpt, 2023d. - -13 - ---- PAGE 14 --- - -Ramakrishna Vedantam, C. Lawrence Zitnick, and Devi Parikh. Cider: Consensus-based image - -description evaluation. 2015 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. -4566–4575, 2014. URL https://api.semanticscholar.org/CorpusID:9026666. - -Jiahui Yu, Yuanzhong Xu, Jing Yu Koh, Thang Luong, Gunjan Baid, Zirui Wang, Vijay Vasudevan, - -Alexander Ku, Yinfei Yang, Burcu Karagol Ayan, Ben Hutchinson, Wei Han, Zarana Parekh, Xin Li, Han Zhang, Jason Baldridge, and Yonghui Wu. Scaling Autoregressive Models for Content-Rich Text-to-Image Generation. Transactions on Machine Learning Research, 2022. URL http://arxiv. -org/abs/2206.10789. - -Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. Weinberger, and Yoav Artzi. BERTScore: Evaluating - -text generation with BERT. In Proceedings of the International Conference on Learning Representations, 2020. - -In this appendix, we provide LLM prompt templates and diagram plans used in the diagram planning stage (Appendix A), AI2D-Caption collection details,(Appendix B), additional DiagramGLIGEN details (Appendix C), experimental setup details (Appendix D), human evaluation setup details (Appendix E), additional results/analysis/visualizations/ablations details (Appendix F), and limitations (Appendix G). - -Prompt templates. -In Fig. 9 and Fig. 10, we show the prompt templates for the planner LLM and auditor LLM, respectively. For both LLMs we provide 10 in-context examples followed by the inputs. For planner LLM, we give it the caption and the topic of the caption (e.g., astronomy, biology, etc.). For auditor LLM, we give the diagram plan generated by the planner and ask if there are any issues. - -Diagram plans. -A diagram plan consists of three components: (1) entities - a dense list of objects (e.g., larva in Fig. 2) and text labels (e.g., “egg” in Fig. 2); (2) relationships - complex relationships between entities (e.g., object-object relationship “[obj 0] has an arrow to [obj 1]” or object-text label relationship “[text label 0] labels [obj 0]”); (3) layouts - 2D bounding boxes of the entities (e.g., “[obj 0]: [20, 30, 14, 14]” in Fig. 2). For object-object relationships, we utilize two types: line and arrow (a line with explicit start and end entities), which are useful when specifying object relationships in diagrams such as flow charts or life cycles. -For object-text label relationships, we specify which object each label refers to. For layouts, we use the [x, y, w, h] format for 2D bounding boxes, whose coordinates are normalized and integer-quantized within {0, 1, · · · 100}, in accordance with VPGen (Cho et al., 2023b). - -Full diagram plan example. -Fig. 11 shows an example of a fully generated diagram plan by our LLM planner for the prompt “A diagram showing the life cycle of a butterfly, going from an egg to larva to pupa to an adult butterfly and repeating.” This diagram plan corresponds to the second example in Fig. 13. - -LLM API Costs The average input token length for the planner stands at 4.6K, while the average output token length is 0.5K. Generating a diagram plan using GPT-4 costs $0.17 USD. - -To create the AI2D-Caption dataset described in main paper Sec. 4.1, we employ LLaVA 1.5 (Liu et al., 2023a), a state-of-the-art multimodal language model, to generate captions and bounding box region descriptions in AI2D diagrams. The original AI2D dataset provides annotations for diagrams, including titles, bounding boxes for object/text labels, and object linkages (e.g. arrows/lines between objects). However, since the dataset is designed for - -Published as a conference paper at COLM 2024 - -Appendix - -A LLM Prompt Templates and Diagram Plans - -B AI2D-Caption Collection Details - -14 - ---- PAGE 15 --- - -Instructions: -Given a caption of a diagram and topic, generate the diagram layout, and then a list of required entities that would be needed to create the described diagram. Then generate a list of the relationships between the entities (i.e. which ones are connected or labeling each other). Finally, generate the location of each entity. -An entity can be an image or text. Entity locations should be generated in [x, y, width, height] format, where 0,0 is the top left corner and 100,100 is max image size. - -Figure 9: Input prompt given to the planner LLM during the initial diagram plan generation step. - -Instructions: -Given a caption and a layout of a diagram, you should determine if something is wrong in the diagram based on the caption. You should explain your answer. Think step-by-step as to why the diagram is correct or not. -The diagram will be described in terms of entities in the diagram, the relationships between the entities, and the location of each entity. -An entity can be an image or text. Entity locations will be in [x, y, width, height] format, where 0,0 is the top left corner and 100,100 is max image size. - -Figure 10: Input prompt given to the auditor LLM during the diagram plan refinement step. - -Published as a conference paper at COLM 2024 - -Think step-by-step, break each caption into parts and generate the required entities, relationships, and locations for each part. - -Here are some rules to follow: -All numbers should be positive, do not generate negative numbers. -Please always generate a list of entities, even if the list is empty. -Entities should not be outside the bounds [0, 0, 100, 100]. -The x coordinate + the width of an entity should not exceed 100. -The y coordinate + the height of an entity should not exceed 100. -Entities of the same type should not overlap. - -In-context examples: -Caption: -A diagram showing the positions of the Earth, the Moon, and the Sun during a lunar eclipse. The Earth is directly between the sun and the moon. The moon is in the earth's shadow. -Topic: -… Diagram Layout: -… Required Entities: -sun image (I0) … "Sun" text label (T0) … Entity Relationships: -I1 is between I0 and I2 … Entity Locations: -I0 is located at [8, 56, 13, 28] … - -Input text: -Caption: -[insert new caption] Topic: -[insert topic of caption] - -Here are some rules the diagrams should follow: -All numbers should be positive, no negative numbers. -Entities should not be outside the bounds [0, 0, 100, 100]. -The x coordinate + the width of an entity should not exceed 100. -The y coordinate + the height of an entity should not exceed 100. -Entities of the same type should not overlap. - -In-context examples: -[diagram plans] What is wrong with this diagram? -“The moon phases are out of order, …” - -Input text: -[insert new diagram plan] What is wrong with this diagram? - -15 - ---- PAGE 16 --- - -Figure 11: Example diagram plan from our LLM planner for the prompt “A diagram showing the life cycle of a butterfly, going from an egg to larva to pupa to an adult butterfly and repeating.” - -the diagram question-answering task rather than diagram generation, the diagram titles are often too short and don’t provide enough information to produce meaningful diagram plans. Additionally, the dataset doesn’t include descriptions for each object (i.e., each object bounding box is labeled simply as ‘blob’). To generate captions for each AI2D diagram, we present the diagram to LLaVA 1.5 and prompt it with the question “What is this diagram showing?”. To collect region descriptions of the bounding boxes in AI2D, we first overlay the bounding box annotations of each object on the diagram, by assigning each box a label (i.e., box 1 would get label “B1”, box 2 would get label “B2”, etc..). Then we provide this annotated image to LLaVA 1.5 and ask the model to describe each box’s content (e.g., “what is the object labeled by ‘B1’”?). An example of this annotation is shown in Fig. 4. - -To ensure the quality of AI2D-Caption annotations, we perform a human evaluation of 50 LLaVA 1.5 annotations. For captioning, LLaVA generates very good captions 80% of the time, and for labeling, LLaVA generates very good bounding box region descriptions 68% of the time. We find that when LLaVA makes a captioning error, it is usually minor points, and for bounding box region errors, sometimes it may give nearby boxes the same description. -While automatic annotations have some errors, we find that they are good enough for the domain adaption of DiagramGLIGEN; However, for the test set of AI2D-Caption and in-context learning annotations for the planner LLM, we manually annotate them to ensure correctness (Sec. 4.1). - -Training. -The parameters of DiagramGLIGEN are initialized from the GLIGEN (Box+Text checkpoint)3, and trained on the AI2D-Caption dataset (see main paper Sec. 4.1 and Ap- - -Published as a conference paper at COLM 2024 - -Required Entities: -egg image (I0) larva image (I1) pupa image (I2) adult butterfly image (I3) “egg” text label (T0) “larva” text label (T1) “pupa” text label (T2) “adult butterfly” text label (T3) - -Entity Relationships: -I3 has an arrow to I0 T3 labels I3 I1 has an arrow to I2 T0 labels I0 I0 has an arrow to I1 T2 labels I2 I2 has an arrow to I3 T1 labels I1 Entity Locations: -I0 is located at [24, 50, 14, 14] I1 is located at [50, 74, 14, 14] I2 is located at [74, 50, 14, 14] I3 is located at [50, 24, 14, 14] T0 is located at [20, 44, 10, 4] T1 is located at [44, 80, 10, 4] T2 is located at [80, 44, 10, 4] T3 is located at [44, 20, 10, 4] - -C Additional DiagramGLIGEN Details - -3https://github.com/gligen/GLIGEN - -16 - ---- PAGE 17 --- - -pendix B for details) for 15k steps (batch size of 5 per GPU), which takes 12 hours with 8 A6000 GPUs (each 48GB memory). Other hyperparameters include: - -GLIGEN gated self-attention layers. -The high-level intuition of the gated self-attention layer is similar to cross-attention layers, which are used to incorporate extra information (e.g., text, layouts) into the model. The cross-attention layer in the image generation backbone is used to incorporate text tokens for text-guided image generation, while the gated selfattention layer takes the grounding tokens for layout-guided image generation. Additional details are clarified in the GLIGEN paper (Li et al., 2023b). - -VPEval (Objects, Counts, Relationships, Texts). -We evaluate the diagrams in terms of the presence of objects (object evaluation), the number of objects (count evaluation), the correctness of spatial and connection relationships (relationship evaluation), and the presence of correct text labels (text evaluation) using the VPEval metric (Cho et al., 2023b). -VPEval works by first generating evaluation programs that call specific evaluation modules (e.g., Object, OCR, VQA) and then running the modules to evaluate the image. Each module evaluates different parts of the image (e.g., Object checks object presence). For object, count, and text rendering evaluation, we use the ground-truth diagram plan to determine which evaluation programs to use. For relationship evaluation, following VPEval, we use an LLM (GPT-4) to generate VQA questions that evaluate the spatial/connection relationships of objects in the diagrams (e.g., if the moon is in between the sun and earth or if a light bulb is connected to a battery). While the original VPEval uses BLIP-2 (Li et al., 2023a) as its visual reasoning model, we employ the recently released LLaVA 1.5 (Liu et al., 2023a) model as our visual reasoning model, as we find it is more faithful in diagram question answering in our initial experiments and is an overall stronger model than BLIP-2 (Liu et al., 2023a). In our initial experimentation, we found that LLaVA 1.5 sometimes struggles with relationship evaluation, so we fine-tune the model on relationships to ensure accurate evaluation. - -For object evaluation, LLaVA determines if the object is present. For count evaluation (i.e., if a diagram requires multiple instances of an object), we ask LLaVA if there are exactly N instances of the object in the diagram, where N represents the count of that object in the ground-truth diagram. For relationship evaluation, we ask LLaVA if the relation (spatial or connection) is true. In our experiments, we find that LLaVA often can generate false positives during relation evaluation, as such we also show human evaluation for object relationships (main paper Sec. 4.4). For text evaluation, in alignment with VPEval, we utilize EasyOCR (AI, 2023) as the OCR model and check if the target text is detected. - -We employ crowd-workers from Amazon Mechanical Turk (AMT) for our human preference study. To ensure high-quality annotations, we set the following requirements for the workers: -they must possess an AMT Masters qualification, have completed more than 1000 HITs, maintain an approval rating above 95%, and come from the United States, Great Britain, Australia, or Canada, given that our task is in English. We pay workers $0.06 to compare two diagrams (roughly $14-15/hr). For each prompt, we show diagrams generated by both our DiagrammerGPT and the fine-tuned Stable Diffusion v1.4 (the order of diagrams are randomly shuffled every time to prevent selection biases) and ask five annotators to indicate their preference based on (1) the accuracy of the generated relationships between objects - -Published as a conference paper at COLM 2024 - -• Optimizer: AdamW (Loshchilov & Hutter, 2019) • Learning Rate: 5e-5 • Warmup Steps: 2500 • Image Size: 512x512 - -D Experimental Setups - -D.1 Metrics - -E Human Evaluation Setup Details - -17 - ---- PAGE 18 --- - -(e.g., spatial relationships and arrows/lines) and (2) alignment to the input prompt (e.g., how well does the generated diagram reflect the input prompt). Then, we take the agreement of the annotators. The task is described to the annotators as such: - -1. Object Relationships is a measure of which diagram better captures the proper relationships of the objects (i.e., spacing/positioning of them, arrows/lines between them, etc.). -2. Alignment is a measure of which diagram is a better representation of the input sentence. - -Fig. 12 shows the interface provided to the annotators during the human evaluation process. - -Qualitative comparison to baselines. -Fig. 13 shows example diagrams generated by the baselines (Stable Diffusion v1.4 and AutomaTikZ) and our DiagrammerGPT (both diagram plan and final generation diagram) on the AI2D-Caption test split. Our diagram plans strongly reflect the prompts and the final diagrams are more aligned to the input prompts. In Fig. 13 top example, our diagram correctly shows the earth in four phases revolving around the sun and in the second example, our diagram plan correctly represents the life cycle of a butterfly and the generated diagram captures the circular flow of the diagram plan as well most aspects of the life cycle. Stable Diffusion v1.4 either over- or under-generates objects in the image (e.g., too many earths in the first example and missing egg/larva/pupa stages in the second example), and AutomaTikZ fails to generate proper layouts and objects. Although our generated diagram plans are generally correct, however, sometimes DiagramGLIGEN can fail to properly follow all aspects (e.g., the egg is misdrawn and the larva/pupa are swapped in Fig. 13 bottom example). As noted in main paper - -Published as a conference paper at COLM 2024 - -Figure 12: Interface provided to annotators for human evaluation. - -F Additional Experiment Results - -F.1 Qualitative Results and Analysis - -18 - ---- PAGE 19 --- - -Figure 13: Example diagram generation results from baselines (fine-tuned Stable Diffusion v1.4 and AutomaTikZ) and our DiagrammerGPT on the AI2D-Caption test split. In the first example, our DiagrammerGPT correctly gets the object count right and has clear text, whereas Stable Diffusion v1.4 overpopulates the entities orbiting around the sun. In the second example, our DiagrammerGPT generates an accurate diagram plan and a diagram that mostly reflects the plan, whereas Stable Diffusion v1.4 fails to show a life cycle (i.e., missing the egg, pupa, and larva). As noted in main paper Sec. 5.2, once a better backbone becomes available, our DiagrammerGPT can produce better diagrams based on the diagram plans. AutomaTikZ struggles to generate the proper layouts and objects for both examples. - -Figure 14: -Examples from our diagram refinement step. Our auditor LLM can help reorganize the connections between the components to be more clear in the first example and prevent overlaps of objects in the second example. - -Published as a conference paper at COLM 2024 - -Fine-tuned SD v1.4 AutomaTikZ - -Input Prompt - -A diagram showing the Earth's position in four - -phases as it revolves - -around the sun. - -A diagram showing the - -life cycle of a butterfly, - -going from an egg to - -larva to pupa to an - -adult butterfly and - -repeating. - -Input Prompt - -switch - -switch - -A diagram showing a simple circuit with a switch and a motor connected to a - -motor - -motor - -battery. - -Waning Crescent - -New Moon - -A diagram showing the eight phases of the moon - -Last Quarter - -with labels as it revolves - -earth - -around Earth. It also indicates the direction of - -the sunlight. - -Waxing Gibbous Waning Gibbous - -Full Moon - -19 - -Diagram Plan (Ours) DiagrammerGPT (Ours) - -Summer - -earth - -Spring - -Autumn - -earth - -earth - -sun - -earth - -Winter - -adult butterfly - -adult butterfly - -egg pupa - -egg - -pupa - -larva - -larva - -Initial Diagram Plan After Refinement - -switch motor - -switch motor - -battery - -battery - -battery - -battery - -New Moon - -Sunlight - -Waning Crescent - -Waxing Crescent - -Waxing Crescent - -First Quarter - -Sunlight - -Last Quarter - -First Quarter - -earth - -Waxing Gibbous Waning Gibbous - -Full Moon - ---- PAGE 20 --- - -Figure 15: Examples of open-domain generation demonstrate that our DiagrammerGPT can create diagrams that adhere to the input text prompts. Although DALL-E 3 yields images with superior visual quality, it tends to generate diagrams with redundant and crowded objects and also struggles to follow the prompt accurately (e.g., in the second example, it is not clear where the locations of layers such as the ‘inner core’, ‘outer core’, and ‘mantle’ are. -In the third example, it generates too many objects that are not in rows). - -Sec. 5.2, once a better backbone becomes available, our DiagramGLIGEN can produce better diagrams following the diagram plans. - -Diagram plan refinement. -In Fig. 14, we show how our diagram refinement step (see main paper Sec. 3.1) improves the diagram plans. In the top example, the switch is not connected to the battery, thus does not affect the circuit. After refinement, the connections are corrected so the switch is now also connected to the circuit and the layouts are adjusted to have a more straightforward flow. In the bottom example, the moon phase of ‘New Moon’ is too low and overlaps with the ‘Earth’ object. After refinement, there is no more overlap. - -Open-domain diagram generation. -Our main diagram generation experiments are conducted on diverse domains such as astronomy, biology, and engineering which are included in the LLM planner’s in-context examples. However, given that the in-context examples do not encompass all diagram domains, we experiment with generating diagrams in areas not covered by our LLM in-context examples, such as geology and botany, to assess whether our DiagrammerGPT maintains its ability to produce more accurate diagrams in previously unseen domains. - -In Fig. 15, we show examples of comparing our open-domain diagram generation to DALL- E 3. While our DiagramGLIGEN struggles in some cases (like the third example), it is able to strongly adhere to the diagram plan. Fig. 16 (bottom) also shows our LLM planner is easily able to generalize to completely new domains (e.g., neural networks and vacation planning). -As mentioned in main paper Sec. 5.2, once a stronger layout-guided image generation model than GLIGEN with Stable Diffusion v1.4 backbone is available, our DiagrammerGPT - -Published as a conference paper at COLM 2024 - -A diagram showing three rows of rocks. - -Each row has 5 rocks. The first row shows different types of igneous rocks, including granite, diorite, felsite, basalt, - -and obsidian. The second row shows - -different types of sedimentary rocks, - -including conglomerate, sandstone, shale, limestone, and dolomite. The - -third row shows different types of metamorphic rocks, including slate, - -schist, serpentine, quartzite, and marble. Include a label for the type of - -rock each row shows and each rock. - -A diagram showing the - -layers of the earth. It includes the inner and outer cores, the mantle, - -and the crust. - -A diagram showing three - -rows of plants. The first - -row has a cactus, a - -sunflower, and a dandelion. The second row has a rose, a tulip, and a daisy. The third row - -has a pine tree, a palm tree, and a fern. Include - -text labels for each. - -F.2 Additional Analysis - -Input Prompt DiagrammerGPT (Ours) - -DALL•E 3 Diagram Plan (Ours) - -granite diorite felsite basalt obsidian - -Granite Diorite Felsite Basalt Obsidian - -Igneous Rocks - -conglo merate - -sandstone shale limestone dolomite - -Conglomerate Sandstone Shale Limestone Dolomite - -Sedimentary Rocks - -slate schist serpentine quartzite marble - -Slate Schist Serpentine Quartzite Marble - -Metamorphic Rocks - -crust - -mantle - -outer core - -inner core - -Inner Core - -Outer Core - -Mantle - -Crust - -cactus sunflower dandelion - -cactus sunflower dandelion - -rose tulip daisy - -rose tulip daisy - -pine tree palm tree fern - -pine tree - -palm tree fern - -20 - ---- PAGE 21 --- - -Figure 16: Additional examples. Our planner LLM is effective at generating dense (top) and open-domain (bottom) diagram plans. - -can produce higher quality results. We find that DALL-E 3 generally produces images with good aesthetic style but tends to generate diagrams with redundant and crowded objects (e.g., excessive unnecessary text descriptions in the rock and Earth examples, and an overabundance of plants in the third example). It also continues to struggle with creating accurate diagrams that adhere to a prompt (e.g., generating incorrect layers in the earth example and generating three columns of plants instead of three rows in the plant example). -The DALL-E 3 system card (OpenAI, 2023a) also notes that DALL-E 3 tends to generate scientifically inaccurate information in diagrams. - -Vector graphic diagram generation in different platforms. -We render our diagram plans in Microsoft PowerPoint via VBA language,4 Inkscape5 via a Python scripting extension6, and Adobe Illustrator7 via JavaScript. We represent objects using icons, which are retrieved via the Noun Project Icons API based on corresponding text descriptions.8 Fig. 16 (top) and Fig. 18 show additional examples of diagram plans rendered in the other platforms. - -GPT-4 vs. GPT-4Vision for diagram plan creation and refinement. -As described in main paper, our DiagrammerGPT employs a text-only GPT-4 model for diagram planning and refinement. To explore whether a multimodal language model can offer improvements over text-only GPT-4, we experiment with using the recently introduced GPT-4Vision (GPT-4V) model (OpenAI, 2023c) as the planner and auditor LLM during the diagram generation and refinements steps. As the GPT-4V model does not provide API access yet, we conduct a small-scale qualitative study via the ChatGPT web UI. In our experiments, for the diagram plan creation stage (see main paper Sec. 3.1), GPT-4V does not provide improvements over - -Published as a conference paper at COLM 2024 - -Many Entities & Relationship - -corn mouse owl - -image of - -image of - -image of - -Example - -corn - -mouse - -owl - -A diagram - -showing three food chains. The - -first one is - -carrot rabbit fox lion - -corn, to a mouse, to an - -image of - -image of - -image of - -image of - -owl. The second one is a carrot to a rabbit, to a fox, to a lion. The third one - -carrot - -rabbit - -fox - -lion - -is grass, to a grasshopper, to a frog, to a - -grass grassho - -pper frog snake hawk - -image of grassho - -image of - -image of - -image of - -snake, to a - -grass - -frog - -snake - -pper - -hawk. - -A diagram of a neural network with 3 input nodes, - -2 hidden nodes, and 1 output node - -image - -of input node - -image - -of hidden - -Open- Domain Diagram - -node - -image - -Plans - -of input node - -image - -of hidden - -node - -image - -of input node - -4https://learn.microsoft.com/en-us/office/vba/api/overview/powerpoint 5https://inkscape.org 6https://github.com/spakin/SimpInkScr 7https://www.adobe.com/products/illustrator.html 8https://thenounproject.com/api/ - -Diagram Plan Rendered via MS PowerPoint - -corn mouse owl - -carrot rabbit fox lion - -grass grassho - -pper frog snake hawk - -image of - -hawk - -A diagram infographic showing the steps I can take to plan a - -vacation, include pictures to help visualize the steps. - -Step 1: Choose a destination - -image of destination - -Step 2: Set a budget - -image of budget - -Step 3: Choose the dates - -image of calendar - -image - -Step 4: Book flights - -of output - -image of flight - -node - -Step 5: Book accommod ation - -image of hotel - -Step 6: Plan activities - -image of activities - -Step 7: Pack your bags - -image of luggage - -21 - ---- PAGE 22 --- - -Figure 17: Example of two GPT-4 generated diagram plans. Given the same prompt, GPT-4 can generate diverse plans between runs. - -Input Prompt Rendered with Adobe Illustrator - -Figure 18: Examples of vector graphic diagrams generated with our diagram plans and exported into Microsoft PowerPoint, Inkscape, and Adobe Illustrator. - -text-only GPT-4. In Fig. 19, we present a comparison between the diagram plans generated by GPT-4 and GPT-4V. GPT-4V does not produce diagram plans that are better than textonly GPT-4, suggesting that our text-only representation is robust enough until better or fine-tuned versions of GPT-4V become available for diagrams. Similarly, during the diagram refinement step (see main paper Sec. 3.1), we observed that GPT-4V tends to overestimate correctness when compared to text-only GPT-4, further indicating the strength of our text representation. Fig. 20 shows two examples comparing the models. While text-only GPT-4 is not perfect, it can identify some errors, whereas GPT-4V says the diagram does not need improvement. - -Using model based text rendering instead of Pillow. -We also experiment with using a model-based text renderer, TextDiffuser-2 (Chen et al., 2023b) instead of Pillow. Fig. 21 shows that while TextDiffuser-2 is capable of producing good text labels, however, it can sometimes merge letters (e.g., the ”mm” in summer). Pillow guarantees there is no rendering error (and can easily allow font color/size adjustments). Due to the modular nature of - -Published as a conference paper at COLM 2024 - -Input Prompt: a diagram showing the first four - -planets of the solar system. - -Mercury Venus Earth Mars - -image - -image - -image of Earth - -image of Mars - -of Mercury - -of Venus - -Rendered with Microsoft PowerPoint Rendered with Inkscape - -A diagram showing two food chains. The left food - -chain, starting from the bottom, goes from lichen, to slug, to toad, to snake, - -to eagle. The right food chain, starting from the bottom, goes from algae, to - -snail, to crayfish, to fish, to - -alligator. - -A diagram showing the eight phases of the moon - -with labels as it revolves - -around Earth. It also indicates the direction of - -the sunlight. - -image of Mercury - -image of - -image of - -image of - -Venus - -Earth - -Mars - -image of Sun - -Mercury Venus Earth Mars - -Generated Diagram Plan 1 Generated Diagram Plan 2 - -22 - ---- PAGE 23 --- - -Figure 19: Comparison examples of diagram plans generated by GPT-4 and GPT-4Vision (GPT-4V). GPT-4 creates diagram plans that are sufficiently accurate in capturing the presence of objects and their relationships and GPT-4V does not provide plans that are better. - -Figure 20: Comparison examples of GPT-4 vs. GPT-4Vision for diagram plan refinement. -While text-only GPT-4 is not perfect, it can identify the errors, whereas GPT-4Vision says the diagram does not need improvement. - -DiagrammerGPT, using a text rendering model can easily be incorporated if the end user wants. - -Is the LLM capable of generating diverse diagram plans? -We find that having an LLM generate the diagram plans results in a wide diversity of diagrams. As we show in Fig. 13, 14, 15, and 16, the LLM can generate plans for many different prompts and domains. In Fig. 17 we show an example of the LLM generating variations of the same prompt, further indicating the LLM (GPT-4 in our case) is capable of producing diverse diagram plans. - -Published as a conference paper at COLM 2024 - -Input Prompt - -A diagram showing an electrical circuit with a light - -light bulb battery - -bulb, a battery, and a - -switch. - -light bulb battery - -grasshopper - -grasshopper - -adult grasshopper - -A diagram showing the life - -egg - -nymph - -cycle of a grasshopper and a mealworm beetle, - -grasshopper - -grasshopper - -adult grasshopper - -illustrating the transformation they undergo during their - -egg - -nymph - -development. The grasshopper goes from an - -mealworm - -mealworm - -mealworm beetle egg - -beetle larvae - -beetle - -egg, to a nymph, to an - -pupa - -adult and then the mealworm beetle goes from an egg, to a larvae, - -mealworm beetle egg - -mealworm beetle larvae - -mealworm beetle pupa - -to a pupa, to an adult. - -Input Prompt - -A diagram showing the life - -cycle of a ladybug, specifically focusing on the different stages from - -egg larva pupa adult - -eggs to adult. - -egg larva pupa - -switch - -switch - -A diagram showing a simple circuit with a switch and a motor connected to a - -motor - -battery. - -GPT-4 (Default) GPT-4Vision - -switch swi tch - -light bulb battery switch - -Light Bulb Battery Switch - -mealworm beetle egg grasshopper egg - -mealwo - -grassh opper - -rm beetle - -egg - -egg - -mealworm beetle larvae - -mealworm beetle larvae - -mealworm - -beetle - -pupa - -grasshopper - -nymph - -adult mealworm - -mealworm beetle pupa - -grasshopper - -beetle - -nymph - -adult mealworm - -grasshopper - -mealworm beetle adult - -beetle - -adult - -grasshopper - -mealworm beetle adult - -adult - -Initial Diagram Plan Generated Feedback - -GPT-4: There should be arrows between each phase to indicate the progression of the life cycle. - -ladybug life - -cycle - -GPT-4Vision: The diagram is correctly aligned with the caption, and all entities are in a circular layout. - -adult - -lady bug - -GPT-4: The entities in the diagram are not connected in a loop, which is necessary for a circuit. - -GPT-4Vision: From the given information, nothing seems to be wrong with this diagram. It adheres to all the rules and correctly represents the caption. - -battery - -motor - -battery - -23 - ---- PAGE 24 --- - -Figure 21: Explicit text rendering via Pillow compared to model-based text rendering via TextDiffuser-2 (Chen et al., 2023b). - -We show ablation studies on our design choices with DiagramGLIGEN: the number of denoising steps with layout guidance and whether to only update the gated self-attention layers parameters. - -Table 5: Ablation of # denoising steps with layout guidance. DiagramGLIGEN uses 50 denoising steps in total. We use α = 15 steps as our default setting. - -Number of denoising steps with layout guidance. -The number of denoising steps with layout guidance (i.e., with the gated self-attention layer activated in each transformer block of the diffusion UNet), denoted as α, is a crucial hyper-parameter in DiagramGLIGEN. -A larger α value indicates stronger layout control. Table 5 presents an ablation study using varying α values. A smaller α value enhances object generation, while a larger value improves count performance. This observation aligns with intuition: rigorous layout control more effectively prevents the generation of extraneous objects in the background but may detract from the visual realism of the generated objects, which is also observed in (Li et al., 2023b; Lin et al., 2023). We set the default value for α as 10 steps, as it ensures a good balance of accuracy for objects and counts while achieving optimal performance in depicting object relationships. - -Table 6: -Ablation of fine-tuning different layers of DiagramGLIGEN. We use the fully fine-tuned model as our default setting. GatedSA: Gated Self-Attention. - -Published as a conference paper at COLM 2024 - -F.3 Ablation Studies - -# Layout guidance steps VPEval - -α = 5 steps 88.0 50.6 48.0 84.9 α = 10 steps 86.4 49.4 47.0 86.6 α = 15 steps (default) 86.4 57.0 47.5 87.9 - -None 68.5 88.9 29.3 GatedSA Layers only 67.4 88.9 30.0 All Layers (default) 71.2 89.4 32.1 - -Our Explicit Text Rendering Text Rending via TextDiffuser-2 - -Object Count Text Relationships - -Updated parameters VPEval Captioning CLIPScore - -Overall BERTScore Img-Txt - -24 - ---- PAGE 25 --- - -Table 7: Ablation of using ground-truth plans from AI2D-Caption (e.g., oracle performance) instead of GPT-4. - -Fine-tuning: all layers vs. layout layers. -In Table 6, we present an ablation study comparing the fine-tuning of only the layout control layers in DiagramGLIGEN with fine-tuning of the entire model, including the Stable Diffusion backbone. Full fine-tuning enhances performance and improves the visual quality of the diagrams. Therefore, we employ the fully fine-tuned version as our default model for all subsequent experiments. - -Using ground-truth diagram plans. -We experiment with generating diagrams using ground-truth diagram plans from AI2D-Caption. Doing this allows us to measure the upper bound of DiagramGLIGEN and see how much room our stage 1 has for improvement. -Table 7 shows that using ground-truth plans does indeed do better than GPT-4 and that DiagramGLIGEN is able to perform better when using ground-truth plans. However, it is interesting to know that using GPT-4 performs very closely to the oracle score. - -Our framework can benefit many educational applications, such as presentation/paper creation, and human-in-the-loop diagram generation/modification. However, akin to other text-to-diagram/text-to-image generation frameworks, our framework can also make some errors and be utilized for potentially harmful purposes (e.g., creating false information or misleading diagrams), and thus should be used with caution in real-world applications (with human supervision, e.g., as described in Sec. 5.4 human-in-the-loop diagram plan editing). Also, generating a diagram plan using the strongest LLM APIs can be costly, similar to other recent LLM-based frameworks. We hope that advances in quantization/distillation and open-source models will continue to lower the inference cost of LLMs. Lastly, DiagramGLIGEN is based on the pretrained weights of GLIGEN and Stable Diffusion v1.4. -Therefore, we face similar limitations to these models, including deviations related to the distribution of training datasets, imperfect generation quality, and only understanding the English corpus. - -Published as a conference paper at COLM 2024 - -Ground-truth + DiagramGLIGEN (oracle) 81.9 89.8 32.3 GPT-4 + DiagramGLIGEN 71.2 89.4 32.1 - -G Limitations - -Diagram Plan Source VPEval Captioning CLIPScore - -Overall BERTScore Img-Txt - -25 \ No newline at end of file diff --git a/experiments/literature_review/raw_text/scidoc.txt b/experiments/literature_review/raw_text/scidoc.txt deleted file mode 100644 index 2563b3d..0000000 --- a/experiments/literature_review/raw_text/scidoc.txt +++ /dev/null @@ -1,1563 +0,0 @@ - - ---- PAGE 1 --- - -SciDoc2Diagrammer-MAF: Towards Generation of Scientific Diagrams - -from Documents guided by Multi-Aspect Feedback Refinement - -Ishani Mondal,1 Zongxia Li,1 Yufang Hou,2 Anandhavelu Natarajan,3 - -1 University of Maryland, College Park, 2 IBM Research Dublin , 3 Adobe Research {imondal@umd.edu, zli12321@umd.edu, YHou@ie.ibm.com, anandhavelu@gmail.com, garimell@adobe.com, ying@umd.edu} - -Findings of the Association for Computational Linguistics: EMNLP 2024, pages 13342–13375 - -Abstract - -Automating the creation of scientific diagrams from academic papers can streamline the development of tutorials, presentations, and posters. -Current text-to-image models struggle with generating accurate and visually appealing diagrams from long-context inputs. We propose SciDoc2Diagram, a task that extracts relevant information from scientific papers and generates diagrams, along with a benchmarking dataset, SciDoc2DiagramBench. We develop a multi-step pipeline SciDoc2Diagrammer that generates diagrams based on user intent using intermediate code generation. -We observed that initial diagram drafts are often incomplete or unfaithful to the source, requiring SciDoc2Diagrammer-Multi-Aspect- Feedback (MAF), a refinement strategy that significantly enhances factual correctness and visual appeal and outperforms existing models on both automatic and human judgement. - -1 Introduction - -Researchers often need to transform detailed studies into compelling diagrams for presentations, going beyond the usual figures in their papers to include summaries, methodologies, and dynamic data visualizations (Chapman et al., 2014).1 Automation of diagram generation from the multimodal components of academic papers—such as text, figures, tables, and plots—could enable the creation of accurate and appealing visual aids for scientific tutorials and presentations (Mondal et al., 2024; Fu et al., 2021; Sun et al., 2021), which can help researchers and educators to improve productivity and enhancing the overall quality of their presentations. We introduce a new task Sci- Doc2Diagram where the input comprises of the content of academic papers and user’s intent and - -1According to Fu et al. (2021), approximately 39.2% of figures in ACL slides are creatively derived from the original papers, taking forms like flowcharts, summary tables, and plots/charts that visually interpret the paper’s content. - -Aparna Garimella,3 Jordan Boyd-Graber1 - -Figure 1: An example of Diagram Generation and Refinement using SciDoc2Diagrammer-MAF with input as a paper and user-defined intent. - -the output is a scientific diagram crafted from the content of papers and aligned with user’s intent, which we call “Extrapolated-Diagram”. - -Consider a scenario where a busy scientist wants to create figures from papers within a tight deadline, with the specific goal of generating a bar chart as shown in the Figure 1. To achieve this quickly, recent advancements in text-to-image (T2I) generation models like Stable Diffusion (Rombach et al., 2022a) and DALL-E (Ramesh et al., 2021, 2022a) offer promising solutions. However, these models face limitations in generating factually correct and visually appealing scientific figures. They primarily produce raster graphics at low resolutions, which is not suitable for scientific figures requiring precision and valid text (Hsu et al., 2021). Belouadi et al. -(2023) introduced a scientific diagram generation method based on only user’s intent. However, creation of informative diagrams using long-content papers as an additional input remains unexplored. - -How effective are the current state-of-the-art Large Language models (LLMs) or Visual Language Models (VLMs) at generating scientific diagrams? Despite their potential, a significant knowledge gap exists: there is currently no standard benchmark to assess their performance in this area. To address this gap, we introduce the Sci- - -13342 - -November 12-16, 2024 ©2024 Association for Computational Linguistics - ---- PAGE 2 --- - -Doc2DiagramBench dataset that features 1080 diagrams from 89 *ACL papers, sourced from existing slides in ACL Anthology (Section 3). This dataset supports researchers in automating scientific diagram creation for presentations. - -We evaluate state-of-the-art LLMs (Jiang et al., 2023; Touvron et al., 2023; OpenAI, 2023; Abdin et al., 2024) to extract relevant information from documents pertinent to the user-specified intent. Specifically, we use a multi-step pipelined method, SciDoc2Diagrammer that first extracts relevant data from academic papers corresponding to intent of the user in creating the diagram, generates an intermediate code that renders the final diagram (Section 4.1) as shown in Figure 1. However, the first-draft images generated by these models suffer from a range of errors: hallucination, incompleteness and unreadability due to their incapabilities of parsing long-prompts (Lee et al., 2024a). - -Therefore, we propose a sequential refinement approach SciDoc2Diagrammer-MAF based on the intial draft generated by SciDoc2Diagrammer. The new approach is guided by several critic models to provide targeted feedback (multiple aspects such as Completeness, Faithfulness, and Look and Feel) in a step-by-step hierarchical manner on the output to further enhance the quality of output (Section 4.2). -Our multi-aspect feedback refinement approach significantly improves diagram quality, as confirmed by both automatic and human evaluations, demonstrating notable enhancements over existing methods (Section 6). Complex flowcharts and tables derived from single or multiple documents show notable enhancements in faithfulness and completeness when refined sequentially. It is particularly advantageous when iteratively synthesizing information from multiple sources to enhance clarity and precision.2 - -2 Task Formulation of SciDoc2Diagram - -Our goal is to automatically generate scientific diagrams from academic papers tailored to user specifications. We process the input document D = {T, F, τ, π}, which includes body text T, figures F, tables τ, and plots π, guided by a user intent I (e.g., “Create a bar chart showing the comparison...” in Figure 18). The output, an Extrapolated- Diagram δ, visually represents content not explicitly illustrated in the paper. - -2Code and data available at https://github.com/ Ishani-Mondal/SciDoc2DiagramGeneration. - -Conversion Type # Diagrams - -Extrapolated-Flowchart 320 Extrapolated-Results 290 Extrapolated-Architecture 270 Extrapolated-Summary 200 - -Total 1080 - -Table 1: Statistics of SciDoc2DiagramBench Dataset which includes 1080 diagrams and the distribution highlights the diversity of diagrams. - -3 SciDoc2DiagramBench: A New Dataset - -Creating Extrapolated-Diagram from source papers is an important yet challenging task. However, no current dataset evaluates state-of-the-art models on this task. To advance research, we have created SciDoc2DiagramBench dataset, repurposing DOC2PPT (Fu et al., 2021) (scientific papers and corresponding slides from ACL Anthology) and TutorialBank (Fabbri et al., 2018) (tutorials from *ACL conferences).3 Our dataset includes the following three parts and the overall statistics of different diagram types are in Table 1 with examples in Appendix 15. - -SciDoc2DiagramBench-Gold. -Our initial goal was to explore what are the types of Extrapolated- Diagram usually created by humans while making presentations. We hired an expert annotator from Upwork with more than five years of authoring NLP-related papers and creating presentations. The annotator had two tasks, first to identify slides containing Extrapolated-Diagram (excludes slides that used external information not found in the paper), second to manually write an intent for the corresponding diagram creation (13, 12). Through this annotation process, we collect 265 gold-standard Extrapolated-Diagram from 34 papers where each instance comprises of a tuple of . The first three authors manually checked the intents, and only instances that were unanimously deemed reasonable were retained. After analyzing these diagrams, 32% were flowcharts illustrating methodology (Extrapolated-Flowchart), 33% were results/plots (Extrapolated-Results), 25% were summary of related works/contributions (Extrapolated-Summary) and 10% were related to clarifying existing architecture (Extrapolated- Architecture). - -3Data Contamination issues are discussed in Appendix 14 - -13343 - ---- PAGE 3 --- - -Create a bar chart showing the comparison of the Precision of NASH on Reuter Dataset with that of the three best performing baselines on 128 bits along with providing short description of each of the models - -Question Generator - -Stage 2: Diagram Rendering - -QA pairs obtained (Diagram - -Figure - -Table - -Data Store - -to Table - -Plan) - -Stage 1: Diagram Planning - -SciDoc2Diagrammer - -CQ1: Is it a bar chart? -CQ2: Is the comparison between 4 models shown? -CQ3: Is there a short description of each of the models? -…… - --Show only the comparison among 4 best performing models - Include short description in the axes of each model - -Refiner - -Completeness - -Evaluator - -FQ1: What are the three best performing baselines? -FQ2: What are the performances corresponding to the three best performing baselines? - …… - -After comparison Layout Evaluator + Refiner - -FQ1: What are the three best performing baselines? -FQ2: What are the performances corresponding to the three best performing baselines? - …… - -Faithfulness Evaluator (Compares answers - -Source PDF \cite{shen-etal-2018-nash} - -of questions from both PDF and Image) - -SciDoc2Diagrammer-MAF - -Figure 2: Outlines the procedure for generating diagrams from academic papers based on specific user intents, followed by refinement of each critic models. - -SciDoc2DiagramBench-Extended. -Using insights from SciDoc2DiagramBench-Gold on how presenters transform paper content into diagrams, we used GPT-4 to generate new diagram intents for this subset of data (Table 14). We also provide guidelines for creating Extrapolated-Diagram that blend text and visuals from academic papers. -Each intent, reviewed by the first author, aims to guide the creation of practical diagrams. Each instance includes but lacks a corresponding human-created gold diagram. - -SciMultiDoc2DiagramBench-Gold. -To evaluate the task on another challenging set, we examined scientific ACL tutorials from TutorialBank (Fabbri et al., 2018), focusing on slides where creators used content from multiple papers to design their diagrams. An expert annotator from Upwork identified these slides with Extrapolated- Diagrams. The first author then manually wrote intents for these tutorial slides, with each instance documented as . - -4 Methodology - -The SciDoc2Diagram task comprises of generating initial first-draft generation of diagrams from documents using base models (Section 4.1), followed by refinement of diagrams using critic modules (Section 4.2). - -4.1 SciDoc2Diagrammer - -We introduce SciDoc2Diagrammer, a two-stage - -pipeline for generating “Extrapolated-Diagrams” from paper content (Upper portion of Figure 2). In the first Diagram Planning stage, a planner VLM takes the PDF (decomposed into sections and figures/tables) and user’s intent and generates a diagram plan. Inspired by the success of extracting in- - -formation using question-generation and retrieving answers from relevant snippets as done by Krithara et al. (2023); Lee et al. (2023); Qiu et al. (2018); -Auer et al. (2023), we also synthesize diagram plan by first generating clarifying questions based on the user’s intent, and then obtain answers of those questions from relevant portions of the paper to come up with question-answer pairs, which we refer to as diagram plan. In the second stage, inspired by the success of LLMs in generating code from natural language instructions (Ugare et al., 2024; -Maddigan and Susnjak, 2023), we use LLMs to take the diagram plan and user intent as inputs to generate an intermediate code and execute the code to finally render the diagram. - -Diagram Planning: -Given an input document D = {T, F, τ, π} consisting of body text T, figures F, tables τ, and plots π, and user intent I, the diagram planning phase includes the following sub-steps: [1] Intent Classification: We use an LLM (Few-shot with 4 exemplars) to classify if the intent of the user is related to results or not. For example in Figure 3, the intent is not related to results, but the one in Figure 2 is related to results. -[2] Question Generator: Using the intent, we ask LLMs to generate clarification questions to create the diagram conditioned on intent. For example, for the intent in Figure 2, we obtain a set of clarification questions. [3] Retrieve Information: We retrieve relevant passages from the paper based on each of questions (Lewis et al., 2020). In Figure 2, a plot and two paragraphs from Experimental Section of the source paper are retrieved to answer the generated questions. If any result type figure is relevant to the question, we first extract data from the figures and convert them into tables for simplification of content retrieval as motivated by prior works (Liu et al., 2023a; Masry et al., 2023). [4] Diagram Plan Synthesis: After generating questions and extracting relevant content from paper, we extract answers of those questions. Then we formulate a dictionary of QA pairs/diagram plan Q = {(q1, a1), (q2, a2), . . . , (qn, an)}, where qi are the questions and ai are the answers. - -Diagram Rendering: -In this phase, we use the user intent I and QA pairs Q from the previous step to generate an intermediate code Θ that constructs diagram. This intermediate code is represented as Python code, as shown in Figure 2. Finally, we execute the intermediate code Θ to render - -13344 - ---- PAGE 4 --- - -“Extrapolated-diagram” δ, as exemplified by the bar - -chart at the top-right corner of Figure 2. - -4.2 SciDoc2Diagrammer-MAF - -Motivation: To gain a qualitative understanding of the diagrams generated from SciDoc2Diagrammer, we randomly sample 50 diagrams from the test set for manual inspection. There are some frequently occurring errors (Table 10) and we classify them into three broad categories: Completeness (essential components are missing or incomplete, affecting the diagram’s informativeness), Faithfulness (misleading information that do not accurately reflect the source data), and Layout (poor visual alignment hindering readability and comprehension). Besides, we also noticed that most of the errors propagate during the transition from our diagram planning to rendering stage; this was due to the inability of LLMs to handle long-context prompts (Nathani et al., 2023; Gani et al., 2024). - -Inspired by the self-refinement strategies by Madaan et al. (2023), we wanted to study whether the quality of our diagrams can be enhanced by using generic feedback for a wide range of errors or by applying targeted feedback for specific error categories. For exploring the second strategy, we introduce an iterative refinement framework, depicted in Figure 2 (Lower part), designed to enhance quality of diagram generation by specifically rectifying above-mentioned error types. - -Overview of the Components: -As illustrated in Figure 3, our MAF framework comprises of the following components: a base model (Sci- Doc2Diagrammer) that generates an initial diagram (O) using both intent and source document (Step 1 in 3). It passes through three specialized feedback critic evaluator modules, each focusing on a critical aspect of diagram quality such as completeness (Ccritic), faithfulness (Fcritic), and layout (Lcritic) on D, a Refiner module R that generates refined diagram O′ based on O and feedback. - -Feedback Modules: -The completeness critic Ccritic (Algorithm 1) decomposes the diagram’s intent I and generates a set of questions Q from the user intent I to determine how well various elements are represented in the diagram (Step 3). It then involves extracting answers from the diagram, scoring each response for adequacy, calculating an average score as the completeness score CScore, and providing textual feedback (CFeedback) to suggest improvements (Step 4). - -The Faithfulness critic Fcritic (Algorithm 2) evaluates how accurately a diagram represents information from its document source. It evaluates the accuracy of a diagram by generating questions, retrieving answers from both the document and the diagram, and scoring their consistency to provide an average score and feedback for improvement. -The layout critic Lcritic (Algorithm 4) evaluates how good is the look and feel of generated diagram, and provides score and feedback for improvement. -We use two refinement techniques: - -Summarization-Based Refinement Algorithm (SumMAF) We adopt summarization-based multi-aspect refinement strategy from Nathani et al. -(2023) for diagram refinement. However, in contrast to their method, we use our critic models that act as human proxies (Algorithm 4), and revisit PDF source and user intent repeatedly to validate and refine the diagram Ccritic(Algorithm 1) and Fcritic(Algorithm 2), mimicking the iterative human process of cross-verifying information through continuous questioning and answering. - -The first draft is sent to Ccritic (Algorithm 1) which checks if all necessary components are present (In Figure 3, it checks the completeness of creating a flowchart). At the same time, the faithfulness evaluator Fcritic (Algorithm 2) ensures the accuracy of diagram by comparing it with source PDF, whether or not all the information in the source figure is true, by generating questions (FQ1, FQ2 in Figure 2) based on the diagram and obtaining answers from both PDF and diagram. Also, the layout evaluator checks the visual arrangement, ensuring good alignment and spacing (Figure 2). All the critic models provide feedback on a single output (Step 3’) and sent to the refiner. The refiner model makes adjustments by summarizing the suggested modifications from all critics and generates a new diagram (Step 4’). This is repeated (Steps 3’, 4’, 5’ in Figure 3) until the evaluator models are satisfied or we reach maximum iterations. - -Sequential Refinement Algorithm (SeqMAF): -Algorithm 3 enhances a diagram in a step-by-step hierarchical manner ensuring that each critic is satisfied (See Figure 18). The intuition is that, complex tables or flowcharts benefit from being accurate and detailed in their depiction of processes or relationships, where sequential refinement allows for gradual addition of details, ensuring that each element is correctly represented. Here the diagram - -13345 - ---- PAGE 5 --- - -Figure 3: The figure (an example from SciDoc2DiagramBench) depicts SciDoc2Diagrammer-MAF, which refines diagrams based on user intent and source document. Initially, SciDoc2Diagrammer creates a diagram (Step 2), which is refined through three critic modules which assess and provide feedback on necessary components, data accuracy, and visual design. The diagram is repeatedly refined, as shown on the left side of the figure (illustrating SumMAF), where feedback from all critics is integrated at Step 4’. The refinement continues, evaluated at Step 5’, until the maximum iterations are reached or the diagram meets the specified standards. - -1 - -C_critic - -C_critic F_critic L_critic SciDoc2Diagrammer - -- Details on how syntactic through composition operators or hierarchical representations could be explicitly visualized - Include the explicit modelling and long range dependency - -2 3 - -3’ - -2’ - -Refiner - -Refiner - -4’ 5’ - -is feed into different critic modules one by one, refine the output until each of them are satisfied. - -In Figure 3, Ccritic first provides feedback (detail composition operators or hierarchical representation, explicit modelling and long range dependency step inclusion) on the diagram in Step 4, and the refiner takes the initial figure and refines based on feedback in Step 5, and passed onto Ccritic to further evaluate and repeat the same steps (4, 5, 6) till maximum iteration is used or the Ccritic provides a score greater than 4.5 (as stated in Algorithm 1. -The same process is repeated by Fcritic and Lcritic modules and generates output after Step 12. - -5 Experimental Setup - -Dataset Preprocessing. -Text on the papers in Sci- Doc2DiagramBench was extracted through scipdf Parser4 which has been built on top of the Grobid (GRO, 2008–2024). Figures and captions were extracted through pdffigures2 (Clark and Divvala, 2016). We use 20 randomly selected diagrams from SciDoc2DiagramBench with stratified sampling to ensure that we have four type of diagrams (Table 1) for creating in-context examples in our prompts - -4https://github.com/titipata/scipdf_parser - -Intent : A flowchart to illustrate the process of how LSTMs learn syntax-sensitive dependencies and how modeling structure can improve their performance. - -13 - -L_critic - -6 - -F_critic Modify the "Explicit Modeling of Structure" node to clarify that the effectiveness of this approach depends on how syntactic information is integrated into the model. - -Apply different colors not only to dashed lines but also to nodes based on their function -Ensure there is sufficient space between nodes and edges. - -8 - -11 - -Refiner Refiner - -9 - -12 - -4 - -5 - -10 - -7 - -and the rest of the dataset is used for evaluation. - -Base LMs. -For generating first-draft of diagrams, we experiment with state-of-the-art models to extract table from existing figures during diagram planning (Figure 2): 1) MatCha (Liu et al., 2023b), 2) Chart-to-Text (Kantharaj et al., 2022), 3) UniChart (Masry et al., 2023), 4) DePlot (Liu et al., 2023a) and GPT-4 Vision (GPT-4V) (OpenAI, 2023). Finally, based on our assessment (Figure 20), we concluded that GPT-4V outperformed other models and was the best choice. For diagram planning, we experiment with three open-source LLMs: 1) LLama3 (Huang et al., 2024), 2) Phi- 3 (Abdin et al., 2024) Mistral-7b (Jiang et al., 2023) and for diagram rendering we use GPT4-o (OpenAI, 2023). - -Critic Models for SciDoc2Diagrammer-MAF. -Inspired by the success of GPT-4V in making pairwise comparison of evaluation of images using Question-Answering driven approaches (OpenAI, 2023; Zhang et al., 2023) and image aesthetic evaluation (Abe et al., 2024), we adopt GPT-4V model to provide feedback on different aspects of our generated diagrams on various criteria (Prompts can be found in 20, 21, 22). - -13346 - ---- PAGE 6 --- - -Models (IV) R BS CS R BS CS R BS CS R BS CS - -DALLE-3 0.12 0.45 0.23 0.11 0.34 0.21 0.25 0.45 0.25 0.04 0.10 0.18 Automatikz 0.20 0.48 0.25 0.20 0.50 0.29 0.24 0.45 0.28 0.27 0.56 0.45 - -w/ GPT4-o (ZS) 0.22 0.58 0.43 0.32 0.49 0.30 0.24 0.50 0.34 0.33 0.61 0.62 w/ GPT4-o (FS) 0.28 0.67 0.43 0.40 0.56 0.38 0.32 0.57 0.45 0.45 0.67 0.62 w/ GPT4-o-SR 0.30 0.70 0.45 0.44 0.56 0.38 0.38 0.65 0.54 0.47 0.68 0.69 w/ GPT4-o-SumMAF 0.35 0.74 0.48 0.50 0.57 0.39 0.34 0.64 0.49 0.50 0.74 0.74 w/ GPT4-o-SeqMAF 0.39 0.79 0.53 0.49 0.49 0.36 0.37 0.58 0.49 0.45 0.67 0.62 - -Table 2: Automatic evaluation of models on various diagrams on SciDoc2DiagramBench-Gold using ROUGE (R), BERTScore (BS), CLIPScore (CS) with the highest values shown in green, highlighting that flowcharts, architectures are of the best quality after sequential refinement, whereas the other ones after summarization-based refinement. - -Model Flowchart Summary Architecture Results C F L C F L C F L C F L GPT4-o w/o refinement 3.3 3.5 4.0 3.9 4.0 4.2 2.8 4.0 4.0 3.7 4.0 4.0 GPT4-o w/ refinement 3.9 4.0 4.0 4.5 4.1 4.2 3.2 4.0 4.0 3.7 4.5 3.8 Phi-3 w/o refinement 3.2 4.0 4.0 3.8 4.0 4.0 2.8 4.0 4.0 3.5 3.6 3.8 Phi-3 w/ refinement 3.8 4.0 4.3 4.4 4.1 4.2 3.0 4.3 4.0 3.8 4.1 3.8 - -Table 3: Comparison of Completeness (C), Faithfulness (F), and Layout (L) assessed by humans for different model variants on SciDoc2DiagramBench-Extended, indicating that on the complex set, Flowcharts and Summary tables improve significantly after refinement, but Layout-satisfaction is still comparatively lower. - -SciDoc2Diagram - -Prior Work and Baselines. -We have compared our approaches with existing text-to-image generation methods like DALLE3 (Rombach et al., 2021) and Automatikz (Belouadi et al., 2023). -For experimenting with Diagram Planning in Sci- Doc2Diagrammer, we evaluate using zero-shot and few-shot (with 3 in-context exemplars) versions of LMs as mentioned in Table 8 (Prompts in 15, 16, 18, 19). We use Self-Refine by Madaan et al. -(2023) as another baseline (Prompt can be found in 25). - -Evaluation Metrics. -SciDoc2DiagramBench- Gold and SciMultiDoc2DiagramBench-Gold include diagrams explicitly crafted by humans for each document or multiple documents, which can be served as a gold standard, allowing us to benchmark the quality of our generated diagrams. To evaluate our diagrams, we use three automated metrics: BERTScore (Zhang et al., 2019), ROUGE-1 (Lin, 2004), and CLIPScore (Hessel et al., 2021). - -However, simple token evaluations and neural evaluations like BERTScore and CLIPScore are only measuring string similarity and lacking a more finegrained evaluation framework that correlates more with human preferences (Li et al., 2024c). -Thus, we go beyond string similarity scores and include a comprehensive analysis of more finegrained aspects through human evaluation and the GPT4-V - -Flowchart Results Architecture Summary - -Evaluation. We assess completeness (measures the degree to which all relevant and necessary information is in the generated figure), faithfulness (assesses how well the figure adheres to the facts, data, or specific instructions provided, ensuring that the content is correct and not misleading or misrepresented), and layout (measures visual clarity, focusing on how well the elements are structured and arranged) on a scale from 1 to 5. - -6 Results and Findings - -Our main objective is to explore the creation of ‘Extrapolated’ scientific visuals from documents us- - -ing zero-shot/few-shot settings of advanced VLMs, and also analyze whether the quality of these visuals without dataset-specific fine-tuning, can be improved using our proposed SciDoc2Diagrammer- MAF. To answer this broad question, we focus our experiments on answering the following research questions as follows: - -6.1 Main Research Questions and Results - -RQ1: What is the best base LM for generating diagrams before refinement? -GPT4-o is clearly the best-performing base/refiner model on human and automatic judgement. In Table 13, GPT-4o consistently outperforms other models across all categories of diagrams on the SciDoc2DiagramBench. - -13347 - ---- PAGE 7 --- - -Figure 4: -Average Human Rating on Completeness, Faithfulness and Layout on three subparts: -SciDoc2DiagramBench-Gold (left), SciDoc2DiagramBench-Extended (middle) and SciMultiDoc2DiagramBench-Gold (right), it implies as the complexity of diagram creation increases, our proposed Multi-Aspect Refinement strategy appears to become more effective. - -Models (IV) FChart (R) FChart (BS) FChart (CS) Results (R) Results (BS) Results (CS) Arc (R) Arc (BS) Arc (CS) Summary (R) Summary (BS) Summary (CS) w/ GPT4-o-SeqMAF 0.39 0.79 0.53 0.49 0.49 0.49 0.36 0.37 0.58 0.45 0.67 0.62 w/o Completeness-SeqMAF 0.25↓ 0.55↓ 0.35↓ 0.45 0.44 0.34↓ 0.36 0.57 0.47 0.30↓ 0.50↓ 0.45↓ w/o Faithfulness-SeqMAF 0.25↓ 0.55↓ 0.35↓ 0.35↓ 0.38↓ 0.25↓ 0.37 0.58 0.49 0.42 0.64 0.57 w/o Layout-SeqMAF 0.33 0.74 0.52 0.49 0.49 0.49 0.36 0.54 0.46 0.41 0.62 0.59 - -Table 4: Ablation Analysis Results: Here the column names FChart indicates the Extrapolated-Flowchart category of images, Arc indicates the Extrapolated-Architecture category of images, Results denotes the Extrapolated- Results category of images, and Summary denotes the Extrapolated-Summary category of images. Significant drop compared to the first row (greater than 0.1) is shown by ↓. - -This superior performance is indicated by the green cells, which show GPT-4o achieving the highest scores in all the automatic metrics. We observe similar trend when we ask annotators to rate on the completeness, faithfulness and layout-satisfaction on 30 randomly selected samples of images from SciDoc2DiagramBench-Gold generated by each model (Figure 19). - -RQ2: How effective is our Scidoc2diagrammer- MAF over SciDoc2Diagrammer? -Based on Table 2, the effectiveness of our refinement strategies on SciDoc2DiagramBench-Gold using GPT4-o with SumMAF and SeqMAF enhancements shows significant improvements over the Zero-shot (ZS) and Few-shot (FS) versions on all the diagram types on automatic evaluation. This finding holds true for SciMultiDoc2DiagramBench-Gold as well, as evident from the results in Table 7. - -Drawing inspiration from Ribeiro et al. (2020), we acknowledge that automatic metrics alone are insufficient to fully evaluate model performance, so we also conduct human evaluation. -For SciDoc2DiagramBench Gold and SciMultiDoc2DiagramBench-Gold, we had human raters evaluate the quality of diagrams produced by different models and compare these to actual human-created slides, considering the - -source paper and the diagram’s intended purpose for 30 randomly sampled diagrams. -In the SciMultiDoc2DiagramBench-Extended, which lacks a gold standard image, only the paper, the intent of the diagram, and the model-generated images were presented for evaluation on 50 randomly sampled images. Two authors and two professional crowdworkers from Upwork assessed the quality of all images, using criteria such as Completeness, Faithfulness, and Layout from 1 to 5. Figure 4 illustrates that diagrams generated using our refinement strategies compared to Few-shot and Self-Refine strategies consistently receive the highest scores from humans on all dataset subsets. - -RQ3: What type of refinement works the best across each diagram type based on automatic evaluation? -From Table 2, SeqMAF shows significant improvement with the highest scores in all metrics compared to both ZS and FS for flowcharts. -SumMAF also improves over ZS and FS, but to a lesser extent than SeqMAF, indicating SeqMAF’s superior refinement capability on both completeness and detail accuracy. -For Results diagrams, both SeqMAF and SumMAF show improvements over non-refined versions, with Seq- MAF slightly leading in overall performance except for ClipScore. However, in Architecture and - -13348 - ---- PAGE 8 --- - -Create a flowchart for the process of detecting rumors in microblog posts using propagation structures - -Paper Detect Rumors in Microblog Posts Using Propagation Structure via Kernel Learning - ACL Anthology - -Feedback Feedback - -Provide more details on data sourcing and preprocessing in the early stages of the flowchart, and elaborate on how SVM utilizes features from the Propagation Tree Kernel (PTK) to make decisions, improving clarity on model integration and operation. - -Utilize color coding and varied shapes to differentiate actions, improve text clarity within nodes for better legibility - -Integrate brief descriptions to explain key components, and visually represent the feedback loop's impact on earlier stages using symbols or arrows to highlight the model's dynamic refinement process. - -Incorporate steps for result validation and feedback mechanisms post-classification - -Before Refinement, incomplete - -Much more complete and faithful for the paper and intent - -Complete, Better layout, but we should have a feedback loop which is missing - -and not a great layout - -Figure 5: Refinement of a flowchart for detecting rumors from Ma et al. (2017), highlighting improvements in clarity and completeness after refinement but emphasizing the continued absence of a critical feedback loop. - -Summary diagrams, SumMAF occasionally outperforms SeqMAF, indicating its strength in contexts requiring synthesized summarization of content. Since summarization-based refinement offers an integrated approach where multiple facets of feedback are considered simultaneously, this might have been more effective for diagrams requiring balanced presentation of dense information. -On SciDoc2DiagramBench-Extended, we explore whether human judgment perceives a significant enhancement in diagram quality across all categories on complex intents. We notice in Table 3, that even though there is a significant improvement of quality post-refinement on human judgement on Faithfulness and Completeness (Flowcharts and tables improve significantly), Layout-satisfaction is the least (which can be attributed to subjectivity and LLM’s inability to refine on specific criteria). - -RQ4: Is refinement more effective on simple images or complex images? -Across three datasets, the Multi-Aspect Refinement strategy consistently outperforms the few-shot approach in diagram production (Figure 4). On the simplest dataset, SciDoc2DiagramBench-Gold, MAF-Refine performs significantly better than few-shot approach, with improvements of 0.9 in completeness, 1.1 in faithfulness, and 0.8 in layout. As complexity increases in SciDoc2DiagramBench-Extended, MAF-Refine still performs strongly, showing improvements of 1.4 in completeness, 1.0 in faithfulness, and 0.5 in layout. Even in the most complex dataset, SciMultiDoc2DiagramBench-Extended, MAF-Refine leads with significant improvements, showing that MAF-Refine significantly boosts diagram quality across datasets and maintains its benefits even as complexity rises. - -6.2 Ablation Analysis - -Impact of individual components in Sci- Doc2Diagrammer. -Table 9 presents the performance impact of different components in the Sci- Doc2Diagrammer model using different metrics. -Removing the Question Generation/Answering (QG/QA) component results in significant performance drops across all metrics, highlighting its importance. The absence of the Summarization component also negatively affects the scores but to a lesser extent. - -Impact of Refinements in SciDoc2Diagrammer. -We conduct ablation analysis (Table 4) with - -SciDoc2Diagrammer-SeqMAF. In particular, we experimented with the removal of each refinement module on the SciDoc2DiagramBench-Gold dataset, and analyzed which component is useful to improve diagram generation. -Completeness removal shows a significant drop in both summary and flowchart metrics, emphasizing the importance of comprehensive overviews and detailed visual representations. Faithfulness removal leads to a significant performance drop in results and flowcharts, underlining the necessity of faithful representation for coherent diagram creation. But layout removal has the least impact on diagram quality improvement. - -6.3 Qualitative Analysis - -Qualitative analysis in feedback-based refinement (Figure 5) identifies gaps in the initial design, such as missing components or unclear integration of elements. After refinement, while the layout and completeness may improve significantly, issues like the absence of critical feedback loops can persist, leading to a less effective refinement process. This shows that even with visible improvements, missing critical features can detract from the model’s intended dynamic refinement functionality. - -Besides, taking a closer look at the diagrams generated before and after refinement (we sample the refined image having the highest average score rated by humans), we show in Figure 6, 21, 22 where our algorithm improves. For such qualitative analysis, we consider all four types of extrapolated images. The refined versions of the diagrams appear to show a higher level of detail and precision. -For example, in the block diagrams and flowcharts, the GPT4-o with refinement diagrams show clearer layouts and better alignment of elements compared to those produced by baselines. The labels and con- - -13349 - ---- PAGE 9 --- - -Showing a readable flowchart No Precise Text - -Bar chart showing the comparison of F1 score of Morfessor, MORSE (tuned on MC) and MORSE (tuned on SD17) on the canonical version of SD17 Paper:[MORSE: Semantic-ally Drive-n MORpheme SEgment-er](https://aclanthology.org/P17-1051) (Sakakini et al., ACL 2017) - -DALLE-3 Automatikz GPT4-o GPT4-o w/ refinement - -Figure 6: Qualitatively examines the images generated by different model-variants, where we show the incompleteness and non-faithfulness errors from our baseline models and how refinement improves the quality. - -Create a flowchart of constructing the propaganda tree datasets based on two reference Twitter datasets. -Paper: [Detect Rumors in Microblog Posts Using Propagation Structure via Kernel Learning](https://aclanthology.org/P17-1066) (Ma et al., ACL 2017) - -Not faithful to the intent at all Showing a bunch of things, but in an extremely cluttered way - -No Precise Text Faithful to the content, but the layout is not good - -nections between components are more accurate, suggesting that the refinement process improves the faithfulness and clarity of the diagrams. In scenarios where specific architectural details or complex data are necessary, the refined versions are closer to the original intent of diagram. This is particularly evident where GPT4-o with refinement accurately captures the necessary components and their interactions, unlike the outputs from baselines, which sometimes present a cluttered visualization. - -7 Related Work - -Our work connects to several distinct but interrelated fields, so we provide a comprehensive review of the most relevant prior work. -Text-to-Image Generation. Text-to-image generation models have seen immense success in recent years (Ramesh et al., 2022b; Zala et al., 2023; Rombach et al., 2022b; Feng et al., 2023), but a handful of them support text conditioning (Belouadi et al., 2023) on scientific images, and to the best of our knowledge, none of these explored challenges to incorporating in a long-text scenario. -Learning from Feedback. Madaan et al. (2023) first introduced ‘Self-Refine’ and then there had been multiple works exploring the use of natural language feedback to improve performance. Lee et al. (2024a) proposes a framework to reduce hal- - -Faithful to the content, but the layout is not good with overlapping axes labels - -Faithful to the content, more complete and the layout is better - -lucination in question answering. Ki and Carpuat (2024) exploits the strengths of LLMs and machine translation by guiding LLMs to automatically post-edit MT with external feedback on its quality. -Lee et al. (2024b) introduced a multimodal selffeedback guided revision model to reduce hallucination. The work most closely related to ours is that of Nathani et al. (2023). However, our approach diverges from theirs in a key aspect: our feedback modules are uniquely grounded in cross-verifying information from long texts. Furthermore, we introduce a novel algorithm that refines this process sequentially, which has proven to be more effective in our specific use case. We also discuss the related works on code generation in Appendix 18. - -8 Discussion and Future Work - -We introduce an innovative task for constructing - -scientific diagrams from scientific document discourse, commonly used in presentations/posters. -We establish a new benchmark using document-to- - -slides dataset and a sequential diagram refinementbased methodology to generate diagrams from academic papers. In future, we would like to assess the granularity of feedback that helps in diagram improvement by designing rubrics about what a good diagram should look like by communicating with the experts in the data visualization communities. - -13350 - ---- PAGE 10 --- - -Acknowledgement - -We thank the anonymous ARR reviewers and UMD - -CLIP members for their constructive comments and feedback on the draft. This material is based upon work supported by the National Science Foundation under Grant No. IIS-2403436 (Boyd-Graber). -Any opinions, findings, and conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the National Science Foundation. Annotations and support for Ishani Mondal were provided by a gift from Adobe Research. - -Limitations - -While our SciDoc2Diagrammer-MAF represents - -a significant advancement in automated diagram generation, some limitations should be noted: - -1. Small Dataset Size. The task of creating diagrams that are both factually correct and visually appealing, based on long academic papers and user intent, requires deep domain knowledge and expert input. In the construction of SciDoc2DiagramBench, expert human annotators with extensive experience in natural language processing (NLP) and diagram creation were employed to ensure that the diagrams met the required standards of completeness, faithfulness, and layout quality. Hence, the current version of SciDoc2DiagramBench includes 1080 diagrams from 89 ACL papers, which, although diverse, is relatively small. -2. Complexity in Layout Refinement. -As noted, our model struggles with improving the layout during the refinement process. This is particularly evident in complex visualizations where spatial and design elements are crucial. The current algorithms may not adequately capture aesthetic and functional aspects of layout design, impacting the overall clarity and effectiveness of the diagrams. -3. Feedback Mechanism Limitations. While the multi-aspect feedback mechanism aims to enhance diagram quality by focusing on completeness, faithfulness, and layout, the effectiveness of this feedback depends on the precision of the underlying evaluative criteria. -Misalignments in feedback interpretation can lead to overfitting on specific diagram aspects at the expense of others. - -Ethics Statement - -The experiments performed in this study involved human participants. All the experiments involving human evaluation were exempt under institutional IRB review. We recruited participants for our human study using Upwork and we have fairly compensated all the Upwork freelancers involved in this study, at an average rate of 20.00 USD per hour (respecting their suggested Upwork hourly wage). We did not collect any personal data during the experiments, and they could choose not to participate in the study. The documents used in the study are distributed under an open license. - -References - -2008–2024. -Grobid. -https://github.com/ kermitt2/grobid. - -Marah Abdin, Sam Ade Jacobs, Ammar Ahmad Awan, - -Jyoti Aneja, Ahmed Awadallah, Hany Awadalla, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Harkirat Behl, et al. 2024. Phi-3 technical report: A highly capable language model locally on your phone. arXiv preprint arXiv:2404.14219. - -Yoshia Abe, Tatsuya Daikoku, and Yasuo Kuniyoshi. - -2024. Assessing the aesthetic evaluation capabilities of gpt-4 with vision: Insights from group and individual assessments. - -Aishwarya Agrawal, Jiasen Lu, Stanislaw Antol, Mar- - -garet Mitchell, C. Lawrence Zitnick, Dhruv Batra, and Devi Parikh. 2016. Vqa: Visual question answering. - -Haozhe An, Christabel Acquaye, Colin Wang, Zongxia - -Li, and Rachel Rudinger. 2024. Do large language models discriminate in hiring decisions on the basis of race, ethnicity, and gender? - -Sören Auer, Dante AC Barone, Cassiano Bartz, Ed- - -uardo G Cortes, Mohamad Yaser Jaradeh, Oliver Karras, Manolis Koubarakis, Dmitry Mouromtsev, Dmitrii Pliukhin, Daniil Radyush, et al. 2023. The sciqa scientific question answering benchmark for scholarly knowledge. Scientific Reports, 13(1):7240. - -Jonas Belouadi, Anne Lauscher, and Steffen Eger. 2023. - -Automatikz: Text-guided synthesis of scientific vector graphics with tikz. ArXiv, abs/2310.00367. - -Mimi V Chapman, William J Hall, Robert Colby, and - -Laurel AG Sisler. 2014. -How images work: An analysis of a visual intervention used to facilitate a difficult conversation and promote understanding. -Qualitative Social Work, 13(4):456–476. - -Ting-Rui Chiang and Yun-Nung Chen. -2019. -Semantically-aligned equation generation for solving and reasoning math word problems. -In - -13351 - ---- PAGE 11 --- - -Proceedings of the 2019 Conference of the North American Chapter of the Association for Computa- - -tional Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pages 2656– - -2668, Minneapolis, Minnesota. Association for Computational Linguistics. - -Christopher Clark and Santosh Kumar Divvala. 2016. - -Pdffigures 2.0: Mining figures from research papers. 2016 IEEE/ACM Joint Conference on Digital Libraries (JCDL), pages 143–152. - -Alexander Fabbri, Irene Li, Prawat Trairatvorakul, Yi- - -jiao He, Weitai Ting, Robert Tung, Caitlin Westerfield, and Dragomir Radev. 2018. TutorialBank: A manually-collected corpus for prerequisite chains, survey extraction and resource recommendation. In Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: -Long Papers), pages 611–620, Melbourne, Australia. -Association for Computational Linguistics. - -Weixi Feng, Wanrong Zhu, Tsu jui Fu, Varun Jampani, - -Arjun Akula, Xuehai He, Sugato Basu, Xin Eric Wang, and William Yang Wang. 2023. Layoutgpt: -Compositional visual planning and generation with large language models. - -Tsu-Jui Fu, William Yang Wang, Daniel J. McDuff, and - -Yale Song. 2021. Doc2ppt: Automatic presentation - -slides generation from scientific documents. In AAAI Conference on Artificial Intelligence. - -Hanan Gani, Shariq Farooq Bhat, Muzammal Naseer, - -Salman Khan, and Peter Wonka. 2024. Llm blueprint: -Enabling text-to-image generation with complex and detailed prompts. - -Jack Hessel, Ari Holtzman, Maxwell Forbes, Ronan - -Le Bras, and Yejin Choi. 2021. -CLIPScore: A reference-free evaluation metric for image captioning. -In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pages 7514–7528, Online and Punta Cana, Dominican Republic. Association for Computational Linguistics. - -Ting-Yao Hsu, C Lee Giles, and Ting-Hao Huang. 2021. - -SciCap: Generating captions for scientific figures. -In Findings of the Association for Computational Linguistics: EMNLP 2021, pages 3258–3264, Punta Cana, Dominican Republic. Association for Computational Linguistics. - -Lei Huang, Weijiang Yu, Weitao Ma, Weihong Zhong, - -Zhangyin Feng, Haotian Wang, Qianglong Chen, Weihua Peng, Xiaocheng Feng, Bing Qin, and Ting Liu. 2023. A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions. - -Wei Huang, Xudong Ma, Haotong Qin, Xingyu Zheng, - -Chengtao Lv, Hong Chen, Jie Luo, Xiaojuan Qi, Xianglong Liu, and Michele Magno. 2024. How good are low-bit quantized llama3 models? an empirical study. - -Albert Q. Jiang, Alexandre Sablayrolles, Arthur Men- - -sch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, Lélio Renard Lavaud, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. 2023. Mistral 7b. - -Shankar Kantharaj, Rixie Tiffany Leong, Xiang Lin, - -Ahmed Masry, Megh Thakkar, Enamul Hoque, and Shafiq Joty. 2022. Chart-to-text: A large-scale benchmark for chart summarization. In Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pages 4005–4023, Dublin, Ireland. Association for Computational Linguistics. - -Dayeon Ki and Marine Carpuat. 2024. Guiding large - -language models to post-edit machine translation with error annotations. - -Anastasia Krithara, Anastasios Nentidis, Konstantinos - -Bougiatiotis, and Georgios Paliouras. 2023. Bioasqqa: A manually curated corpus for biomedical question answering. Scientific Data, 10(1):170. - -Dongyub Lee, Eunhwan Park, Hodong Lee, and - -Heuiseok Lim. 2024a. Ask, assess, and refine: Rectifying factual consistency and hallucination in LLMs with metric-guided feedback learning. In Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 2422–2433, St. Julian’s, Malta. Association for Computational Linguistics. - -Seongyun Lee, Hyunjae Kim, and Jaewoo Kang. 2023. - -Liquid: a framework for list question answering dataset generation. In Proceedings of the AAAI Conference on Artificial Intelligence, volume 37, pages 13014–13024. - -Seongyun Lee, Sue Hyun Park, Yongrae Jo, and Min- - -joon Seo. 2024b. Volcano: Mitigating multimodal hallucination through self-feedback guided revision. - -Patrick Lewis, Ethan Perez, Aleksandara Piktus, Fabio - -Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich Kuttler, Mike Lewis, Wen tau Yih, Tim Rocktäschel, Sebastian Riedel, and Douwe Kiela. 2020. -Retrieval-augmented generation for knowledgeintensive nlp tasks. ArXiv, abs/2005.11401. - -Kaixin Li, Yuchen Tian, Qisheng Hu, Ziyang Luo, and - -Jing Ma. 2024a. Mmcode: Evaluating multi-modal code large language models with visually rich programming problems. - -Zongxia Li, Andrew Mao, Daniel Stephens, Pranav - -Goel, Emily Walpole, Alden Dima, Juan Fung, and Jordan Boyd-Graber. 2024b. Improving the TENOR of labeling: Re-evaluating topic models for content analysis. In Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 840–859, St. Julian’s, Malta. Association for Computational Linguistics. - -13352 - ---- PAGE 12 --- - -Zongxia Li, Ishani Mondal, Yijun Liang, Huy Nghiem, - -and Jordan Lee Boyd-Graber. 2024c. Pedants: Cheap but effective answer equivalence. - -Chin-Yew Lin. 2004. ROUGE: A package for auto- - -matic evaluation of summaries. In Text Summarization Branches Out, pages 74–81, Barcelona, Spain. -Association for Computational Linguistics. - -Fangyu Liu, Julian Eisenschlos, Francesco Piccinno, - -Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, and Yasemin Altun. 2023a. DePlot: One-shot visual language reasoning by plot-to-table translation. In Findings of the Association for Computational Linguistics: ACL 2023, pages 10381–10399, Toronto, Canada. Association for Computational Linguistics. - -Fangyu Liu, Francesco Piccinno, Syrine Krichene, - -Chenxi Pang, Kenton Lee, Mandar Joshi, Yasemin Altun, Nigel Collier, and Julian Eisenschlos. 2023b. -MatCha: Enhancing visual language pretraining with math reasoning and chart derendering. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pages 12756–12770, Toronto, Canada. Association for Computational Linguistics. - -Jing Ma, Wei Gao, and Kam-Fai Wong. 2017. Detect - -rumors in microblog posts using propagation structure via kernel learning. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pages 708–717, Vancouver, Canada. Association for Computational Linguistics. - -Aman Madaan, Niket Tandon, Prakhar Gupta, Skyler - -Hallinan, Luyu Gao, Sarah Wiegreffe, Uri Alon, Nouha Dziri, Shrimai Prabhumoye, Yiming Yang, Shashank Gupta, Bodhisattwa Prasad Majumder, Katherine Hermann, Sean Welleck, Amir Yazdanbakhsh, and Peter Clark. 2023. Self-refine: Iterative refinement with self-feedback. - -Paula Maddigan and Teo Susnjak. 2023. Chat2vis: Gen- - -erating data visualisations via natural language using chatgpt, codex and gpt-3 large language models. Ieee Access. - -Ahmed Masry, Parsa Kavehzadeh, Xuan Long Do, Ena- - -mul Hoque, and Shafiq Joty. 2023. UniChart: A universal vision-language pretrained model for chart comprehension and reasoning. In Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pages 14662–14684, Singapore. Association for Computational Linguistics. - -Ishani Mondal, Yufang Hou, and Charles Jochim. 2021. - -End-to-end construction of NLP knowledge graph. -In Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021, pages 1885–1895, Online. Association for Computational Linguistics. - -Ishani Mondal, Shwetha S, Anandhavelu Natarajan, - -Aparna Garimella, Sambaran Bandyopadhyay, and - -Jordan Boyd-Graber. 2024. Presentations by the humans and for the humans: Harnessing LLMs for generating persona-aware slides from documents. In Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 2664–2684, St. Julian’s, Malta. Association for Computational Linguistics. - -Kazuma Murao, Ken Kobayashi, Hayato Kobayashi, - -Taichi Yatsuka, Takeshi Masuyama, Tatsuru Higurashi, and Yoshimune Tabuchi. 2019. A case study on neural headline generation for editing support. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computa- - -tional Linguistics: Human Language Technologies, Volume 2 (Industry Papers), pages 73–82, Minneapo- - -lis, Minnesota. Association for Computational Linguistics. - -Deepak Nathani, David Wang, Liangming Pan, and - -William Wang. 2023. MAF: Multi-aspect feedback for improving reasoning in large language models. -In Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pages 6591–6616, Singapore. Association for Computational Linguistics. - -Huy Nghiem, John Prindle, Jieyu Zhao, and Hal - -Daumé III au2. 2024. "you gotta be a doctor, lin": An investigation of name-based bias of large language models in employment recommendations. - -R OpenAI. 2023. Gpt-4 technical report. ArXiv, 2303. - -Lin Qiu, Hao Zhou, Yanru Qu, Weinan Zhang, Suo- - -heng Li, Shu Rong, Dongyu Ru, Lihua Qian, Kewei Tu, and Yong Yu. 2018. Qa4ie: A question answering based framework for information extraction. In The Semantic Web–ISWC 2018: 17th International Semantic Web Conference, Monterey, CA, USA, October 8–12, 2018, Proceedings, Part I 17, pages 198– 216. Springer. - -Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey - -Chu, and Mark Chen. 2022a. -Hierarchical textconditional image generation with clip latents. ArXiv, abs/2204.06125. - -Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey - -Chu, and Mark Chen. 2022b. -Hierarchical textconditional image generation with clip latents. - -Aditya Ramesh, Mikhail Pavlov, Gabriel Goh, Scott - -Gray, Chelsea Voss, Alec Radford, Mark Chen, and Ilya Sutskever. 2021. Zero-shot text-to-image generation. In Proceedings of the 38th International Conference on Machine Learning, volume 139 of Proceedings of Machine Learning Research, pages 8821–8831. PMLR. - -Marco Tulio Ribeiro, Tongshuang Wu, Carlos Guestrin, - -and Sameer Singh. 2020. -Beyond accuracy: Behavioral testing of NLP models with CheckList. In - -13353 - ---- PAGE 13 --- - -Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pages 4902– 4912, Online. Association for Computational Linguistics. - -R. Rombach, A. Blattmann, D. Lorenz, P. Esser, and - -B. Ommer. 2022a. High-resolution image synthesis with latent diffusion models. In 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pages 10674–10685, Los Alamitos, CA, USA. IEEE Computer Society. - -Robin Rombach, A. Blattmann, Dominik Lorenz, - -Patrick Esser, and Björn Ommer. 2021. -Highresolution image synthesis with latent diffusion models. 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pages 10674– 10685. - -Robin Rombach, Andreas Blattmann, Dominik Lorenz, - -Patrick Esser, and Björn Ommer. 2022b. -Highresolution image synthesis with latent diffusion models. In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pages 10684–10695. - -Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten - -Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Romain Sauvestre, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, and Gabriel Synnaeve. 2024. Code llama: Open foundation models for code. - -Tarek Sakakini, Suma Bhat, and Pramod Viswanath. - -2017. MORSE: Semantic-ally drive-n MORpheme SEgment-er. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pages 552–561, Vancouver, - -Canada. Association for Computational Linguistics. - -Shreya Shankar, J. D. Zamfirescu-Pereira, Björn Hart- - -mann, Aditya G. Parameswaran, and Ian Arawjo. -2024. Who validates the validators? aligning llmassisted evaluation of llm outputs with human preferences. - -Chufan Shi, Cheng Yang, Yaxin Liu, Bo Shui, Junjie - -Wang, Mohan Jing, Linran Xu, Xinyu Zhu, Siheng Li, Yuxiang Zhang, Gongye Liu, Xiaomei Nie, Deng Cai, and Yujiu Yang. 2024. Chartmimic: Evaluating lmm’s cross-modal reasoning capability via chart-tocode generation. - -Chenglei Si, Yanzhe Zhang, Zhengyuan Yang, Ruibo - -Liu, and Diyi Yang. 2024. Design2code: How far are we from automating front-end engineering? - -Edward Sun, Yufang Hou, Dakuo Wang, Yunfeng - -Zhang, and Nancy X. R. Wang. 2021. -D2S: -Document-to-slide generation via query-based text - -summarization. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pages 1405–1418, Online. Association for Computational Linguistics. - -Yi Tay, Anh Tuan Luu, Siu Cheung Hui, and Jian - -Su. 2018. Reasoning with sarcasm by reading inbetween. In Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pages 1010–1020, Melbourne, Australia. Association for Computational Linguistics. - -Hugo Touvron, Louis Martin, Kevin Stone, Peter Al- - -bert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez, Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushkar Mishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric Michael Smith, Ranjan Subramanian, Xiaoqing Ellen Tan, Binh Tang, Ross Taylor, Adina Williams, Jian Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, and Thomas Scialom. 2023. Llama 2: Open foundation and finetuned chat models. - -Shubham Ugare, Tarun Suresh, Hangoo Kang, Sasa Mi- - -sailovic, and Gagandeep Singh. 2024. Improving llm code generation with grammar augmentation. arXiv preprint arXiv:2403.01632. - -Yifan Yao, Jinhao Duan, Kaidi Xu, Yuanfang Cai, Zhibo - -Sun, and Yue Zhang. 2024. A survey on large language model (llm) security and privacy: The good, the bad, and the ugly. High-Confidence Computing, 4(2):100211. - -Abhay Zala, Han Lin, Jaemin Cho, and Mohit Bansal. - -2023. Diagrammergpt: Generating open-domain, open-platform diagrams via llm planning. - -Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. - -Weinberger, and Yoav Artzi. 2019. -Bertscore: -Evaluating text generation with BERT. -CoRR, abs/1904.09675. - -Xinlu Zhang, Yujie Lu, Weizhi Wang, An Yan, Jun Yan, - -Lianke Qin, Heng Wang, Xifeng Yan, William Yang Wang, and Linda Ruth Petzold. 2023. Gpt-4v(ision) as a generalist evaluator for vision-language tasks. - -Tiancheng Zhao, Ran Zhao, and Maxine Eskenazi. 2017. - -Learning discourse-level diversity for neural dialog models using conditional variational autoencoders. - -13354 - ---- PAGE 14 --- - -In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pages 654–664, Vancouver, Canada. -Association for Computational Linguistics. - -Yuhang Zhou, Jing Zhu, Paiheng Xu, Xiaoyu Liu, Xiyao - -Wang, Danai Koutra, Wei Ai, and Furong Huang. -2024. Multi-stage balanced distillation: Addressing long-tail challenges in sequence-level knowledge distillation. arXiv preprint arXiv:2406.13114. - -9 User-Study Guided Motivation of Task - -Why do we need this Task? -Our motivation is primarily inspired by the SlideGeneration dataset (Fu et al., 2021), which we have already cited in Footnote 1 of the paper, indicating user’s need. -However, to better motivate our task, we survey 11 participants (Instructions in Figure 7), ranging from beginners to experts in NLP, comprising of 6 MS/PhD students with at least 1 *CL or ML conference paper, 2 people from ML research lab, 3 beginners in NLP. We asked them to review 3 *CL papers each and determine if additional visuals were needed for slide presentations “Would you like to create some more figures that you think would be required for better visualization in the slides, which are not present inside the paper directly? If yes, can you provide some description of these additional images?” Out of 33 responses (11*3) from the papers, 28 answers (84.8%) indicate the need for additional visualization in slides. - -The survey shows that each paper requires an average of 2.3 “Extrapolated-Diagrams”, with a focus on flowcharts, converting complex tables into charts, and synthesizing related work to highlight novelty. These findings indicate the potential of including such diagrams in our new dataset, which aligns with the additional visuals found in (Fu et al., 2021). - -10 Effectiveness of our DiagramGeneration Pipeline - -Based on the survey responses in 9, we selected the paper requiring the most number of “Extrapolated- Diagrams” and asked two participants from the set of 11 people (Control Group) to create a flowchart and a bar chart from scratch after reading the paper content. On average, it took them 30-35 minutes per diagram. We then provided two other participants (Experimental Group) with access to our interactive SciDoc2Diagrammer-MAF system to create the same set of diagrams (a flowchart and - -a bar chart). With our tool, they completed each diagram in 8-10 minutes. We also provided intermediate questions and answers to extract data from the paper (Figure 8), code (Figure 9) which helped reduce the cognitive load by simplifying the process of diagram creation. The participants found it easier to tweak a template diagram generated through code, instead of searching for the relevant data from paper, and writing a code from scratch to generate figures (Sample shown in (Figure 10)). - -11 Hiring Upwork Participants - -11.1 Hiring Workers for Dataset Creation - -Using Upwork, we hired three workers familiar with Machine learning and NLP with almost 5 years of experience and well-versed with creating presentations from documents, sorted by having a skill set of Presentation making. The hiring was made after shortlisting them through interviews, where they were initially asked to read the paper (?) and answer questions like : 1) What is the novelty of this approach? 2) What is the motivation behind the main algorithm? 3) What are the strengths and weaknesses of this paper? 4) What was the state-of-art algorithm before this model came in? -5) What kind of evaluation has been made using this approach? Moreover, they were asked to make a presentation suitable for presenting it in an AI conference. Based on their answers and the quality of the presentation being made, the first two authors of the paper made a hiring decision. - -12 Instruction Guidelines on Annotating and Identifying Extrapolated Diagrams - -Objective: You are provided with a dataset containing scientific papers and their corresponding slide decks. Your task is to annotate each slide in the paper according to predefined categories based on the content origin and presentation style. - -12.1 Categories: - -1. Mere Extraction from the Paper: Slides that directly copy and paste content from the paper. -2. Extrapolated Diagrams: Slides that include more elaborative visualizations or explanations beyond mere copying. This includes: - -• Diagrams with textual explanations, highlights, or key takeaways. - -13355 - ---- PAGE 15 --- - -Figure 7: User Study Survey Form to Understand how many people would like to create new Figures from a few selected *CL papers while creating presentations - -• Flowcharts that summarize or clarify the content. -• Examples that illustrate theoretical concepts. -• Conversions of tabular data into graphical representations. -3. External: Slides containing content sourced from external references outside of the paper. - -Procedure: - -1. Initial Check: Begin by reviewing the slide deck to determine if there are any "Extrapolated Diagrams". Only proceed with the annotation if such slides are present. -2. Comprehensive Review: Read through the entire paper to understand the context and origin of images and diagrams used in the slides. -3. Annotation: Classify each slide into one of the categories based on your understanding and findings. Consider the source and nature of the content on each slide. - -Ensure accuracy in your annotations by thoroughly comparing slide content with the original paper and other sources mentioned within the slides. - -13 Instructions on Annotating Intent of the Diagram Creation - -Objective: Your role is to provide clear, comprehensive, and complete descriptions for diagrams in scientific slide decks, focusing on their intent and content. This will enable any individual to understand or recreate the diagram based purely on your description. - -Steps to Follow: - -1. Identify the Diagram: Start by identifying whether the slide includes bar charts, plots, or tables. Note the context in which these diagrams are used, such as summarizing results or discussing related works. -2. Determine the Intent: -– What is the diagram illustrating? Under- - -stand what the main point or data the diagram conveys. -– Why is this diagram used? Analyze why - -the author chose this type of diagram to present the data. -– How does the diagram achieve its pur- - -pose? Look at the elements of the diagram like axes, labels, legends, and how they contribute to conveying the intended message. - -13356 - ---- PAGE 16 --- - -Figure 8: Shows the Page 1 of the DiagramGeneration Interface. The left panel displays a scientific document titled "Retrieve, Rerank and Rewrite: Soft Template Based Neural Summarization," including the title, authors, affiliations, and abstract. The right panel shows the interface used to generate diagrams based on user intent. -The first section captures the "Intent of the Diagram," which in this case is "Create a flowchart for the procedure to explain Soft Template Based Neural Summarization." The second section includes "Data Extracted from the Multimodal Content of the Document," while the third section displays "Model-generated Clarification Questions for Creating the Diagram," asking for additional information such as figure names in comma-separated format. -This layout illustrates a pipeline where user input and model clarifications are integrated to guide the generation of scientific diagrams. - -3. Write the Description: -– Your description should allow someone - -who hasn’t seen the diagram to visualize and possibly recreate it accurately. -– Include details about: - -- Data Representation: What data is - -being represented and how? -- Layout and Design: Describe the - -layout, color scheme, and any textual elements. -- Interpretative Elements: Explain - -any elements that help in interpreting the data, such as legends, scales, or reference lines. -4. Verify Completeness: Review your description to ensure it captures all aspects of the diagram comprehensively. - -Example of a Good Intent: - -“Create a bar chart titled ’Annual Growth’ display- - -ing the yearly sales growth from 2015 to 2020. -Each bar represents a year, colored in gradient from dark blue (2015) to light blue (2020). The - -Y-axis is labeled ’Percentage Growth’ and ranges - -from 0% to 50%, increasing by 5% increments. -A legend in the bottom right explains the color gradient as representing successive years.” - -Note: Ensure that your descriptions are free of assumptions and interpretations beyond what is explicitly shown in the diagrams. - -14 Addressing Data Leakage Concerns - -We address the potential data leakage due to the use - -of existing datasets such as DOC2PPT (Fu et al., 2021) and TutorialBank (Fabbri et al., 2018) in constructing the SciDoc2DiagramBench by clarifying that our task is fundamentally distinct from these datasets’ original objectives, such as slide creation. Specifically, SciDoc2Diagram aims to evaluate large language models’ (LLMs) ability to generate scientific diagrams conditioned on user intent and long-document context, which presents a more nuanced and challenging task than the straightforward generation of slides from text. - -Our task focuses on creating scientific diagrams - -13357 - ---- PAGE 17 --- - -Figure 9: shows the Page 2 of DiagramGeneration Interface. The left panel shows a scientific paper titled "Retrieve, Rerank and Rewrite: Soft Template Based Neural Summarization," including the abstract and introduction sections. -The middle panel contains a code snippet that represents a Graphviz script for creating the diagram. The script allows users to manually modify the code to customize the flowchart. The right panel displays the generated flowchart based on the provided code. The flowchart visualizes the steps involved in the summarization process. - -based on detailed user instructions and content from the academic paper itself. This involves synthesizing complex ideas into visual form, which is substantially different from simply converting scientific text into slides. As such, the focus shifts from summarization or slide creation to a more advanced task of diagrammatic representation, ensuring that any overlap with DOC2PPT (Fu et al., 2021) or TutorialBank (Fabbri et al., 2018) is minimized. Additionally, no direct train-test splits are used, and our dataset is crafted to assess the ability to generalize across varied diagrammatic intents. - -Besides, it is worth noting that many early benchmarks in the field have been developed by reusing existing datasets or open-source repositories to create new tasks. Examples include works like Automatikz (Belouadi et al., 2023), PersonaD2S (Mondal et al., 2024), and E2EKG (Mondal et al., 2021) which repurpose existing data to establish more specific, challenging benchmarks. Our work follows a similar approach, using existing resources to create a more complex benchmark aimed at evaluating how diagrams can be generated not only based on the academic paper but also in line with the specific user intent, a dimension absent in DOC2PPT and TutorialBank. - -To further mitigate concerns about data leakage and to enhance the diversity and representativeness of the SciDoc2DiagramBench, we have incorporated expert human annotations. These annotations include additional metadata such as the intent behind the created diagram. The inclusion of this metadata makes each instance more unique and specifically tailored to the task of diagram creation, thereby distinguishing it from simple text-to-slide generation datasets. This human-curated layer, not present in DOC2PPT or TutorialBank, ensures that our dataset adds substantial value and differentiation from prior resources. - -In conclusion, while we leverage existing datasets for constructing SciDoc2DiagramBench, the way we have formulated the task introduces a significant layer of complexity and novelty. By focusing on user intent, incorporating human annotations, and avoiding standard train-test splits, we ensure that SciDoc2DiagramBench remains a challenging and representative benchmark that builds on existing resources while addressing concerns about potential data leakage. Besides, a significant number of early researchers have created benchmarks by reusing existing datasets or open-source repositories such as Automatikz (Belouadi et al., - -13358 - ---- PAGE 18 --- - -Figure 10: displays a feedback form designed to allow users to evaluate and improve the generated diagram and its underlying code. Users can provide ratings for the completeness and correctness of the diagram on a scale from 1 to 5. Additionally, users can ask clarification questions to verify the completeness of the generated content by submitting their queries in a comma-separated format. They can also input correct data or corrections in the text box provided to regenerate the code. The interface includes buttons to either "Regenerate" or "Save" the revised code based on user input. This tool enables interactive refinement of generated diagrams based on user feedback. - -2023), PersonaD2S (Mondal et al., 2024), Endto-End Scientific Knowledge Graph Construction (Mondal et al., 2021), VQA (Agrawal et al., 2016). - -15 Overview of SciDoc2DiagramBench - -15.1 Examples from SciMultiDoc2DiagramBench-Gold - -Figure 13 corresponds to Intent “Construct a detailed and organized table to summarize various human-generated datasets used in natural language processing research which includes the name of each dataset, the annotators involved, the format of the data, and the size of the dataset.” - -Figure 14 (Extrapolated-Summary) corresponds to Intent “Construct a summary table showing the comparison of exisiting studies where you should specify the source paper, tables, KnowledgeBase and the target task on which it is based on’ and the source Papers are As mentioned in the slides.” - -15.2 Examples from SciDoc2DiagramBench-Gold - -Figure 15 (Extrapolated-Flowchart) corresponds to Intent “Construct a flowchart or diagram showing where the same context is provided to both humans and models, the human creates some response and the model creates other response along with the equations of precision and recall for appropriateness and diversity” and the source paper is (Zhao et al., 2017). - -Figure 16 (Extrapolated-Results) corresponds to Intent “Create a Bar chart showing the comparison of F1 score of MORSE, Morfessor S+W, Morfessor S+W+L and MorphoChain against published stateof-the-art results” and the source paper is (Sakakini et al., 2017). - -16 Details of the Evaluation Metrics used - -BERTScore (Zhang et al., 2019) measures the semantic similarity between the captions generated by GPT-4V for gold-standard diagrams and those for our diagrams, assessing how well our visual representations capture the intended mean- - -13359 - ---- PAGE 19 --- - -Types Description Mere Extraction from the Paper Directly copied and pasted into slides Extrapolated Diagrams 1) Explain a figure with text/highlights/takeaways, 2) Flowcharts, 3) Explanation through example, 4) Table to Graph or some other forms of visualization External Included from other sources (papers / web) - -Figure 11: shows a user interface where the generated flowchart is displayed at the top. The flowchart outlines the process for soft template-based neural summarization, detailing steps such as document input, retrieving candidate soft templates, extending the Seq2Seq framework, and generating a readable and stable summary. Below the diagram, the user is provided with a feedback section where they can rate the layout on a scale from 1 to 5 and provide remarks regarding the visual layout of the flowchart. In this example, the user suggests using a different contrast of colors and improving the figure’s lock. At the bottom, the interface displays the code used to generate the diagram, written in Python using the Graphviz package, allowing for further edits or regeneration of the diagram based on user input. This setup encourages interactive refinement and iterative improvements. - -Types Intent Source Paper Extrapolated- Flowchart - -Create a flowchart to explain the process of how the proposed model understands the semantics of problems and decides which symbol to generate next." - -Extrapolated- Summary - -Create a table to summarize related works in sarcasm detection, highlighting the novelty of the proposed model in terms of its ability to model contrast and incongruity - -Extrapolated- Results - -Convert the table describing the different translations of titles into a bar chart, grouping each translation by its source (Headline, Lead, Editor, OpenNMT, HybridFusion), and using color coding to highlight differences in the length and tone of each translation - -Table 5: Some Examples from SciDoc2DiagramBench-Extended - -Table 6: Types of Diagrams and Their Descriptions - -(Chiang and Chen, 2019) - -(Tay et al., 2018) - -(Murao et al., 2019) - -13360 - ---- PAGE 20 --- - -Table 7: Automatic evaluation of models on various diagrams on SciMultiDoc2DiagramBench-Gold using ROUGE (R), BERTScore (BS), and CLIPScore (CS). The table highlights the comparative performance of DALLE- 3, Automatikz, and several versions of SciDoc2Diagrammer (Few-shot Strategy) across all categories of diagrams. -SciDoc2Diagrammer with GPT4-o consistently shows the best performance across most categories, indicated by the green cells, while DALLE-3 generally underperforms, as shown by the red cells. - -Models (IV) R BS CS R BS CS R BS CS R BS CS - -DALLE-3 0.10 0.23 0.11 0.03 0.21 0.23 0.11 0.25 0.16 0.03 0.12 0.13 Automatikz 0.21 0.28 0.21 0.12 0.43 0.17 0.13 0.54 0.45 0.49 0.34 0.27 - -SciDoc2Diagrammer - -w/ GPT4-o (ZS) 0.22 0.36 0.40 0.23 0.45 0.29 0.21 0.46 0.29 0.28 0.47 0.50 w/ GPT4-o (FS) 0.28 0.47 0.43 0.33 0.50 0.38 0.24 0.48 0.39 0.42 0.54 0.52 w/ GPT4-o-SR 0.34 0.62 0.42 0.42 0.51 0.35 0.33 0.61 0.50 0.47 0.62 0.60 w/ GPT4-o-SumMAF 0.35 0.74 0.48 0.50 0.57 0.39 0.34 0.64 0.49 0.50 0.74 0.74 w/ GPT4-o-SeqMAF 0.39 0.79 0.53 0.49 0.49 0.36 0.37 0.58 0.49 0.45 0.67 0.62 - -ings. ROUGE-1 (Lin, 2004) examines 1-gram overlap between these sets of captions, providing insight into the textual alignment with expected content. CLIPScore (Hessel et al., 2021) evaluates alignment between gold-standard images and corresponding GPT-4V captions for our generated images, indicating how closely our outputs match visual and contextual expectations. - -17 Additional Experiments - -Individual Components are necessary in Sci- Doc2Diagrammer! -Table 9 presents the performance impact of different components in the Sci- Doc2Diagrammer model using several metrics: - -Figure 12: Page 5 of the DiagramGeneration Interface - -Extrapolated-Flowchart Extrapolated-Results Extrapolated-Architecture Extrapolated-Summary - -CLIPScore (CS), BERTScore (BS), Completeness (C), Faithfulness (F), and Layout (L). Removing the Question Generation/Answering (QG/QA) component results in significant performance drops across all metrics, highlighting its importance. The absence of the Summarization component also negatively affects the scores but to a lesser extent. - -18 Comparison with Contemporary Research on code generation - -Unlike ChartMimic (Shi et al., 2024) and MM- Code (Li et al., 2024a), which primarily focus on either visual charts or programming challenges as input, our approach uses the high-level intent pro- - -13361 - ---- PAGE 21 --- - -Figure 13: An Example from SciMultiDoc2DiagramBench-Gold where the Intent is “Construct a detailed and organized table to summarize various human-generated datasets used in natural language processing research which includes the name of each dataset, the annotators involved, the format of the data, and the size of the dataset.,” and the source Papers are As mentioned in the slides. - -Figure 14: An Example from SciMultiDoc2DiagramBench-Gold where the Intent is “Construct a summary table showing the comparison of exisiting studies where you should specify the source paper, tables, KnowledgeBase and the target task on which it is based on’ and the source Papers are As mentioned in the slides.” - -vided by the user to generate complex scientific diagrams. This makes our work distinct as it emphasizes understanding longer document contexts and user-specific requests rather than predefined visual or mathematical data. - -Existing works such as MMCode (Li et al., 2024a) and Design2Code (Si et al., 2024) operate on well-defined inputs like programming challenges or website designs. In contrast, our approach supports the processing of long scientific - -13362 - ---- PAGE 22 --- - -Figure 15: An Example from SciDoc2DiagramBench-Gold where the Intent is “Construct a flowchart or diagram showing where the same context is provided to both humans and models, the human creates some response and the model creates other response along with the equations of precision and recall for appropriateness and diversity ..” and the source paper is (Zhao et al., 2017) - -Figure 16: An Example from SciDoc2DiagramBench-Gold where the Intent is “Create a Bar chart showing the comparison of F1 score of MORSE, Morfessor S+W, Morfessor S+W+L and MorphoChain against published state-of-the-art results” and the source paper is (Sakakini et al., 2017) - -documents, capturing intricate diagram intent from user-specified slides or text sources, making our method applicable for use in research papers and presentations. - -While ChartMimic and MMCode focus on gen- - -erating code that addresses visual understanding or mathematical problems, our method generates scientific diagrams for seamless integration into academic presentations. This involves translating user intent into structured diagrams that comply with - -13363 - ---- PAGE 23 --- - -Figure 17: An Example from SciDoc2DiagramBench-Gold where the Intent is “Create a Bar chart showing the comparison of F1 score of MORSE, Morfessor S+W, Morfessor S+W+L and MorphoChain against published state-of-the-art results” and the source paper is (Zhao et al., 2017) - -Figure 18: A busy scientist is inclined to automate the creation of diagrams, where our refinement-based approach in SciDocDiagrammarMAF yields a complete and faithful diagram. - -Oh no! I only have an hour before my presentation. - -How can I quickly create factually correct and informative visuals from the papers I need to present? - -You should first try out the existing text-to-image models with a textual description as your prompt - -SciDoc2Diagr - -Automat - -Stable Diffusion - -ammer w/ - -ikz - -GPT4 - -Source PDF - -scientific standards. To the best of our knowledge, none of the previous approaches have specifically addressed the challenge of multimodal code generation for scientific diagrams with the focus on preserving layout, completeness, and faithfulness in a highly structured and accurate form. This gap is what SciDoc2DiagramBench fills, making it a novel contribution. - -Okay, let me “Create a flowchart showing the methodology of how similar context is used for human and model response generation and the way it has to be evaluated” (Intent) - -SciDoc2Diagrammer-MAF - -Include Equations that need to be present for showing evaluation - -C - -- Show clear input and outputs from humans and models - Precision and Recall equations have to be included - Arrows should be directed in nature - -F - -L - -- There should not be overlapping nodes - There should not be arrow markers inside the node - -In addition, human-computer interaction is an emerging topic as artificial intelligence (AI) advances in diverse tasks. However, AI still faces challenges such as gender and race discrimination (Nghiem et al., 2024; An et al., 2024), producing unfaithful information through hallucinations (Huang et al., 2023), privacy breaches (Yao et al., 2024; Zhou et al., 2024), etc. Fully rely- - -13364 - ---- PAGE 24 --- - -DALLE3 with GPT4-o (Rombach et al., 2021) Uses GPT4-o output for diagram planning and diagram rendering using DALLE3 in SciDoc2Diagrammer. -Automatikz (Belouadi et al., 2023) Uses GPT4-o output for diagram planning and diagram rendering using Clima-13B (https://huggingface.co/nllg/tikz-clima-13b/) in SciDoc2Diagrammer. -Zero-shot SciDoc2Diagrammer w/ Phi-3 (Abdin et al., 2024) - -Implements Phi-3 (https://huggingface.co/microsoft/Phi-3-vision-128k-instruct) for diagram planning and GPT4-o for diagram rendering with code generation. -Few-shot SciDoc2Diagrammer w/ Phi-3 (Abdin et al., 2024; Rozière et al., 2024) - -Utilizes GPT4-o for diagram planning and diagram rendering in SciDoc2Diagrammer. -Few-shot SciDoc2Diagrammer w/ GPT4-o (OpenAI, 2023) - -Table 9: Influence of components in SciDoc2Diagrammer evaluated on SciDoc2DiagramBench-Gold using metrics such as CLIPScore (CS), BERTScore (BS), Completeness (C), Faithfulness (F), and Layout (L) using GPT4-Evaluation, showing that QA/QG and Information summarization contributes to the best performance across all metrics. - -Strategy Description - -Zero-shot SciDoc2Diagrammer w/ Mistral (https://huggingface.co/mistralai/Mistral- 7B-Instruct-v0.2) (Jiang et al., 2023) - -Few-shot SciDoc2Diagrammer w/ Mistral (Jiang et al., 2023; Rozière et al., 2024) - -Zero-shot SciDoc2Diagrammer w/ GPT4-o (OpenAI, 2023) - -Table 8: Description of Base and Refiner LMs - -ing on LLMs for diagram generation may overlook specific user intentions and use cases. Moreover, users learn their mental models of the data and refine their criteria and results through the interaction with AI (Shankar et al., 2024; Li et al., 2024b). Future work could focus on interactive diagram generation, allowing users to iteratively engage with AI to refine and generate diagrams that best suit their specific use cases. - -Like Zero-shot but uses 3 examples for guidance. - -Uses Mistral for diagram planning and GPT4-o for diagram rendering with code generation. - -Extends Zero-shot by using 3 example inputs. - -Adds 3 exemplars to GPT4-o based generation process. - -Models CS BS C F L SciDoc2Diagrammer 0.51 0.61 3.8 4.3 3.8 - w/o QG/QA 0.34 0.49 2.8 4.0 3.7 - w/o Summarization 0.43 0.53 3.2 4.0 3.8 - -13365 - ---- PAGE 25 --- - -Error Propagation from which step Missing Step or Node Omission of a crucial process step or component - -Results/plots Data Extraction, IE to Code Generation Wrong Reasoning Logical errors leading to incorrect conclusions - -Table 10: Comprehensive Error Taxonomy which categorizes the common errors identified on a randomly selected set of 50 generated diagrams from scientific documents from the test set of SciDoc2DiagramBench w/ the best performing GPT4-o which had GPT4-Eval score less than 3.5 detailing the nature of each error. - -Error Taxonomy in Generated Diagrams - -Definition of Error Categories - -Missing Arrow Absence of an arrow that indicates flow between elements - -Misleading Information - -Incorrect details that contradict the intended data/narrative - -Intent Mismatch Diagram does not align with user-defined intent - -Incorrect Order of Nodes - -Nodes appear in an incorrect sequence - -Wrong Numerical Value/Trends - -Numerical inaccuracies or incorrect trends - -Wrong Axes Labels Axes mislabeling, including incorrect titles or units - -Incomplete Reasoning Diagrams that leave out critical aspects of analysis - -Unreadable Fonts Fonts that are difficult to read, diminishing accessibility - -Overlapping Legends or Title of the Text - -Legends or titles overlap with other elements, obscuring information - -Incorrect Bounding Box - -Poorly sized/misplaced bounding boxes - -Prevalence in the Type of Diagrams - -Tables Summary, Flowcharts - -IE to Code Generation - -Flowcharts IE to Code Generation - -Flowcharts, Summary Tables - -IE to Code Generation - -All Types Intent Classification - -Flowcharts IE to Code Generation - -Results/plots IE to Code Generation - -Results/plots Data Extraction, IE to Code Generation - -Results/plots, Summary Tables - -IE to Code Generation - -Results/plots, Flowcharts - -IE to Code Generation - -Results/plots, Flowcharts - -IE to Code Generation - -Explanation of Figures IE to Code Generation - -13366 - ---- PAGE 26 --- - -Figure 19: Humans Rated on the diagrams generated by the SciDoc2Diagrammer on three evaluation criteria. We observe that GPT4-o is clearly the winner model. - -Figure 20: Faithfulness evaluated on a sample of 30 diagrams by two annotators (On a Likert Scale 1-5) when shown the original figure and the generated figure. - -13367 - ---- PAGE 27 --- - -Table 11: Average Completeness (C), Faithfulness (F), and Layout (L) comparing the performance of models, evaluated by both GPT4-Vision and human evaluators. - -Table 13: Automatic evaluation of models on various diagrams on SciDoc2DiagramBench-Gold using ROUGE (R), BERTScore (BS), and CLIPScore (CS). The table highlights the comparative performance of DALLE-3, Automatikz, and several versions of SciDoc2Diagrammer (Few-shot Strategy) across all categories of diagrams. -SciDoc2Diagrammer with GPT4-o consistently shows the best performance across most categories, indicated by the green cells, while DALLE-3 generally underperforms, as shown by the red cells. We observe that GPT4-o is clearly the winner model. - -Critic Base LM and Variants C F L - -GPT4-V - -Human - -Table 12: LLM-Human Agreement Scores for Refined vs. Non-Refined Images - -Models (IV) R BS CS R BS CS R BS CS R BS CS - -DALLE-3 0.12 0.45 0.23 0.11 0.34 0.21 0.25 0.45 0.25 0.04 0.10 0.18 Automatikz 0.20 0.48 0.25 0.20 0.50 0.29 0.24 0.45 0.28 0.27 0.56 0.45 - -SciDoc2Diagrammer - -w/ LLama2 0.28 0.58 0.40 0.28 0.39 0.28 0.24 0.48 0.45 0.45 0.58 0.54 w/ Phi-3 0.26 0.66 0.52 0.32 0.47 0.38 0.28 0.41 0.38 0.40 0.67 0.56 w/ Mistral 0.26 0.64 0.45 0.39 0.53 0.34 0.29 0.48 0.42 0.39 0.59 0.62 w/ GPT4-o 0.28 0.67 0.43 0.40 0.56 0.38 0.32 0.57 0.45 0.45 0.67 0.62 - -DALLE3 1.5 1.0 2.0 Automatikz 2.6 3.2 3.5 Phi-3 3.4 4.3 3.5 GPT4o 3.8 4.3 3.8 GPT4o + MAF 4.6 4.7 4.6 - -DALLE3 1 1.2 2.1 Automatikz 2.8 2.4 3.2 Phi-3 3.5 3.4 3.1 GPT4o 3.6 4.0 3.4 GPT4o + MAF 4.2 4.4 3.6 - -Refinement Completeness Faithfulness Layout w/ MAF 81% 75% 56% w/ SR 51% 54% 51% - -Extrapolated-Flowchart Extrapolated-Results Extrapolated-Architecture Extrapolated-Summary - -Figure 21: Qual Examples - -13368 - ---- PAGE 28 --- - -2: Decompose I and generate a set of questions Q based on the decomposed intent - -4: -ai ←extract(qi, O) ▷Get answer determining completeness of O - -5: -si ←[ai is adequately represented in O] ▷Assign Score from 1-5 - -i=1 si ▷Calculate average score - -Algorithm 1 Completeness Assessment (Ccritic) Require: O: Diagram to be evaluated, Code: Associated Code to be Checked Require: PDF: Document source Require: I: Defined intent for diagram creation Ensure: C: Completeness score of O and CFeedback to improve the score - -1: S = [] - -3: for each question qi in Q do - -6: -Append si to score list S - -∑︁|Q| - -7: CScore ← 1 |Q| - -8: CFeedback ←textual feedback to improve C - -9: return CScore and CFeedback - -Require: D: Diagram to be evaluated Require: R: Design rules for layout and aesthetics Ensure: S: Aesthetics score of diagram D Ensure: F: Feedback for improving D - -1: Function EvaluateLayout(D, R) - -2: S ←0 - -3: Feedback ←[] - -4: for each rule r in R do - -5: -result, comment ←APPLYRULE(r, D) - -6: -S ←S + result - -7: -Feedback.append(comment) - -8: return S, Feedback - -Figure 22: Qual Examples - -13369 - ---- PAGE 29 --- - -Figure 23: Zero-Shot Evaluation of SciDoc2Diagrammer with various LLMs using Automatic Evaluation Metrics. -We observe that GPT4-o is clearly the winner model. - -1: Using O and Code, generate questions that would validate the correctness of O - -i ←retrieve(qi, PDF) ▷Retrieve answer from PDF - -i ←extract(qi, O) ▷Retrieve answer from diagram O - -i=1 si ▷Calculate average score - -Algorithm 2 Faithfulness Assessment (Fcritic) Require: O: Diagram to be evaluated and corresponding code Code Require: PDF: Document source Require: I: Defined intent for diagram creation Ensure: F: Faithfulness score of O and FFeedback to improve the score - -2: S = [] - -3: for each question qi in Q do - -4: -apdf - -5: -aimage - -6: -si ←[aimage - -i == apdf - -7: -Append si to score list S - -∑︁|Q| - -8: Fscore ← 1 |Q| - -9: FFeedback ←textual feedback to improve C - -10: return FScore and FFeedback - -i ] ▷ ▷Assign Score from 1-5 - -13370 - ---- PAGE 30 --- - -7: -feedback ←Evaluate(D, P, type) ▷Evaluation based on type - -9: -break ▷Exit loop if score exceeds the threshold - -10: -D ←RefineDiagram(D, feedback) ▷Refine diagram based on feedback - -11: -score ←Score(feedback) ▷Update score based on feedback - -Algorithm 3 Sequential Refinement of Diagrams Require: Diagram D, Source PDF P Ensure: Refined Diagram Dref - -1: for type in {’Completeness’, ’Faithfulness’, ’Layout’} do - -2: -maxIterations ←3 - -3: -threshold ←4.5 - -4: -iteration ←0 - -5: -score ←0 - -6: -while iteration ≤maxIterations and score < threshold do - -8: -if Score(feedback) > threshold then - -12: -iteration ←iteration + 1 - -13: return D - -Algorithm 4 Summarization-Based Refinement Require: Diagram D, Source PDF P Ensure: Refined Diagram Dref - -1: iteration ←0, maxIterations ←3, threshold ←4.5, score ←0 - -2: while iteration < maxIterations OR score < threshold do - -3: -CScore, Cfeedback ←Ccritic(D, P) - -4: -FScore, Ffeedback ←Fcritic(D, P) - -5: -LScore, Lfeedback ←Lcritic(D) - -6: -Initialize allFeedback ←{} - -7: -if Score(CScore) < threshold then - -8: -allFeedback ←allFeedback ∪{Cfeedback} - -9: -if Score(FScore) < threshold then - -10: -allFeedback ←allFeedback ∪{Ffeedback} - -11: -if Score(LScore) < threshold then - -12: -allFeedback ←allFeedback ∪{Lfeedback} - -13: -D ←RefineDiagram(D, allFeedback) - -14: -score ←Min(CScore, FScore, LScore) - -15: -iteration ←iteration + 1 - -16: -Dref ←D - -17: return Dref - -13371 - ---- PAGE 31 --- - -Prompt template for Intent Generation [1] You will be given the title, abstract and Summarized section content of the Paper, figure and table captions in the paper, and you will have to come up with the intent of possible type of a diagram that is not there in the paper, but will be useful to create. -[2] Come up with an intent which will require you to process mathematical, logical reasoning on top of the extracted data from the paper, and come up with such intent that will require you to blend text and visuals from the paper to come up with the final diagram. -[3] The intent should be clear, comprehensive and any human can create a diagram from the paper content based on the intent. -SOME IN-CONTEXT EXAMPLES Title: -Abstract : -Section Content : -Table Captions: -Image Captions: -Your Intent: - -Prompt template for Intent Classification [1] You will be provided with an intent to task is to understand the intent of the diagram creation and classify into one of the following labels: -1) Extrapolated-Flowchart: If the intent is related to creating flowchart 2) Extrapolated-Summary: If the intent is related to summarization of related work, contributions, methods, datasets or hyperparameter or experimental details mentioned in the paper 3) Extrapolated-Architecture: If the intent is related to modifying an architecture or existing image in the paper 4) Extrapolated-Results: If the intent is related to generate plots or visualize results SOME IN-CONTEXT EXAMPLES Intent: -Label : - -Prompt template for Question Generation [1] Your intent of coming up with the diagram creation is provided below. Generate clarification questions based on the intent what information needs to be extracted so that you can generate the diagram. -SOME IN-CONTEXT EXAMPLES Intent: -Clarification Questions: - -Table 14: The prompt template for Intent Generation. - -Table 15: The prompt template for Intent Classification. - -Table 16: The prompt template for Question Generation. - -Prompt template for Data Extraction From Figures [1] Extract Raw data in the form of markdown from the image. -Image: -Extracted Markdown: - -Table 17: The prompt template for Data Extraction From Figures - -13372 - ---- PAGE 32 --- - -Prompt template for Answer Extraction [1] Your intent of diagram creation is presented along with the section text or image data. For each of the question, if the section text or image data is relevant, extract answer for the questions, if not relevant then, say ’NA’. Make sure that you do not extract information that is not present in the source code or image. -[2] Format your output as a list of JSON objects (Question/Answer pairs) where the keys are your questions and answers are the values. SOME IN-CONTEXT EXAMPLES intent: -Section/Image : -Questions : -Question/Answer Pairs: - -Prompt template for Code Generation [1] Generate a code in python where the intent of the diagram is provided to you, along with the intent type. The information to be presented is also in front of you. You should use the information to display the content. -[2] If the intent is about creating a flowchart, it has to be in graphviz. If the intent is about creating plots/line charts/graphs, it has to be clear and legible, ideally in plotnine. If the intent is to create or highlight a portion of image, you should use the pillow library to include bounding box or textual explanation. Also show that if you want to create a summary, it should be in a good layout with proper table header and fonts. -SOME IN-CONTEXT EXAMPLES Intent: -Intent Type : -Question-Answer pairs: -Generated Code: - -Prompt template for Line 2 in Algorithm 1 [1] You are provided with the intent of diagram creation. Decompose the intent and ask questions such that the answers to those questions will determine completeness of information. -SOME IN-CONTEXT EXAMPLES Intent: -Questions: - -Prompt template for Answer Extraction of the Question from Image/Associated Code [1] You are provided with the intent of diagram creation, image and the corresponding code that generated the image as input. Your goal is to extract answer of the question from the image or the code. Make sure that you do not extract information that is not present in the source code or image. -Intent: -Image: -Question: -Corresponding Code: -Answer of the Question: - -Table 21: Prompt template for Answer Extraction of the Question from Image/Associated Code - -Table 18: The prompt template for Answer Extraction. - -Table 19: The prompt template for Code Generation. - -Table 20: Prompt template for Line 2 in Algorithm 1 - -13373 - ---- PAGE 33 --- - -Prompt template for Refinement Step [1] You are provided with diagram and the associated code that generated it. Based on criteria name, the image has received a score of the given score out of 5, and a feedback is generated to improve. -Refine the code to incorporate the following feedback. -Image: -criteria name: -Corresponding Code: -Score: -Feedback : -Refined Code: - -Prompt template for Completeness Critic Evaluation [1] You are provided with the intent of the diagram creation, answers from PDF and the answers obtained from the Image or the code. You have to determine if the answer from the PDF/intent is completely present in the answer from image/code. Please provide a Completeness score from 1-5, where 1 denotes there is minimal Completeness and 5 when there is an exact match with the PDF Content. If your score is less than 4.5, generate feedback on what is needed to improve the Completeness. -Image: -Corresponding Code: -Intent: -Answer from the PDF: -Answer from the Diagram/Code: -Completeness Score: -Feedback : - -Prompt template for Faithfulness Critic Evaluation [1] You are provided with the intent of the diagram creation, answers from PDF and the answers obtained from the Image or the code. You have to determine if the answer from the Image or the code is faithful or true with respect to answer from the PDF. Please provide a Faithfulness score from 1-5, where 1 denotes there is minimal faithfulness and 5 when there is an exact match with the PDF Content. If your score is less than 4.5, generate feedback on what is needed to improve the faithfulness. -Image: -Corresponding Code: -Intent: -Answer from the PDF: -Answer from the Diagram/Code: -Faithfulness Score: -Feedback : - -Table 22: Prompt template for Refining generated Image/Associated Code - -Table 23: Prompt template for Completeness Critic Evaluation - -Table 24: Prompt template for Faithfulness Critic Evaluation - -13374 - ---- PAGE 34 --- - -Prompt template for Layout Critic Evaluation [1] You are provided with the intent of the diagram creation and the image, source PDF. -[2] You have to determine how much readable, comprehensible, precise in terms of the look-and-feel of the diagram. By precision, it means that all the necessary scientific information conveyed in the image can be easily deciphered. Please provide a score from 1-5, where 1 denotes there is minimal satisfaction on the look-and-feel aspect and 5 when there is nothing to complain about the look-and-feel aspect. If your score is less than 4.5, generate feedback on what is needed to improve the look-and-feel. -Image: -Intent: -Layout Score: -Feedback : - -Prompt template for Self-Refine Evaluation [1] You are provided with the intent of the diagram creation and the image. -[2] There might be some problem inside the image. Please generate feedback if you feel that there has been any inconsistency. -Image: -Intent: -One-step Score: -Feedback : - -Table 25: Prompt template for Layout Critic Evaluation - -Table 26: Prompt template for Self-Refine Evaluation - -13375 \ No newline at end of file diff --git a/experiments/literature_review/raw_text/scirex.txt b/experiments/literature_review/raw_text/scirex.txt deleted file mode 100644 index 8db86b3..0000000 --- a/experiments/literature_review/raw_text/scirex.txt +++ /dev/null @@ -1,591 +0,0 @@ - - ---- PAGE 1 --- - -Sarthak Jain2∗ Madeleine van Zuylen1 Hannaneh Hajishirzi1,3 Iz Beltagy1 - -Allen Institute for AI1 Northeastern University2 University of Washington3 - -Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pages 7506–7516 - -Abstract - -Extracting information from full documents is an important problem in many domains, but most previous work focus on identifying relationships within a sentence or a paragraph. -It is challenging to create a large-scale information extraction (IE) dataset at the document level since it requires an understanding of the whole document to annotate entities and their document-level relationships that usually span beyond sentences or even sections. In this paper, we introduce SCIREX, a document level IE dataset that encompasses multiple IE tasks, including salient entity identification and document level N-ary relation identification from scientific articles. We annotate our dataset by integrating automatic and human annotations, leveraging existing scientific knowledge resources. We develop a neural model as a strong baseline that extends previous state-of-the-art IE models to documentlevel IE. Analyzing the model performance shows a significant gap between human performance and current baselines, inviting the community to use our dataset as a challenge to develop document-level IE models. -Our data and code are publicly available at https: -//github.com/allenai/SciREX - -1 Introduction - -Extracting information about entities and their relationships from unstructured text is an important problem in NLP. Conventional datasets and methods for information extraction (IE) focus on within-sentence relations from general Newswire text (Zhang et al., 2017). However, recent work started studying the development of full IE models and datasets for short paragraphs (e.g., information extraction from abstracts of scientific articles as in SCIERC (Luan et al., 2018)), or only extracting - -∗Work done while at AI2 - -July 5 - 10, 2020. c⃝2020 Association for Computational Linguistics - -SCIREX: A Challenge Dataset for Document-Level - -Information Extraction - -jain.sar@northeastern.edu {madeleinev,hannah,beltagy}@allenai.org - -We evaluate our model on the task of question answering using - -Section : Dataset SQuAD is a machine comprehension dataset on a large set of Wikipedia articles , ….. . Two metrics are used to evaluate models : Exact Match ( EM ) and a softer metric , F1 score ….. . - -Section: Model Details . … Each paragraph and question are tokenized by a regular - expression - based word tokenizer ( PTB Tokenizer ) and fed into the model . …. - -Section : Results . The results of our model and competing approaches on the hidden test are summarized in Table [ reference ] . BiDAF ( ensemble ) achieves an EM score of 73.3 and an F1 score of 81.1 , outperforming all previous approaches . - -Figure 1: An example showing annotations for entity mentions ( Dataset , Metric , Task , Method ), coreferences (indicated by arrows), salient entities (bold), and N-ary relation (SQuaD, Machine Comprehension, BiDAF (ensemble), EM/F1) that can only be extracted by aggregating information across sections. - -relations (given ground truth entities) on long documents (e.g. Jia et al. (2019)). While these tasks provide a reasonable testbed for developing IE models, a significant amount of information can only be gleaned from analyzing the full document. To this end, not much work has been done on developing full IE datasets and model for long documents. - -Creating datasets for information extraction at the document level is challenging because it requires domain expertise and considerable annotation effort to comprehensively annotate a full document for multiple IE tasks. In addition to local relationships between entities, it requires identifying document-level relationships that go beyond sentences and even sections. -Figure 1 shows an example of such document level relation (Dataset: SQuAD, Metric: EM, Method: BiDAF, Task:machine comprehension). - -In this paper, we introduce SCIREX, a new comprehensive dataset for information extraction from - -7506 - ---- PAGE 2 --- - -scientific articles. Our dataset focuses on the task of identifying the main results of a scientific article as a tuple (Dataset, Metric, Task, Method) from raw text. It consists of three major subtasks, identifying individual entities, their document level relationships, and predicting their saliency in the document (i.e., entities that take part in the results of the article and are not merely, for example, mentioned in Related Work). Our dataset is fully annotated with entities, their mentions, their coreferences, and their document level relations. - -To overcome the annotation challenges for large documents, we perform both automatic and manual annotations, leveraging external scientific knowledge bases. An automatic annotation stage identi- fies candidate mentions of entities with high recall, then an expert annotator corrects these extracted mentions by referring to the text of the article and an external knowledge base.1 This strategy signifi- cantly reduces the time necessary to fully annotate large documents for multiple IE tasks. - -In addition, we introduce a neural model as a strong baseline to perform this task end-to-end. -Our model identifies mentions, their saliency, and their coreference links. It then clusters salient mentions into entities and identifies document level relations. We did not find other models that can perform the full task, so we evaluated existing stateof-the-art models on subtasks, and found our baseline model to outperform them. Experiments also show that our end-to-end document level IE task is challenging, with the most challenging subtasks being identifying salient entities, and to a lesser extent, discovering document level relations. - -The contributions of our paper are as follows, 1. we introduce SCIREX, a dataset that evaluates a comprehensive list of IE tasks, including N-ary relations that span long documents. This is a unique setting compared to prior work that focuses on short paragraphs or a single IE task. 2. We develop a baseline model that, to the best of our knowledge, is the first attempt toward a neural full document IE. -Our analysis emphasizes the need for better IE models that can overcome the new challenges posed by our dataset. We invite the research community to focus on this important, challenging task. - -2 Related Work - -Scientific IE In recent years, there has been multiple attempts to automatically extract structured - -1Papers with Code: paperswithcode.com - -information from scientific articles. These types of extractions include citation analysis (Jurgens et al., 2018; Cohan et al., 2019), identifying entities and relations (Augenstein et al., 2017; Luan et al., 2019, 2017), and unsupervised detection of entities and their coreference information (Tsai et al., 2013). - -Most structured extraction tasks from among these have revolved around extraction from sentences or abstracts of the articles. A recent example is SCIERC (Luan et al., 2018), a dataset of 500 richly annotated scientific abstracts containing mention spans and their types, coreference information between mentions, and binary relations annotations. -We use SCIERC to bootstrap our data annotation - -procedure (Section 3.2). - -There has been a lack of comprehensive IE datasets annotated at the document level. Recent work by Hou et al. (2019); Jia et al. (2019) tried to rectify this by using distant supervision annotations to build datasets for document-level relation extraction. In both datasets, the task of relation extraction is formulated as a binary classification to check if a triplet of ground-truth entities is expressed in the document or not. Instead, our work focuses on a comprehensive list of information extraction tasks “from scratch”, where the input is the raw docu- - -ment. This makes the IE model more interesting as it requires to perform entity extraction, coreference resolution, saliency detection in addition to the relation extraction.2 - -General IE Most work in general domain IE focus on sentence-level information extraction (Stanovsky et al., 2018; Qin et al., 2018; Jie and Lu, 2019). Recently, however, Yao et al. (2019) introduced DocRED, a dataset of cross-sentence relation extractions on Wikipedia paragraphs. The paragraphs are of a comparable length to that of SCIERC, which is significantly shorter than documents in our dataset. - -Previous IE work on the TAC KBP competitions (Ellis et al., 2017; Getman et al., 2018) comprise multiple knowledge base population tasks. -Our task can be considered a variant of the TAC KBP “cold start” task that discovers new entities and entity attributes (slot filling) from scratch. Two aspects of our task make it more interesting, 1) our model needs to be able to extract facts that - -2Another approach is to perform entity extraction then use the binary classification approach with a list of all possible combinations of relation tuples. This might work for short documents, but it is intractable for long documents because of the large number of entities. - -7507 - ---- PAGE 3 --- - -are mentioned once or twice rather than rely on the redundancy of information in their documents (e.g Rahman et al. (2016)), 2) TAC KBP relations are usually sentence-level binary relations between a query entity and an attribute (e.g Angeli et al. -(2015)), while our relations are 4-ary, span the whole document, and can’t be split into multiple binary relations as discussed in Section 3.1. - -End-to-End Neural IE models With neural networks, a few end-to-end models have been proposed that perform multiple IE tasks jointly (Miwa and Bansal, 2016; Luan et al., 2018; Wadden et al., 2019). The closest to our work is DYGIE++ (Wadden et al., 2019), which does named entity recognition, binary relation extraction, and event extraction in one model. DYGIE++ is a span-enumeration based model which works well for short paragraphs but does not scale well to long documents. Instead, we use a CRF sequence tagger, which scales well. -Our model also extracts 4-ary relations between salient entity clusters, which requires a more global view of the document than that needed to extract binary relations between all pairs of entity mentions. - -3 Document-Level IE - -Our goal is to extend sentence-level IE to documents and construct a dataset for document-level information extraction from scientific articles. This section defines the IE tasks we address, and describe the details of building our SCIREX dataset. - -3.1 Task Definition - -Entity Recognition Our entities are abstract objects of type Method, Task, Metric, or Dataset that appear as text in a scientific article. We define “mentions” (or spans) as a specific instantiation - -of the entity in the text – this could be the actual name of the entity, its abbreviation, etc. The entity recognition task is to identify “entity mentions” and classify them with their types. - -Salient Entity Identification Entities appear in a scientific article are not equally important. For example, a task mentioned in the related work section is less important than the main task of the article. In our case, salient entity identification refers to finding if an entity is taking part in the article evaluation. Salient Datasets, Metrics, Tasks, and Methods are those needed to describe the article’s results. For the rest of this paper, we will use - -the term salient to refer to entities that belong to a result relation tuple. - -Coreference is the task of identifying a cluster of mentions of an entity (or a salient entity) that are coreferred in a single document. - -Relation Extraction is the task of extracting Nary relations between entities in a scientific article. -We are interested in discovering binary, 3-ary, and 4-ary relations between a collection of entities of type (Dataset, Method, Metric, and Task). It is important to note that this 4-ary relation can’t be split into multiple binary relations because, e.g., a dataset might have multiple tasks, and each one has its own metric, so the metric cannot be decided solely based on the dataset or the task. - -3.2 Dataset Construction - -Document-level information extraction requires a global understanding of the full document to annotate entities, their relations, and their saliency. -However, annotating a scientific article is timeconsuming and requires expert annotators. This section explains our method for building our SCIREX dataset with little annotation effort. It combines distant supervision from an existing KB and noisy automatic labeling, to provide a much simpler annotation task. - -Existing KB: Papers with Code Papers with Code (PwC)3 is a publicly available corpus of 1,170 articles published in ML conferences annotated with result five-tuples of (Dataset, Metric, Method, Task, Score). The PwC curators collected this data from public leaderboards, previously curated results by other people, manual annotations, and from authors submitting results of their work. - -This dataset provides us with distant supervision signal for a task that requires document-level understanding - extracting result tuples. The signal is “distant” (Riedel et al., 2010) because, while we know that the PwC result tuple exists in the article, we don’t know where exactly it is mentioned (PwC does not provide entity spans, and PwC entity names may or may not appear exactly in the document). - -PDF preprocessing PwC provides arXiv IDs for their papers. To extract raw text and section information, we use LaTeXML (https://dlmf.nist. - -3https://github.com/paperswithcode/ paperswithcode-data - -7508 - ---- PAGE 4 --- - -Statistics (avg per doc) SCIREX SCIERC - -Words 5,737 130 Sections 22 1 Mentions 360 16 Salient Entities 8 — Binary Relations 16 9.4 4-ary Relations 5 — - -Table 1: Comparison of SCIREX with next biggest ML Information Extraction dataset SCIERC. SCIREX consists of 438 documents. All dataset statistics are perdocument averages. 57% of binary and 99% of 4-ary relations occur across sentences. 20% binary and 55% 4-ary relations occur across sections. This highlight the need for document level models. - -Dataset Metric Task Method Deleted - -Dataset 3.55 0.01 0.07 0.16 0.03 Metric 0.02 7.95 0.00 0.03 0.00 Task 0.32 0.07 17.92 0.44 0.01 Method 0.65 0.21 0.24 53.27 0.02 Added 2.40 1.30 2.82 8.50 - - -Table 2: Confusion Matrix for the mention-level corrections (change type, add span, or delete span). Values are average percentages “per document” (not per type). -For example, cell at intersection of row Metric and column Task contains document-average percentage of span-type change from Metric to Task. The column Deleted represents percent spans that were deleted. -The row Added represents percent spans added. Diagonal represent percent spans of each type that are correctly labeled by the automatic labeling and didn’t need to change by the human annotator. - -gov/LaTeXML/) for papers with latex source (all 438 annotated papers), or use Grobid (GRO, 2008– 2020) for papers in PDF format (only 10% of remaining papers did not have latex source). La- TeXML allowed us to extract clean document text with no figures / tables / equations. We leave it as future work to augment our dataset with these structured fields. To extract tokens and sentences, we use the SpaCy (https://spacy.io/) library. - -Automatic Labeling Given the length of the document is on the order of 5K tokens, we simplify the human annotation task by automatically labeling the data with noisy labels, then an expert annotator only needs to fix the labeling mistakes. - -One possible way to augment the distant supervision provided by PwC is finding mention spans of PwC entities. Initial experiments showed that this did not work well because it does not provide - -enough span-level annotations that the model can use to learn to recognize mention spans. - -To get more dense span-level information, we want to label salient (corresponding to PwC entities) and also non-salient spans. We train a standard BERT+CRF sequence labeling model on the SCI- ERC dataset (described in Section 2). We run this model on each of the documents in the PwC corpus, and it provides us with automatic (but noisy) predictions for mention span identification. - -The next step is to find mention spans that correspond to PwC entities. For each mention predicted by our SCIERC-trained model, we compute a Jaccard similarity with each of the PwC entities. -Each mention is linked to the entity if the threshold exceeds a certain ϵ. To determine ϵ, two expert annotators manually went through 10 documents to mark identified mentions with entity names, and ϵ was chosen such that the probability of this assignment is maximized. We use this threshold to determine a mapping for the remaining 1,170 documents. Given that Jaccard-similarity is a coarse measure of similarity, this step favors high recall over precision. - -Human Annotation Given this noisily labeled data, we ask our annotator to perform necessary corrections to generate high-quality annotations. -Annotators are provided with a list of papers-withcode entities that they need to find in the document, making their annotations deliberate (as opposed to not knowing which entities to annotate). -Our annotator deleted and modified types of spans for salient entities (belong to PwC result tuple) and non-salient entities, while only adding missed spans for salient ones. Also, if a mention was linked to a wrong PwC entity, then our annotator was also asked to correct it. Full annotation instructions are provided in Appendix B. - -3.3 Dataset and Annotation Statistics - -Dataset statistics and Cross-section Relations Using the annotation procedure mentioned above, we build a dataset of 438 fully annotated documents. Table 1 provides dataset statistics and shows the proportion of relations in our dataset that requires reasoning across sentence/section. It shows that the majority of the relations, especially 4-ary relations span multiple sentences or even multiple sections. An example of such cross-section reasoning can be found in Figure 1. - -7509 - ---- PAGE 5 --- - -Corrections Table 2 provides information about the average number of changes made during the human annotation. It shows that 83% (sum of diagonal) are correct automatic labels, 15% (sum of bottom row) are newly added spans, 2% are type changes, and a negligible percentage is deleted entities (sum of the last column). Also, on average, 12% (not in the table) of the final mentions in the document had the wrong PwC links and needed to be corrected, with a majority of changes being removing links from Method spans. - -Inter-annotator agreement We also asked four experts (Ph.D. students in ML/NLP field) to annotate five documents to compute the inter-annotator agreement. For mention classification, we achieve 95% average cohen-κ scores between each pair of experts and our main annotator. - -Annotation Speed To measure if automatic labeling is making the human annotation faster, we also asked our annotator to perform annotations on five documents without automatic labeling. We compute the difference in time between these two forms of annotation per entity annotated. Note that here, we only ask our annotator to annotate salient mentions. With the automatic labeling, annotation speed is 1.34 sec per entity time vs. 2.48 sec per entity time on documents without automatic labeling (a 1.85x speedup). We also observe 24% improvement in recall of salient mentions by including non-salient mentions, further showing the utility of this approach. - -4 Model - -We develop a neural model that performs document- - -level IE tasks jointly in an end-to-end fashion.4 - -This section details our model design (also summarized in Figure 2). - -Document Representation An input document D is represented as a list of sections [s1, ..., s|S|]. -We encode the document in two steps, section-level, - -then document-level. We use pretrained contextualized token encodings using SciBERT (Beltagy et al., 2019) over each section separately to get embeddings for tokens in that section. 5 To allow document-level information flow, we concatenate - -4with the exception of coreference resolution 5If the section is bigger than 512 tokens (SciBERT limit), it is broken into 512 token subsections, and each subsection is encoded separately. - -the section-level token embeddings and add a BiL- STM on top of them. This allows the model to take into account cross-section dependencies. Thus for each token wi in the document, this step outputs an embedding ei. - -Mention Identification and Classification Given token embeddings, our model applies a sequence tagger that identifies mentions and classifies their types. We train a BIOUL based CRF tagger on top of the BERT-BiLSTM embeddings of words to predict mention spans mj and their corresponding types. - -Mention Representation Given the words {wj1, ..., wjN } of a mention mj, our model learns a mention embedding mej of the mention, which will be used in later saliency identification and relation classification steps. -The mention embedding is the concatenation of first token embedding ej1, last token embedding ejN and attention weighted average of all embeddings in the mention span PN - -k=1 αjkejk, where ejk is the embedding of word wjk and αjk are scalars computed by passing the token embedding through an additive attention layer (Bahdanau et al., 2015). -We concatenate these embeddings with additional - -features — span’s relative position in the document, an indicator showing if the sentence containing the mention also contains some marker words like ‘experiment’ or ‘dataset’ and the mention type. - -Salient Mention Classification Each mention mj is classified as being salient or not (i.e., should it belong in a relation tuple) by passing its span embedding mej through a feedforward layer. Because saliency is a property of entities, not mentions, this mention saliency score is just an input to the salient entity cluster identifications. - -Pairwise Coreference Resolution The coreference step is given a list of all pairs of identified mentions, and it decides which pair is coreferring. This component is separate from the end-to-end model. -It concatenates the “surface forms” of two spans mi and mj, embed them using SciBERT, then use a linear classification layer on top of [CLS] embedding to compute the pairwise coreference score cij. We also tried integrating it into our model, where we classify pairs of “span embeddings” (not the surface form) but found the separate model that uses surface forms to work much better. - -7510 - ---- PAGE 6 --- - -Figure 2: Overview of our model; it uses a two-level BERT+BiLSTM method to get token representations which are passed to a CRF layer to identify mentions. Each mention is classified as being salient or not. A coreference model is trained to cluster these mentions into entities. A final classification layer predicts relationships between 4-tuple of entities (clusters). - -Related - -PTB - -SQuaD Exact Match, EM BiDAF MC, QA - -SQuaD - -MC - -Exact Match - -SQuaD - -MC - -Exact Match - -Section : Dataset - -We use PTB - -SQuaD is MC - -tokenizer... - -dataset .... - -Mention clustering Given a list of span pairs mi and mj, and their pairwise coreference scores cij, they are grouped into clusters that can be thought of as representing a single entity. We generate a coreference score matrix for all pairs and perform agglomerative hierarchical clustering (Ward, 1963) on top of it to get actual clusters. The number of clusters is selected based on the silhouette score (Rousseeuw, 1987) which optimizes for the cohesion and separation of clusters and does not depend on having gold standard cluster labels. - -Salient Entity Cluster Identification This step filters out clusters from the previous step, and only keep salient clusters for the final relation task. To do so, we take a simple approach that identifies a salient cluster as the one in which there is at least one salient mention (as determined previously). The output of this step is a set of clusters C1, ..., CL where each cluster Ci is a set of mentions {mi1, ..., mij} of the same type. - -Relation Extraction Given all the clusters of mentions identified in a document from the previous step, our task now is to determine which of these belong together in a relation. To that end, we follow (Jia et al., 2019) methodology. We consider all candidate binary and 4-tuples of clusters and classify them as expressed or not expressed in the document. Here we describe the classification of 4-ary relations. For binary relation, the method is similar. - -Consider such a candidate relation (4-tuple of - -Entity Clusters (bold=salient) - -Pairwise Coreference - -Salient Mentions - -BiDAF - -EM - -PTB BiDAF - -QA - -EM - -Mentions - -CRF Decoding - -BiLSTM - -BERT BERT BERT - -Section Results. -Our model BiDAF - -gets EM ... - -clusters) R = (C1, C2, C3, C4) where each Ci is a set of mentions {mi1, ..., mij} in the document representing the same entity. We encode this relation into a single vector by following a two-step procedure – constructing a section embedding and aggregating them to generate a document level embedding. For each section s of the document, we create a section embedding Es - -R for this relation as follows - - -For each cluster Ci ∈R, we construct its section embedding Es - -i by max-pooling span embeddings of the mentions of Ci that occur in section s (along with a learned bias vector b in case no mentions of Ci appear in section s). Then the section s embedding of tuple R is Es - -1; Es 2; Es 3; Es 4]) where ; denotes concatenation and FFN is a feedforward network. We then construct a document level embedding of R, ER as mean of section embeddings 1 |S| - -R = FFN([Es - -P|S| - -s=1 Es - -R. The final classification for relationship is done by passing the ER through another FFN, which returns a probability of this tuple expressing a relation in this document. - -Training Procedure While mention identification, span saliency classification, and relation extraction share the base document and span representation from BERT + BiLSTM and trained jointly, each of these subparts is trained on ground truth input. Note that we require the saliency classification and relation extraction to be independent of mention identification task since the output of this task (essentially the span of mention text) is non- - -7511 - ---- PAGE 7 --- - -differentiable. 6 The model jointly optimizes three losses, negative log-likelihood for mention identifi- cation, binary cross-entropy for saliency classification, and binary cross-entropy for relation extraction, with all three losses weighted equally. - -5 Evaluation - -We compare our model with other recently intro- - -duced models. Since we cannot apply previous models directly to our task, we evaluate on subtasks of our dataset and also evaluate on SCIERC (Section 5.2). The other goal of the evaluation is to establish a baseline performance on our dataset and to provide insights into the difficulty of each subtask. To that end, we evaluate the performance of each component separately (Section 5.3), and in the overall end-to-end system (Section 5.4). In addition, we perform diagnostic experiments to identify the bottlenecks in the model performance. -We report experimental setup and hyperparameters - -in appendix A. - -5.1 Evaluation Metrics - -Mention Identification is a sequence labeling task, which we evaluate using the standard macro average F1 score of exact matches of all mention types. -Salient Mentions and Pairwise Coreference are binary classification tasks which we evaluate using the F1 score. -Salient Entity Clustering evaluation relies on some mapping between the set of predicted clusters and gold clusters. Given a predicted cluster P and a gold cluster G, we consider P to match G if more than 50% of P’s mentions belong to G,7 that is |P∩G| - -|P| > 0.5. The 0.5 threshold enjoys the property that, assuming all predicted clusters are disjoint from each other (which is the case by construction) and gold clusters are disjoint from each other (which is the case for 98.5% of them), a single predicted cluster can be assigned to atmost one gold cluster. This maps the set of predicted clusters to gold clusters, and given the mapping, it is straightforward to use the F1 score to evaluate predictions. This procedure optimizes for identifying all gold clusters even if they are broken into multiple predicted clusters. - -6It is conceivable that mixing the gold mention spans with predicted mention spans might give an improvement in performance; therefore, we leave this as future work. - -7We consider two mention spans to be a match if their Jaccard similarity is greater than 0.5. - -Relation Extraction evaluation relies on the same mapping used in the evaluation of salient entity clustering. Under such mapping, each predicted Nary relation can be compared with gold relations, and decide if they match or not. This becomes a binary classification task that we evaluate with positive class F1 score. We report F1 scores for binary and 4-ary relation tuples. We get binary relations by splitting each 4-ary relation into six binary ones. - -5.2 Comparing with Baselines - -We compare our model with DYGIE++ (Wadden - -et al., 2019) and DocTAET (Hou et al., 2019) on subtasks of our SCIREX dataset and on the SCI- ERC dataset wherever they apply. Our results show that only our model can perform all the subtasks in an end-to-end fashion and performs better than or on par with these baselines on respective subtasks. - -5.2.1 Evaluation on SCIREX - -DYGIE++ (Wadden et al., 2019) is an end-toend model for entity and binary relation extraction (check Section 2 for details). Being a span enumeration type model, DYGIE++ only works on paragraph level texts and extracts relations between mentions in the same sentence only. Therefore, we subdivide SCIREX documents into sections and formulate each section as a single training example. We assume all entities in relations returned by DYGIE++ are salient. We map each binary mention-level relation returned to entity-level by mapping the span to its gold cluster label if it appears in one. We consider 3 training configurations of DYGIE++, 1. trained only on the abstracts in our dataset, 2. trained on all sections of the documents in our dataset. 3. trained on SCIERC dataset (still evaluated on our dataset), At test time, we evaluate the model on all sections of the documents in the test set. - -Results in Table 3 show that we perform generally better than DYGIE++. The performance on end-to-end binary relations shows the utility of incorporating a document level model for crosssection relations, rather than predicting on individual sections. Specifically, We observe a large difference in recall, which agrees with the fact that 55% of binary relation occur across sentence level. DY- GIE++ (All sections) were not able to identify any binary relations because 80% of training examples have no sentence level binary relations, pushing the model towards predicting very few relations. In - -7512 - ---- PAGE 8 --- - -Model P R F1 - -Mention Identification DYGIE++ 0.703 0.676 0.678 Our Model 0.707 0.717 0.712 End-to-end binary relations DYGIE++ (Abstracts Only) 0.003 0.001 0.002 DYGIE++ (All sections) 0.000 0.000 0.000 DYGIE++ (SCIERC) 0.029 0.128 0.038 Our Model 0.065 0.411 0.096 4-ary relation extraction only DocTAET 0.477 0.885 0.619 Our Model 0.531 0.718 0.611 - -Table 3: Evaluating state-of-the-art models on subtasks of SCIREX dataset because we did not find an existing model that can perform the end-to-end task. - -Task Model P R F1 - -Mention Ident. -DYGIE++ 0.676 0.694 0.685 Our Model 0.637 0.640 0.638 - -Pairwise Coref. -DYGIE++ 0.577 0.455 0.476 and Clustering Our Model 0.187 0.552 0.255 - -Table 4: Comparison of DYGIE++ with our model on various subtasks of SCIERC dataset - -contrast, training on SCIERC (and evaluating on SCIREX) gives better results because it is still able to find the few sentence-level relations. - -DocTAET (Hou et al., 2019) is a documentlevel relation classification model that is given a document and a relation tuple to classify if it is expressed in the document. It is formulated as an entailment task with the information encoded as [CLS] document [SEP] relation in a BERT style model. This is equivalent to the last step of our model but with gold salient entity clusters as input. Table 3 shows the result on this subtask, and it shows that our relation model gives comparable performance (in terms of positive class F1 score) to that of DocTAET. - -5.2.2 Evaluation on SCIERC - -Table 4 summarizes the results of evaluating our model and DYGIE++ on the SCIERC dataset. For mention identification, our model performance is a bit worse mostly because SCIERC has overlapping entities that a CRF-based model like ours can not handle. For the task of identifying coreference clusters, we perform significantly worse than DY- GIE++’s end-to-end model. This provides future avenues towards improving coreference resolution for SCIREX by incorporating it in an end-to-end fashion. - -Task P R F1 - -Component-wise (gold Input) Mention Identification 0.707 0.717 0.712 Pairwise Coreference 0.861 0.852 0.856 Salient Mentions 0.575 0.584 0.579 Salient Entity Clusters 1.000 0.984 0.987 Binary Relations 0.820 0.440 0.570 4-ary Relations 0.531 0.718 0.611 End-to-end (predicted input) Salient Entity Clusters 0.223 0.600 0.307 Binary Relations 0.065 0.411 0.096 4-ary Relations 0.007 0.173 0.008 End-to-end (gold salient clustering) Salient Entity Clusters 0.776 0.614 0.668 Binary Relations 0.372 0.328 0.334 4-ary Relations 0.310 0.281 0.268 - -Table 5: Analysis of performance of our model and its subtasks under different evaluation configurations. - -5.3 Component-wise Evaluation - -The main contribution of our model is to connect multiple components to perform our end-to-end task. This section evaluates each step of our model separately from all other components. To do so, we feed each component with gold inputs and evaluate the output. This gives us a good picture of the performance of each component without the accumulation of errors. - -The first block of Table 5 summarizes the results of this evaluation setting. We know from Tables 3, 4 that our mention identification and relation identification components are working well. For pairwise coreference resolution, we know from Table 4 that it needs to be improved, but it is performing well on our dataset likely because the majority of coreferences in our dataset can be performed using only the surface form of the mentions (for example, abbreviation reference). The worst performing component is identifying salient mentions, which requires information to be aggregated from across the document, something the current neural models lack.8 - -5.4 End-to-End Evaluation - -Evaluation with Predicted Input. -The second block in Table 5 gives results for the end-to-end performance of our model in predicting salient entity clusters, binary relations, and 4-ary relations. -We noticed that there is quite a drop in the end-to- - -8Performance of Salient Entity Clusters is close to 1.0 because it is a deterministic algorithm (clustering followed by filtering) that gives perfect output given gold input. The reason the recall is not 1.0 as well is because of small inconsistencies in the gold annotations (two distinct entities merged into one). - -7513 - ---- PAGE 9 --- - -end performance compared to the component-wise performance. This is particularly clear with relations; even though the relation extraction component performance is reasonably good in isolation, its end-to-end performance is quite low because of the accumulation of errors in previous steps. - -Evaluation with Gold Salient Clustering. -Through manual error analysis, we found that the identification of salient clusters is the most problematic step in our model. The third block in Table 5 quantifies this. In this setting, we run our end-to-end model but with “gold cluster saliency” information. -In particular, we predict clusters of mentions using our model (mention identification, pairwise coreference, and mention clustering). -Then instead of filtering clusters using our mention saliency score, we keep only those clusters that have any overlap with at least one gold cluster. -Predicted clusters that match the same gold cluster are then combined. Finally, we feed those to the relation extraction step of our model. Under this setting, we found that the performance of 4- ary relations improves considerably by more than 10x. This confirms our hypothesis that identifying salient clusters is the key bottleneck in the end-toend system performance. This is also consistent with the component-wise results that show low performance for salient mentions identification. - -Error Analysis for Identifying Salient Clusters. -Our error analysis shows that the average number of mentions in a salient cluster classified correctly is 15 mentions, whereas for the misclassified ones is six mentions. This indicates that our model judges the saliency of an entity strongly based on how frequently it is mentioned in the document. -While this is a perfectly reasonable signal to rely - -on, the model seems to trust it more than the context of the entity mention. For example, in the following snippet, “... For each model, we report the test perplexity, the computational budget, the parameter counts, the value of DropProb, and the computational efficiency ....”, the entity “the parameter counts” is misclassified as non-salient, as it only appears twice in the document. One possible way to address this issue with salient entity identification is to replace its simple filtering step with a trained model that can do a better job at aggregating evidence from multiple mentions. - -Overall, these results indicate that identifying the saliency of entities in a scientific document is - -a challenging task. It requires careful documentlevel analysis, and getting it right is crucial for the performance of an end-to-end document-level IE model. Also, the difference between results in the third block of the results and the component-wise results indicate that the whole model can benefit from incremental improvements to each component. - -6 Conclusion - -We introduce SCIREX, a comprehensive and chal- - -lenging dataset for information extraction on full documents. We also develop a baseline model for our dataset, which, to the best of our knowledge, is the first attempt toward a neural document level IE that can perform all the necessary subtasks in an end-to-end manner. We show that using a document level model gave a significant improvement in terms of recall, compared to existing paragraphlevel approaches. - -This task poses multiple technical and modeling challenges, including 1. the use of transformer-based models on long documents and related device memory issues, 2. aggregating coreference information from across documents in an end-to-end manner, 3. identifying salient entities in a document and 4. performing N-ary relation extraction of these entities. Each of these tasks challenges existing methodologies in the information extraction domain, which, by and large, focus on short text sequences. An analysis of the performance of our model emphasizes the need for better document-level models that can overcome the new challenges posed by our dataset. As our research community moves towards document level IE and discourse modeling, we position this dataset as a testing ground to focus on this important and challenging task. - -Acknowledgments - -This research was supported by the ONR MURI N00014-18-1-2670, ONR N00014-18-1-2826, DARPA N66001-19-2-4031, and Allen Distinguished Investigator Award. We thank the Semantic Scholar team at AI2, UW NLP, and anonymous reviewers for their insightful comments. We are especially grateful to Kyle Lo for help with Grobid parser, the complete Papers With Code team for making their data publicly available, Dan Weld and Robert Stojnic for helpful discussion and feedback. - -7514 - ---- PAGE 10 --- - -References - -2008–2020. -Grobid. -https://github.com/ kermitt2/grobid. - -Gabor Angeli, Victor Zhong, Danqi Chen, Arun Te- - -jasvi Chaganty, Jason Bolton, Melvin Jose Johnson Premkumar, Panupong Pasupat, Sonal Gupta, and Christopher D. Manning. 2015. Bootstrapped self training for knowledge base population. Theory and Applications of Categories. - -Isabelle Augenstein, Mrinal Das, Sebastian Riedel, - -Lakshmi Vikraman, and Andrew McCallum. 2017. -SemEval 2017 task 10: -ScienceIE - extracting keyphrases and relations from scientific publications. -In Proceedings of the 11th International Workshop on Semantic Evaluation (SemEval-2017). - -Dzmitry Bahdanau, Kyunghyun Cho, and Yoshua Ben- - -gio. 2015. -Neural machine translation by jointly learning to align and translate. -3rd International Conference on Learning Representations, ICLR 2015. - -Iz Beltagy, Kyle Lo, and Arman Cohan. 2019. Scib- - -ert: A pretrained language model for scientific text. -In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing. - -Arman Cohan, Waleed Ammar, Madeleine van Zuylen, - -and Field Cady. 2019. -Structural scaffolds for citation intent classification in scientific publications. -In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computa- - -tional Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). - -Joe Ellis, Jeremy Getman, Dana Fore, Neil Kuster, - -Zhiyi Song, Ann Bies, and Stephanie Strassel. 2017. -Overview of linguistic resources for the tac kbp 2017 evaluations: Methodologies and results. Theory and Applications of Categories. - -Matt Gardner, Joel Grus, Mark Neumann, Oyvind - -Tafjord, Pradeep Dasigi, Nelson F. Liu, Matthew Peters, Michael Schmitz, and Luke S. Zettlemoyer. -2017. Allennlp: A deep semantic natural language processing platform. - -Jeremy Getman, Joe Ellis, Stephanie Strassel, Zhiyi - -Song, and Jennifer Tracey. 2018. Laying the groundwork for knowledge base population: Nine years of linguistic resources for tac kbp. In Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018). - -Yufang Hou, Charles Jochim, Martin Gleize, Francesca - -Bonin, and Debasis Ganguly. 2019. Identification of tasks, datasets, evaluation metrics, and numeric scores for scientific leaderboards construction. -In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, Florence, Italy. Association for Computational Linguistics. - -Robin Jia, Cliff Wong, and Hoifung Poon. 2019. - -Document-level n-ary relation extraction with multiscale representation learning. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). - -Zhanming Jie and Wei Lu. 2019. Dependency-guided - -LSTM-CRF for named entity recognition. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing, Hong Kong, China. Association for Computational Linguistics. - -David Jurgens, Srijan Kumar, Raine Hoover, Daniel A. - -McFarland, and Dan Jurafsky. 2018. Measuring the evolution of a scientific field through citation frames. -Transactions of the Association for Computational Linguistics, 6. - -Yi Luan, Luheng He, Mari Ostendorf, and Hannaneh - -Hajishirzi. 2018. -Multi-task identification of entities, relations, and coreference for scientific knowledge graph construction. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing, Brussels, Belgium. Association for Computational Linguistics. - -Yi Luan, Mari Ostendorf, and Hannaneh Hajishirzi. - -2017. Scientific information extraction with semisupervised neural tagging. In Proceedings of Empirical Methods in Natural Language Processing. - -Yi Luan, Dave Wadden, Luheng He, Amy Shah, Mari - -Ostendorf, and Hannaneh Hajishirzi. 2019. A general framework for information extraction using dynamic span graphs. -In Proceedings of The North American Chapter of the Association for Computa- - -tional Linguistics (NAACL). - -Makoto Miwa and Mohit Bansal. 2016. End-to-end re- - -lation extraction using LSTMs on sequences and tree structures. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Berlin, Germany. Associ- - -ation for Computational Linguistics. - -Pengda Qin, Weiran Xu, and William Yang Wang. - -2018. -Robust distant supervision relation extraction via deep reinforcement learning. In Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Melbourne, Australia. Association for Computational Linguistics. - -Rashedur Rahman, Brigitte Grau, Sophie Rosset, Yoann Dupont, J´er´emy Guillemot, Olivier Mesnard, - -Christian Lautier, and Wilson Fred. 2016. Tac kbp 2016 cold start slot filling and slot filler validation systems by irt systemx. Theory and Applications of Categories. - -7515 - ---- PAGE 11 --- - -Sebastian Riedel, Limin Yao, and Andrew McCallum. - -2010. Modeling relations and their mentions without labeled text. -In Joint European Conference on Machine Learning and Knowledge Discovery in Databases. Springer. - -Peter J. Rousseeuw. 1987. Silhouettes: A graphical aid - -to the interpretation and validation of cluster analysis. Journal of Computational and Applied Mathematics, 20. - -Gabriel Stanovsky, Julian Michael, Luke Zettlemoyer, - -and Ido Dagan. 2018. Supervised open information extraction. In Proceedings of The 16th Annual Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics. - -Chen-Tse Tsai, Gourab Kundu, and Dan Roth. 2013. - -Concept-based analysis of scientific literature. -In Proceedings of the 22nd ACM International Conference on Information & Knowledge Management. - -David Wadden, Ulme Wennberg, Yi Luan, and Han- - -naneh Hajishirzi. 2019. Entity, relation, and event extraction with contextualized span representations. -In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing. - -Joe H. Ward. 1963. Hierarchical grouping to optimize - -an objective function. Journal of the American Statistical Association, 58(301). - -Yuan Yao, Deming Ye, Peng Li, Xu Han, Yankai Lin, - -Zhenghao Liu, Zhiyuan Liu, Lixin Huang, Jie Zhou, and Maosong Sun. 2019. DocRED: A large-scale document-level relation extraction dataset. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics. - -Yuhao Zhang, Victor Zhong, Danqi Chen, Gabor An- - -geli, and Christopher D. Manning. 2017. Positionaware attention and supervised data improve slot filling. In Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing. - -A Model Details - -We divide our 438 annotated documents into train- - -ing (70%), validation (30%) and test set (30%). -The base document representation of our model is formed by SciBERT-base (Beltagy et al., 2019) and BiLSTM with 128-d hidden state. We use a dropout of 0.2 after BiLSTM embeddings. All feedforward networks are composed of two hidden layers, each of dimension 128 with gelu activation and with a dropout of 0.2 between layers. For additive attention layer in span representation, we collapse the token embeddings to scalars by passing through - -the feedforward layer with 128-d hidden state and performing a softmax. We train our model for 30 epochs using Adam optimizer with 1e-3 as learning rate for all non BERT weights and 2e-5 for BERT weights. We use early stopping with a patience value of 7 on the validation set using relation extraction F1 score. All our models were trained using 48Gb Quadro RTX 8000 GPUs. The multitask model takes approximately 3 hrs to train. - -For the BERT coreference model, we use SciBERT-base embeddings with two mentions encoded as [CLS] mention 1 [SEP] mention 2 [SEP]. -We use a linear layer on top of [CLS] token em- - -bedding to compute the mention pair’s coreference score. - -All our models were implemented in AllenNLP library(Gardner et al., 2017). - -B Annotation Guidelines - -Our Annotation guidelines can be found at - -https://github.com/allenai/SciREX/blob/ - -master/Annotation%20Guidelines.pdf Note, for Method type entities, we specifically ask our annotator to break down complex entities into simpler ones before looking for mentions in the text. -For example, a method entity DLDL+VGG-Face is composite and broken into two parts DLDL and VGG-Face. Currently, our model considers all mentions of subentities as mentions of the corresponding Method entity. We leave the task of extracting relation between subentities explicitly as future work. - -7516 \ No newline at end of file diff --git a/experiments/literature_review/raw_text/structured_ie.txt b/experiments/literature_review/raw_text/structured_ie.txt deleted file mode 100644 index 44e84eb..0000000 --- a/experiments/literature_review/raw_text/structured_ie.txt +++ /dev/null @@ -1,615 +0,0 @@ - - ---- PAGE 1 --- - -Article https://doi.org/10.1038/s41467-024-45563-x Structured information extraction from scientific text with large language models - -1Lawrence Berkeley National Laboratory, Berkeley, CA, USA. 2Materials Science and Engineering Department, University of California, Berkeley, CA, USA. -3These authors contributed equally: John Dagdelen, Alexander Dunn. -e-mail: ajain@lbl.gov - -Nature Communications| (2024) 15:1418 1 - -Received: 17 March 2023 - -Accepted: 22 January 2024 - -Check for updates - -1234567890():,; - -1234567890():,; - -The majority of scientific knowledge about solid-state materials is scattered across the text, tables, and figures of millions of academic research papers. Thus, it is difficult for researchers to properly understand the full body of past work and effectively leverage existing knowledge when designing experiments. Moreover, machine learning models for direct property prediction are being increasingly employed as screening steps for materials discovery and design workflows1–3, but this approach is limited by the amount of training data available in tabulated databases. While databases of materials property data derived from ab initio simulations are relatively common, they are limited to the subset of computationally accessible properties whereas databases of experimental property measurements and other useful experimental data are comparatively small (if they exist at all). - -In recent years, researchers have made significant advances in the application of natural language processing (NLP) algorithms for materials towards structuring the existing body of textual materials science knowledge4–7. The majority of this work has focused on named entity recognition (NER), where entity labels such as “material" or “property" are applied to words from the text. These tagged sequences of words can sometimes be used with additional post-processing to construct auto-generated tabular databases of materials property data - -John Dagdelen 1,2,3, Alexander Dunn 1,2,3, Sanghoon Lee1,2, Nicholas Walker1, Andrew S. Rosen 1,2, Gerbrand Ceder1,2, Kristin A. Persson 1,2 & Anubhav Jain 1 - -Extracting structured knowledge from scientific text remains a challenging task for machine learning models. Here, we present a simple approach to joint named entity recognition and relation extraction and demonstrate how pretrained large language models (GPT-3, Llama-2) can be fine-tuned to extract useful records of complex scientific knowledge. We test three representative tasks in materials chemistry: linking dopants and host materials, cataloging metal-organic frameworks, and general composition/phase/morphology/ application information extraction. Records are extracted from single sentences or entire paragraphs, and the output can be returned as simple English sentences or a more structured format such as a list of JSON objects. This approach represents a simple, accessible, and highly flexible route to obtaining large databases of structured specialized scientific knowledge extracted from research papers. - -aggregated from text entries8–12. Prior information extraction studies in the domain of solid-state materials include NER labeling of chemical synthesis parameters in methods section texts13–16, quantitative results of battery cycling experiments17, or peak absorption wavelengths for UV-Vis experiments18, among others4,5,9–12,19. Regular expressions, BiLSTM recurrent neural networks, and smaller transformer-based language models such as BERT are sufficient for such tasks. In these studies, entities (e.g., LiCoO2, "350K") rather than relations (e.g., "350K" is an experimental synthesis parameter for LiCoO2) are the primary target of extraction. - -Yet, a key challenge in scientific natural language processing is the development of robust, simple, and general relation extraction (RE) techniques to accurately extract the relationships between named entities. Downstream tasks such as supervised machine learning or the construction of knowledge graphs require the transformation of unstructured text into sets of structured relationships between semantic entities of interest. RE models are used to determine which entities are linked by a predefined set of relations. For example, in the sentence “LiCoO2 is studied as a Li-ion battery material", the material entity “LiCoO2" is linked to the application entity “Li-ion battery". Until recently, there has been relatively little work on relation extraction in - ---- PAGE 2 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 2 - -materials science text, but there has been much research interest in RE on general-purpose text, especially related to linking people, organizations, locations, and dates20,21. These methods have traditionally relied on pipeline-based approaches where named entity recognition is the first step followed by one or more additional steps and a final relation classification step (see Fig. 1, top row). Each of these steps typically uses a separate machine learning model, which may or may not share weights or architectures with each other. State-of-the-art transformer-based implementations of pipeline implementations have been shown to perform document level relation extraction relatively well on a variety of general-knowledge corpora22 and more specialized domains such as chemical-disease relations23 and gene-disease relations24. Recently, this kind of two-step approach was demonstrated on a benchmark dataset of procedures for the synthesis of polycrystalline materials encoded as directed graphs extracted from materials science text25. - -However, scientific information often cannot be modeled as simple pairwise relations between entities. This is particularly apparent in inorganic materials science, where a compound’s properties are determined by a complex combination of its elemental composition, atomic geometry, microstructure, morphology (e.g., nanoparticles, heterostructures, and interfaces), processing history, and environmental factors such as temperature and pressure. Furthermore, inorganic materials knowledge is often inherently intertwined such that the relations may only be valid between one entity type and a compound entity (itself comprised of several entities and relationships). -For example, we may consider zinc oxide nanoparticles (a compostion “ZnO" linked to the morphology “nanoparticles") to be a catalyst, but “ZnO" and “nanoparticles" alone are not necessarily catalysts in - -Fig. 1 | Schematic comparison of previous relation extraction (RE) methods to this work. The objective of each method is to extract entities (colored text) and their relationships from unstructured text. a An example multi-step pipeline approach first performs entity recognition, then intermediate processing such as coreference resolution, and finally classification of links between entities. b seq2seq - -themselves. When parts of these compound relations are lost, scientific meaning will change. A sample of an “epitaxial La-doped thin film" of HfZrO4 will have different physical properties than a “La-doped thin film" of HfZrO4 and a “La-doped" sample of HfZrO4. In theory, relationships between n entities can be modeled as n-tuples (e.g., ("ZnO", “nanoparticles", “catalyst")), but comprehensively enumerating all possible variations is both impractical and not amenable to conventional relation extraction methods, since a sufficient number of training examples is required for each relation type. For example, a model extracting 10 distinct entity classes may have 10C3 = 120 3-tuple entity relation types, each requiring at least several annotation examples. -Current relation extraction models are not designed to practically extract or preserve such kinds of highly complex, intricately related, and hierarchical relationships between arbitrary numbers of named entities; a more flexible strategy is required. - -Large language models (LLMs) such as GPT-3/426,27, PaLM28, Megatron29, LLaMA 1/230,31, OPT32, Gopher33, and FLAN34 have been shown to have remarkable ability to leverage semantic information between tokens in natural language sequences of varying length. They are particularly adept at sequence-to-sequence (seq2seq) tasks, where text input is used to seed a text response from the model. In this paper, we will refer to these inputs as “prompts" and the outputs as “completions." Use cases for seq2seq are broad35 and include machine translation36, answering general factual knowledge questions33,37, performing simple arithmetic33, translating between languages36,38, summarizing text28,39, and chatbot applications26,40. It stands to reason that these models may also be adept at complex scientific information extraction. - -Recently, end-to-end methods that use a single machine learning model have been investigated for joint named entity recognition and - -approaches encode relationships as 2-tuples in the output sequence. Named entities and relationship links are tagged with special symbols (e.g., “@FORMULA@", “@N2F@"). c The method shown in this work outputs entities and their relationships as JSON documents or other hierarchical structures. - ---- PAGE 3 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Table 1 | Overview of approaches tested on the three materials information extraction tasks - -Task Schema Training samples Task level Completion format - -Doping Doping-JSON 413 sentences Sentence JSON - -Doping Doping-English 413 sentences Sentence English sentences - -Doping DopingExtra-English 413 sentences Sentence English sentences - -MOFs MOF-JSON 507 abstracts Abstract JSON - -General Materials General-JSON 634 abstracts Abstract JSON - -All three tasks are tested with a JSON schema, and we additionally test the doping task with alternate schemas resembling written English. The MOF and general materials models are trained and evaluated on abstracts, while doping tasks are evaluated on sentences. - -Nature Communications| (2024) 15:1418 3 - -relation extraction (NERRE)41–43. These methods take a sequence-tosequence approach where a model is trained to output tuples of two or more named entities and the relation label belonging to the predefined set of possible relations between them (Fig. 1, middle row). These methods perform well on relation extraction, but they fundamentally remain n-ary relation extraction systems that are not suited to highly intricate and hierarchical NERRE. - -In the domain of materials science, Huang & Cole recently finetuned a BERT model on battery publications and trained a model to enhance a database of NLP-extracted battery data11. Their approach employed a “question and answer" (Q/A) approach that extracted limited device-level information (e.g., “What is the cathode?", “What is the anode?", “What is the electrolyte?") in tandem with conventional information extraction methods11. We note that this approach cannot be used on passages that contain information about more than one device, and it required the BERT language model to be trained on hundreds of thousands of battery research papers before being finetuned on the Q/A task. More recently, Zheng et al.44 designed a prompt-engineering approach (ChemPrompt w/ ChatGPT45) for extracting data from scientific papers. This method is focused on structuring text into tabular forms, creating semi-structured summaries, and collating existing knowledge from the pretraining corpus. -Similarly, Castro Nascimento and Pimentel46 examined ChatGPT’s general knowledge of chemistry; however, they find that, as opposed to methods using considerable prompt engineering47, ChatGPT without prompting “tricks" performs poorly on several simple tasks in chemistry. Xie et al.’s48 approach utilizes LLMs fine-tuned on a large, broad materials science corpus for a range of Q/A, inverse design, classification, and regression tasks. While these methods44,46–48 - -demonstrate LLMs might act as materials science knowledge engines, they have not been shown to extract structured representations of complex hierarchical entity relationships generalizing outside of the pretraining corpus. - -In this work, we investigate a simple approach to complex information extraction where a large language model is fine-tuned to simultaneously extract named entities and their relationships. This method is able to flexibly handle complex inter-relations (including cases where information exists as lists of multiple items) without requiring enumeration of all of possible n-tuple relations or preliminary NER. Our approach differs from the supervised learning (e.g., regression and classification for chemistry) and inverse design approaches of Jablonka et al.49,50 and Xie et al.48; rather than using LLMs to directly influence design or predict properties, we aim to (accurately) extract structured hierarchies of information for use with downstream models. We fine-tune a pretrained large language model (e.g., GPT-326 or Llama-231) to accept a text passage (for example, a research paper abstract) and write a precisely formatted “summary" of knowledge contained in the prompt. This completion can be formatted as either English sentences or a more structured schema such as a list of JSON documents. To use this method, one only has to define the desired output structure—for example, a list of JSON objects with a predefined set of keys—and annotate ~100–500 text passages using - -this format. The LLM is then fine-tuned on these examples, and the resulting model is able to accurately output extracted information in the same structured representation, such as the format shown in Fig. 1. -In essence, a domain expert can show an LLM-NERRE model both what it should extract and how that information should be represented, and then the model learns how to perform the task independently. - -This method shows strong performance using both OpenAI’s GPT-3 (closed source) and Llama-2 (open access) on both sentence-level and document-level materials information extraction. -Moreover, the method can leverage online LLM APIs, which allows users to train bespoke models without extensive knowledge of how LLMs work internally; the LLM may be simply treated by the user as a black-box that transforms passages into precisely-formatted, structured summaries of scientific text. Therefore, researchers may use this method with little NLP experience. We also discuss how intermediate models can be used to pre-suggest entities for annotation, vastly increasing the speed and ease of annotating documents so that large training sets can be constructed relatively quickly. Although the example tasks shown are from materials science, the generality and accessibility of the method implies it may be readily applied to other domains such as chemistry, health sciences, or biology. In particular, this approach does not appear to require fine-tuning on a large corpus of domain-specific data (e.g., millions of article abstracts or paragraphs) as in previous methods; rather, the comprehensive pretraining of the LLMs along with the user-provided annotations are sufficient to accomplish a broad array of complex tasks. - -Results We use the described approach on three joint named entity recognition and relation extraction (NERRE) materials information extraction tasks: solid-state impurity doping, metal–organic frameworks (MOFs), and general materials information extraction. Details for each dataset are summarized in Table 1. Further details of each task are presented in the Methods section. Briefly, the solid-state impurity task is to identify host materials, dopants, and potentially additional related information from text passages (sentences). The MOF task is to identify chemical formulae, applications, guest species, and further descriptions of MOF materials from text (materials science abstracts). The general materials information task is to identify inorganic materials, their formulae, acronyms, applications, phase labels, and other descriptive information from text (materials science abstracts). The general and MOF models were trained on data including normalization and error correction, while doping models were trained to extract data exactly as it appears in text. Each base LLM model is fine-tuned per-task to adhere to a particular schema that encapsulates the entities of interest, relevant relationships, and format. All schemas are shown in Table 1 and further details are available in the Methods and Supplementary Note 1. - -Relation extraction performance A comparison between GPT-3 and Llama-2 on NERRE precision, recall, and F1 scores across the three tasks using a JSON schema is shown in Table 2. Details on each of the task’s JSON schemas are explained in the Methods section. The performances are calculated with an exact word- - ---- PAGE 4 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Table 2 | Named entity recognition and relation extraction scores for three tasks in materials science using models with a JSON output schema - -Doping host-dopant 0.772 0.684 0.726 0.836 0.807 0.821a - -General formula-name 0.507 0.429 0.456 0.462 0.417 0.367 - -General formula-acronym 0.500 0.250 0.333 0.333 0.250 0.286 - -0.538 0.439 0.482 0.551 0.432 0.47 - -General formula-application 0.542 0.543 0.537 0.545 0.496 0.516 - -General formula-description 0.362 0.35 0.354 0.347 0.342 0.340 - -MOFs name-formula 0.425 0.688 0.483 0.460 0.454 0.276 - -MOFs name-guest specie 0.789 0.576 0.616 0.497 0.407 0.408 - -MOFs name-application 0.657 0.518 0.573 0.507 0.562 0.531 - -MOFs name-description 0.493 0.475 0.404 0.432 0.411 0.389 - -Exact match (E.M.) scores are evaluated on a per-word basis, and links are only correct if both entities and the relationship are correct. The exact match metric scores output that contains the correct information but is written differently as incorrect, making such scores a rough lower bound on the true performance of models. F1, precision, and recall reflect the scores on a hold out test set for doping models and averages over five cross-validation sets for the general and MOF models. - -Nature Communications| (2024) 15:1418 4 - -Task Relation E.M. Precision (GPT-3) - -E.M. Recall (GPT-3) - -General formula-structure/phase - -aBest F1 scores for each task are shown in bold. - -match basis, a lower-bound metric described in more detail in the Methods section. Because this joint task involves both named-entity recognition and relation extraction, it reflects both the NER and RE performance of the models (a relation cannot be correctly identified if the entities are not correct.) GPT-3 achieves the highest F1 scores for the General and MOF tasks across all the entity relationships we tested. -Exact match F1 scores for these two extraction tasks are generally ~30% lower than in the host-dopant task. The highestF1 for the general task is found for relationships between formulae and applications (F1 = 0.537) while formula-acronym and formula-description relationships are much less reliable. A similar finding occurs for the MOF task, where the name-application (F1 = 0.573) and name-guest species (F1 = 0.616) relationships are extracted most accurately. The Llama-2 NERRE scores are on average 20 −30% lower than their GPT-3 counterparts, indicating a significant advantage for GPT-3. In the dopant task, Llama-2 has the highest precision (0.836), recall (0.807), and F1 (0.821), representing an improvement of 13% over GPT-3 wrt. F1. - -The F1 scores for the general and MOF tasks in Table 2 are generally 0.3–0.6, which is, on first inspection, seemingly too low to be useful for a large scale information extraction task. However, the scores for the MOF and general tasks are subject to an important caveat. These tasks’ annotations include implicit normalization (e.g. -“Lithium ion" →“Li-ion") and error correction ("silcion" →“silicon"), while the doping task aims to extract hosts and dopants exactly as they appear in text. Thus, the exact word-match basis scores shown above are an approximate lower bound on information extraction performance, since this metric compares only exact matches between words. -When outputs of the general and MOF models are read by human experts, it becomes obvious that the models are often extracting true information with slight changes in phrasing or notation. There is also an effect on performance from inherent ambiguity in real-world information extraction tasks. For example, in MOF information extraction, MOF names (e.g., “ZIF-8") are qualitatively easier to delimit than descriptions (e.g., “mesostructured MOFs formed by Cu2+ and 5- hydroxy-1,3-benzenedicarboxylic acid"), which can be written with many different wordings. - -To account for these factors, we manually scored outputs against the original human (true) annotations for a random 10% test set of the general materials information extraction dataset. We calculated “manual scores" by marking extractions as correct if the core information from entities is extracted in the correct JSON object (i.e., grouped with the correct material formula) and incorrect if they are in the wrong JSON object, are not extracted at all, or are not plausibly inferred from the original abstract. In contrast to the exact match - -E.M. F1 (Llama-2) - -E.M. F1 (GPT-3) - -E.M. Precision (Llama-2) - -E.M. Recall (Llama-2) - -Table 3 | Manual scores for the general materials task using GPT-3 with General-JSON schema - -Extraction F1 - -Entity Extraction recall Extraction precision - -formula 0.943 0.943 0.943 - -name 0.692 1.0 0.818 - -acronym 0.667 0.400 0.500 - -applications 0.797 0.870 0.832 - -structure or phase 0.754 0.920 0.829 - -description 0.576 0.905 0.704 - -Scores measure the model’s ability to extract inter-related data together (i.e. assigning entities correct labels and grouping them appropriately). - -scores (Table 2), manual scores allow for flexibility with respect to three aspects: (1) entity normalization, (2) error correction, and (3) multiple plausible annotations of an entity under different labels (e.g., “thermoplastic elastomer" may be considered either an application or description). Whereas Table 2 assesses whether the model can extract pairs of words exactly as they appear in the true annotation, the manual scores shown in Table 3 assess if the model extracts equivalent information to that of the true annotation - regardless of the exact form. Simply, if a domain expert would agree the model’s extraction and the true extraction are equivalent, the model’s extraction is marked as correct. We provide precise details on this procedure in the Methods section and detailed examples with explanations in Supplementary Discussion 4. - -Table 3 shows the adjusted scores based on manual scoring. We stratify these scores by entity; the “name", “acronym", “application", “structure", and “description" manual scores can be compared to Table 2’s exact-match formula-{name, application, structure, description} relation scores. For example, “description" reflects how often the model extracts a description entity which is both equivalent in meaning to that of the true annotation (according to a domain expert) and is grouped in the correct JSON object (linked to the correct formula). We see that exact-match scoring severely underpredicts performance for materials’ names (0.456 vs 0.818), applications (0.537 vs 0.832), structures/phases (0.482 vs 0.829), and descriptions (0.354 vs 0.704). Manual scoring reveals that our models are actually able to correctly extract structured knowledge from scientific text on a wide variety of materials science topics, and readers can inspect the model’s output on test set examples (included in the Supplementary Discussion 4) for themselves. We observe that - ---- PAGE 5 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 5 - -acronyms have the lowest information extraction scores, which we attribute to the fact that acronyms are relatively rare in the training class compared to the others (appearing in only 52 abstracts across the entire dataset, ~9% of the documents) and that the model can confuse acronyms with chemical formulae (e.g., “AuNP" is the acronym for gold nanoparticle but is also a valid chemical formula). Usually, context clues are the only way to disambiguate cases like this, and we expect including more training data with acronyms may improve the acronym extraction score. - -Overall, these scores indicate the model is highly capable at extracting meaningfully complex representations of material knowledge from passages. Precision scores for the various categories (other than acronyms) are all roughly 0.87 or better, which indicates that when information is extracted, it contains true relational information from the passage rather than spurious connections. - -The advantage of the LLM-NERRE method reflected in these manual scores is the ability to automatically correct errors and normalize common entity patterns. While the doping models were trained to extract snippets of text exactly as they appeared in the text prompt, the General-JSON model’s training data included simple normalizations and error corrections of entities. For example, the erroneous inclusion of white spaces in chemical formulae is common in the raw abstract text. We observe that including corrected formulae instead of the raw string in the output training sequences results in LLMs that automatically resolve extracted entities to cleaner forms. For example, “Li Co O2" is corrected to “LiCoO2" by the model without additional post-processing. Similarly, because there are sufficient training examples, the models using General-JSON schema resolve text such as “PdO functionalized with platinum" to a normalized form such as {formula: “PdO", description: ["Pt-functionalized"]}. The built-in normalization and correction abilities of LLM models may prove useful for domain specialists who desire structured entity formats rather than exact string excerpts pulled directly from the text, as entity normalization is a common post-processing task. - -Effect of different schemas For the host-dopant extraction task, we evaluated three different output schemas to determine whether one format of output is exclusively better than any other. The models using the Doping-English schema output English sentences with a particular structure (e.g., “the host ’’ was doped with ’’.") and the DopingExtra-English models likewise output English sentences but also includes some additional information (e.g., if one of the hosts is a solid solution and/or the concentration of a particular dopant). For the Doping-JSON schema, we used a JSON object schema with keys “hosts", “dopants", and “hosts2dopants" (which in turn has a key-value object as its corresponding value). For readers familiar with the Python programming language, these are identical to python dictionary objects with strings as keys and strings or other dictionaries as values. We include a baseline comparison to seq2rel41, a comparable sequence-tosequence method, trained on the same doping dataset. We also compare to MatBERT-Doping5, an NER model trained on ~450 abstracts, combined with a simple heuristic for determining host-dopant relationships; that is, all hosts and dopants within the same sentence (sample) are related. We refer to this model as MatBERT-Proximity. Full descriptions and examples of all schemas are available in the Methods section, and further details on seq2rel and MatBERT-Proximity are available in Supplementary Notes 4–5. Because the general materials information extraction and MOF information extraction tasks are far more complex, we did not attempt to train models to output English sentences (as opposed to JSON formatted strings), as the resulting sequences would be difficult to parse into structured database entries. - -We find that all three of our LLM-NERRE host-dopant extraction models perform significantly better than either the MatBERT- Proximity or seq2rel baseline models. Of the two baselines, the - -seq2rel model achieves higher precision (0.420) and recall (0.605) resulting in F1 = 0.496, which is slightly higher than MatBERT-Proximity (0.390) but substantially lower than any of the LLM-NERRE models. -This seq2rel benchmark model is derived from the PubMedBERT51 - -pretrained BERT model as per the original implementation41, and it may be possible to improve the seq2rel method by using a BERT model pretrained exclusively on materials text rather than biomedical text. -However, this improvement is not expected to be dramatic because previous comparisons between SciBERT and MatBERT show relatively minor differences in materials NER tasks5. We also observe that all three LLM-NERRE models exceed the performance of the two baselines in pure NER performance (see Supplementary Discussion 2) despite being trained on less text than the MatBERT-NER model (413 sentences vs. 455 abstracts.) Of the six LLM-based models, the Llama-2 model with Doping-JSON schema performs the best (F1 = 0.821) with GPT-3/ DopingExtra-English (F1 = 0.809) and Llama-2/Doping-English (F1 = 0.814) both within a 2% margin. We summarize both LLMs' performances with all three schemas alongside the baseline models in Table 4. - -Within the GPT-3 results, the DopingExtra-English and Doping- English schemas have the highest F1. In particular, GPT-3/DopingExtra- English tops the GPT-3 models despite being trained on the same number of samples as the Doping-English and Doping-JSON models. -This is notable because GPT-3/DopingExtra-English is both more accurate and more capable (i.e., this model extracts “results" and “modifiers" entities in addition to host-dopant relationships) than the GPT-3 models using other schemas. The opposite observation is true of the Llama-2 models, where the JSON format outperforms both English schemas and the DopingExtra-English schema suffers from low precision (0.694). Roughly, the GPT-3 models tend to perform optimally when using natural language like schemas, while Llama-2 performs optimally using JSON. - -Human-in-the-loop annotation As a separate experiment, we evaluated the use of partially trained LLMs in a “human-in-the-loop" annotation process for constructing outputs with the GPT-3/General-JSON, as seen in Fig. 2. In each trial of the experiment, the human annotator received 10 abstracts and 10 schemas that were pre-populated by an intermediate version of the model which was trained on n samples of training data (n = 1, 10, 50, 100, 300). Instead of completing annotations from scratch, the human annotator corrected these intermediate models’ suggestions, and the time to complete each annotation was recorded. As shown in Fig. 3, the annotation time sharply decreases as the number of training samples used in the intermediate models increases; the n = 300 intermediate model was able to reduce the average annotation time per abstract by 57% in comparison with the n = 1 model, indicating that the model was completing many sections of the annotation correctly. - -At low numbers of training samples, the models’ predictions are not valid JSON objects, and the annotator had to redo annotations from scratch. At higher numbers of training samples, particularly those above 50, the intermediate model predictions required very little error correction from the annotator. As a lower bound, we also report the time needed by the annotator to simply verify whether an entry was entirely correct (verification time) which reflects the annotation rate of a human annotator using a perfect model, which only requires the human annotator to check the outputs. We find that by three metrics (time per abstract, time per material entry, and time per prompt token), the human annotator annotated substantially faster with a welltrained model in the loop (n samples > 50) than with a poorly trained model (n samples < 50) or no model. For example, the n = 300 model reduced the annotation time per token ~60% compared to the n = 1 model and is only 38% slower than the verification time. Given additional training samples for intermediate models, we expect the annotation to asymptotically approach the verification time. Thus, this - ---- PAGE 6 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 6 - -Fig. 2 | Overview of the proposed sequence-to-sequence approach to document-level joint named entity recognition and relationship extraction task. In the first step, lists of JSON documents are prepared from abstracts according to a predefined schema, and the large language model (LLM) is trained. In the second step, this preliminary (intermediate) model is used to accelerate the preparation of additional training data by pre-annotation with the - -Fig. 3 | Annotation time as a function of intermediate large language model (LLM) fine-tuning samples for the named entity recognition and relation extraction (NERRE) method. We show the time taken for a domain expert to annotate new abstracts for the general materials chemistry task with assistance - -method may serve as a useful tool for building even larger benchmark datasets for information extraction tasks. - -We note that the LLM-NERRE method requires the model to learn both the correct structure of the output data as well as the information to populate into that data structure, particularly when asking the model to output English sentences that can later be parsed to a structured format. To determine the minimum number of training examples required for models with sentence-format outputs that have a parseable sentence structure, we trained intermediate models on varying training set sizes for the GPT-3/Doping-English model. Precision, recall, and F1 scores as a function of training set size are plotted in Fig. 4. We observe that output sequences are not properly structured for training set sizes below ~10 samples, but there is a sharp increase in the number of correctly structured outputs at ~20 samples, which seems to be the minimum number of examples GPT-3 needs to learn a desired output format when using simple sentence-type schemas. - -partially trained model and manual correction. An example error is shown highlighted in red. This step may be repeated multiple times with each subsequent partial fine-tuning improving in performance. In the final step, the LLM is fine-tuned on the complete dataset and used for inference to extract desired information from new text. - -from intermediate (partially-trained) LLM-NERRE models on a (a) word basis, (b) material entry basis, and (c) token basis. Outputs from models trained on more data contain fewer mistakes and require less time to correct. Source data are provided as a Source Data file. - -Discussion Overall, we find excellent performance on three diverse tasks for materials science and engineering: solid-state impurity doping, metal–organic frameworks, and general materials relations. The nontechnical nature of this approach implies scientists without NLP training can utilize existing models such as GPT-3 to extract large structured relational datasets for highly-specific problems. As the LLM is treated essentially as a black-box, we anticipate this approach may be used for LLMs other than GPT-3 or Llama-2, including LLMs released in the near future. We hope this approach enables domain specialists to rapidly extract relational datasets for the advancement of scientific knowledge. - -The NERRE scores in Tables 2–4 provide a quantitative score for performance, but some of the best features of this method are not directly shown by F1 scores. The primary advantage of this method is its accessibility and ease-of-use, as LLM-NERRE requires only - ---- PAGE 7 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 7 - -Fig. 4 | Test set performance vs. number of training samples for the doping extraction task using GPT-3 with the Doping-English schema. This schema specifically requires the model to learn a new and specific sentence structure to use as the output. We separate scores by (a) host-dopant links (relations), (b) host entities alone, and (c) dopant entities alone. We note that below approximately 10 samples, the scores are zero because the model has not learned the specific structure of the desired output sentences. Source data are provided as a Source Data file. - -specifying a basic schema, annotating a minimal number of examples, and fine-tuning a model via a publicly available API without extensive NLP knowledge or hyperparameter tuning; the final result is a useful model with the ability to extract specialized technical information that is nuanced and semantically complex. Additionally, error correction and normalization may be embedded directly into training examples to reduce the need for post-processing. In essence, one can show an LLM-NERRE model both what it should extract and how it can be condensed and presented. - -Like many others, we have found using a human-in-the-loop process can help decrease the time required to collect a full training set52. Our particular process is shown in Fig. 2. Annotation for scientific information extraction tasks is often a tedious and error-prone process whereas checking intermediate model outputs for errors is qualitatively easier than creating annotations from scratch. Additionally, fine tuning GPT-3/Llama-2 requires fewer training examples to match or exceed the performance of BERT-based models. Figure 4 shows how performance of the fine-tuned models improves quickly at relatively small training set sizes. However, more and more text-completion pairs are required to achieve the same rate of improvement as training set size is increased. - -One limitation of our model is that valid output schema formatting is not rigorously enforced in the generation step. The LLM may, for any given sample, output an unparsable sequence. This is particularly apparent when the inference token limit is less than 512 tokens and the schema is JSON, as JSON schema typically requires a larger number of tokens for correct formatting. For example, a nearly-correct output sequence containing 10+ correct entities may be missing a “}" ending character and therefore will not be parsable. Outputs are nearly always parsable (~99% success rate), especially as the number of training examples increases. Failures predominantly occur when the sample exceeds the prompt-completion token limit of the LLM (early termination), which in this work was 512-1024 tokens for both GPT-3 and Llama-2. Because of this, some abstracts that are too long or too dense with information to be processed with this method. This was the - -Table 4 | Comparison of large language models with different joint named entity recognition and relation extraction (NERRE) schemas to baseline models on host-dopant extraction task - -F1 (exact match) - -Model Schema Precision (exact match) - -Recall (exact match) - -MatBERT- Proximity - -n/a 0.377 0.403 0.390 - -Seq2rel n/a 0.420 0.605 0.496 - -GPT-3 Doping-JSON 0.772 0.684 0.725 - -GPT-3 Doping-English 0.803 0.754 0.778 - -GPT-3 DopingExtra- English - -0.820 0.798 0.809 - -Llama-2 Doping-JSON 0.836a 0.807 0.821 - -Llama-2 Doping-English 0.787 0.842 0.814 - -Llama-2 DopingExtra- English - -0.694 0.815 0.750 - -NERRE exact match scores are evaluated on a per-word basis, and links are only correct if both entities and relationship are correct. DopingExtra-English scores here refer to only host-dopant relation prediction. We note that exact match scores output that contains the correct information but is written differently as incorrect, making such scores an approximate lower bound on the true performance of models. F1, precision, and recall are computed on a hold-out test set from 77 sentences. Best scores for precision, recall, and F1 are shown in bold. - -aBest scores among all models in each category (exact match precision, recall, F1) are shown in bold. - -case in the few unparseable completions where the passage and partial completion exceed the token limit and cut off early before the full completion could be output by the model. This limitation may be mitigated by increasing the token limit up to 2048 (GPT-3) or 4096 (Llama-2); we expect the token limitation will become less of a concern as the maximum token size of such models increase. - -Another limitation is the tendency of LLMs to generate or invent information that is not actually present in the input text, a phenomenon termed “hallucination"53,54 in LLM literature. The main manifestation of hallucination we observed was the addition of names or chemical formulae for a material when only one or the other was mentioned (for example writing “SiO2" in the formula field even though the paragraph only mentions “silica"). Although these hallucinations could potentially be correct, because the source text does not include them, we believe they should not be included in the output of information extraction models. We could enforce this by the requirement that all extracted entities should occur word-for-word in the source text, but the fact that these models do not extract phrases exactly can also be a useful feature because it allows for automatic entity normalization. For example, an abstract may mention both “p- ZnSe doped with N" and “nitrogen-doped ZnSe" in the same passage. Is “doped with N" or “nitrogen-doped" the correct description to extract? -Clearly, both are correct and either one could be reasonably chosen. -Moreover, “N-doped" could also be extracted and would be factually correct even though “N-doped" never occurs in the passage. Because LLMs can learn implicit normalization rules, if the annotator is consistent in how they normalize cases like this (such as always using “Xdoped" and/or “p(n)-type"), the model generally follows the same normalization scheme and it can greatly reduce the amount of entity normalization post-processing required later. We differentiate this from hallucination in that the inference is fully justified by the content in the source text rather than simply plausible. - -Finally, the choice of LLM poses a practical tradeoff for researchers: essentially, ease of use vs. control. Using a proprietary LLM such as GPT-3 through an online API enables the LLM in our method to be treated as a “black box", and abstracting away LLM finetuning details allows researchers to focus entirely on their domainspecific information extraction tasks. However, the underlying LLM is exclusively controlled by a private entity, posing problems of - ---- PAGE 8 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 8 - -reproducibility and security. Regarding security, potentially sensitive or confidential data must be sent to the entity for processing; -regarding reproducibility, the models cannot be shared, and the entity controlling the LLM may atany time change the model, amend the finetuning method, or revoke access to the model altogether. More, the cost for inference on large datasets using trained models may be prohibitive. In contrast, using self-hosted models such as Llama-231 or GPT-NeoX 20B55 favors control over ease of use. The weights and code for the model are fully accessible, and inference cost is restricted only by the user’s budget on a cluster with capable GPUs. However, successfully running, fine-tuning, and deploying LLMs such as Llama-2 on cluster infrastructure is non-trivial for many scientists. Cloud-hosted open-access models (e.g., Llama-2 hosted on a managed cloud instance) may provide a solution to the ease of use vs. control tradeoff, as the technical details of fine-tuning are abstracted away from the user but the fine-tuned models themselves can remain open-access. Similarly, zero-shot approaches without fine-tuning may make scientific information extraction more accessible at the expense of accuracy(see Supplementary Discussion 6). Methods for reducing the number of parameters needed for LLM inference and fine-tuning56–59 are also a promising avenue for reducing the complexity and cost of self-hosting LLMs. As these methods advance and LLM codebases become more mature, we expect fine-tunable models compatible with LLM-NERRE will become simultaneously powerful, easy to self-host, reproducible, and under researchers’ full control. We hope the code examples of both fine-tuning and running inference using the published model weights we provide in Methods are a first step in the direction of powerful and open source NERRE models. - -In summary, this work demonstrates that LLMs that are fine-tuned on a few hundred training examples are capable of extracting scientific information from unstructured text and formatting the information in user-defined schemas. This is in contrast to past models which were successful in extracting entities from text but struggled to relate those entities or structure them in meaningful ways. The proposed method is simple and broadly accessible given the APIs and interfaces currently available such as GPT-3. Furthermore, we have made the Llama-2 LoRA weights of all models shown in this paper available for download (see Methods and Code Availability), allowing researchers to investigate the LLM-NERRE method on their own hardware. We expect these advancements to greatly facilitate the rate and accuracy by which historical scientific text can be converted to structured forms. - -Methods General sequence-to-sequence NERRE We fine-tune Llama-2 and GPT-3 models to perform NERRE tasks using 400−650 manually annotated text-extraction (prompt-completion) pairs. Extractions contain the desired information formatted with a predefined, consistent schema across all training examples. These schemas can range in complexity from English sentences with predefined sentence structures to lists of JSON objects or nested JSON objects. In principle, many other potential schemas (e.g., YAML, psuedocode) may also be valid, though we do not explore those here. -Once fine-tuned on sufficient data adhering to the schema, a model will be capable of performing the same information extraction task on new text data with high accuracy. The model outputs completions in the same schema as the training examples. We refer to this approach generally as “LLM-NERRE". - -Our general workflow for training GPT-3 and Llama-2 to perform NERRE tasks is outlined in Fig. 2. Annotations are performed by human domain experts to create an initial training set, and then a partially trained model (GPT-3) is used to accelerate the collection of additional training examples. Fine-tuning is then performed on these examples to produce a “partially trained" model, which is used to pre-fill annotations that are subsequently corrected by the human annotator before being added to the training set. Once a sufficient number of - -annotations have been completed, the final fine-tuned model is capable of extracting information in the desired format without human correction. Optionally, as illustrated in Figs. 5 and 6, the structured outputs may be further decoded and post-processed into hierarchical knowledge graphs. - -Task and schema design Solid-state impurity doping schema. The Doping-English and Doping-JSON schemas aim to extract two entity types (host and dopant) and the relations between them (host-dopant), returned as either English sentences or a list of one or more JSON objects.Hosts are defined as the host crystal, sample, or material class along with crucial descriptors in its immediate context (e.g., “ZnO2 nanoparticles", “LiNbO3", “half-Heuslers"). Dopants are taken to be any elements or ions that are minority species, intentionally added impurities, or specific point defects or charge carriers ("hole-doped", “S vacancies"). One host may be doped with more than one dopant (e.g., separate singledoping or co-doping), or the same dopant may be linked to more than one host material. There may also be many independent pairs of dopant-host relations, often within a single sentence, or many unrelated dopants and hosts (no relations). We impose no restriction on the number or structure of the dopant-host relations beyond that each relation connects a host to a dopant. The Doping-JSON schema represents the graph of relationships between hosts and dopants within a single sentence, where unique keys identify dopant and host strings. The model aims to learn this relatively loose schema during fine-tuning. A separate key, “hosts2dopants", describes the pairwise relations according to those unique keys. The Doping-English schema encodes the entity relationships as quasi-natural language summaries. -The Doping-English schema represents the same information as the Doping-JSON schema, but more closely mimics the natural language pre-training distribution of the LLMs we tested. When there are multiple items to extract from the samesentence, the output sentences are separated by newlines. - -For the DopingExtra-English schema, we introduce two additional entities: modifiers and result, without explicit linking (i.e., NER only). The results entity represents formulae with algebra in the stoichiometric coefficients such as AlxGa1−xAs, which are used for experiments with samples from a range of compositions or crystalline solid solutions (e.g., CaCu3−xCoxTi4O12). We also include stoichiometries where the algebra is substituted (i.e., x value specified) and the doped result is a specific composition (e.g., CaCu2.99Co0.1Ti4O12). -Modifiers are loosely bounded entity encapsulating other descriptors of the dopant-host relationship not captured by dopant, host, or result. These can be things like polarities (e.g., “n-type", “n-SnSe"), dopant quantities (e.g., “5 at.%", “x < 0.3"), defect types (e.g., “substitutional", “antisite", “vacancy") and other modifiers of the host to dopant relationship (e.g., “high-doping", “degenerately doped"). -These entities (host, dopant, result, and modifiers) were chosen to define a minimal effective schema for extracting basic doping information. - -All doping-related models are trained to work only on single sentences. The main motivation for this design choice is that the vast majority of dopant-related data can be found within single sentences, and the remaining relational data is often difficult to resolve consistently for both human annotators and models. We expand on problems with annotations and ambiguity in Supplementary Discussion 5 and we further explain the doping task schemas in Supplementary Note 1. - -General materials information schema. In our previous work4,5, we focused on NER for a specific set of entity types that are particularly relevant in materials science: materials, applications, structure/phase labels, synthesis methods, etc. However, we did not link these labeled entities together to record their relations beyond a simple “bag-of- - ---- PAGE 9 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 9 - -Fig. 5 | Diagrams of doping information extraction using large language models (LLMs) for joint named entity and relation extraction (NERRE). In all three panels, an LLM trained to output a particular schema (far left) reads a raw text prompt and outputs a structured completion in that schema. The structured completion can then be parsed, decoded, and formatted to construct relational diagrams (far right). We show an example for each schema (desired output structure). Parsing refers to the reading of the structured output, while decoding refers to the programmatic (rule-based) conversion of that output into JSON form. Normalization and postprocessing are programmatic steps which transform raw strings (e.g., “Co+2") into structured entities with attributes (e.g., Element: Co, - -Fig. 6 | Diagrams of general information extraction and metal organic framework (MOF) information extraction using large language models (LLMs) for joint named entity and relation extraction (NERRE). In both panels, an LLM trained using a particular schema (desired output structure, far left) is prompted with raw text and produces a structured completion as JSON. This completion can then be parsed to construct relational diagrams (far right). Each task uses a different schema representing the desired output text structure from the LLM. -a Schema and labeling example for the general materials-chemistry extraction task. -Materials science research paper abstracts are passed to an LLM using General- - -entities" approach. In this work, we train an LLM to perform a “general materials information extraction" task that captures both entities and the complex network of interactions between them. - -The schema we have designed for this task encapsulates an important subset of information about solid compounds and their applications. Each entry in the list, a self-contained JSON document, corresponds one-to-one with a material mentioned in the text. Materials entries are ordered by appearance in the text. The root of each - -Oxidation state +2). a Raw sentences are passed to the model with Doping-English schema, which outputs newline-separated structured sentences that contain one host and one or more dopant entities. b Raw sentences are passed to a model with Doping-JSON schema, which outputs a nested JSON object. Each host entity has its own key-value pair, as does each dopant entity. There is also a list of host2dopant relations that links the corresponding dopant keys to each host key. c Example for the extraction with a model using the DopingExtra-English schema. This first part of the schema is the same as in a, but additional information is contained in doping modifiers, and results-bearing sentences are included at the end of the schema. - -JSON schema, which outputs a list of JSON objects representing individual material entries ordered by appearance in the text. Each material may have a name, formula, acronym, descriptors, applications, and/or crystal structure/phase information. -b Schema and labeling example for the metal-organic frameworks extraction task. -Similar to the General-JSON model, the MOF-JSON model takes in full abstracts from materials science research papers and outputs a list of JSON objects. In the example, only MOF name and application were present in the passage, and both MOFs (LaBTB and ZrPDA) are linked to both applications (luminescent and VOC sensor). - -entry starts with a compound’s name and/or its chemical formula. If a name or formula is not mentioned for a material, no information about that material is extracted from the text. We also extract acronyms mentioned for a material’s name/formula, although in cases where only an acronym is mentioned we do not create a material entry for the compound. Compounds that are not solids (ions, liquids, solvents, solutions, etc) are generally not extracted. The name, formula, and acronym fields are exclusively given string value in the JSON document - ---- PAGE 10 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 10 - -for each material whereas the description, structure_or_phase, and applications fields are lists of an arbitrary number of strings. -We label this model General-JSON, and an example is shown in Fig. 6 (a). - -Description entities are defined as details about a compound’s processing history, defects, modifications, or the sample’s morphology. For example, consider the hypothetical text “Pt supported on CeO2 nanoparticles infused with Nb...". In this case, the description value for the material object referring to “Pt" might be annotated as "['supported on CeO2']", and the description entities listed for “CeO2" would be "['nanoparticles', 'Nb-doped']". - -Structure_or_phase entities are defined as information that directly implies the crystalline structure or symmetry of the compound. Crystal systems such as “cubic" or “tetragonal", structure names such as “rutile" or “NASICON", and space groups such as “Fd3m" or “space group No. 93" are all extracted in this field. We also include any information about crystal unit cells, such as lattice parameters and the angles between lattice vectors. “Amorphous" is also a valid structure/phase label. - -Applications are defined as high-level use cases or major property classes for the material. For example, a battery cathode material may have "['Li-ion battery', 'cathode']" as its applications entry. Generally, applications are mentioned in the order they are presented in the text, except for certain cases such as battery materials, in which case the type of device is generally mentioned before the electrode type, and catalysts, where the reaction catalyzed is generally listed following the “catalyst" entity in the list (e.- g.,"['catalyst', 'hydrogenation of citral']"). - -More details about the general materials information task schema are provided in the Supplementary Discussion 4. - -Metal–organic framework (MOF) schema. The schema used for the MOF cataloging task is based on the general materials information schema described in the previous section, which was modified to better suit the needs of MOF researchers. We developed this schema to extract MOF names (name), an entity for which there is no widely accepted standard60, and chemical formulae (formula), which form the root of the document. If no name or formula is present, no information is extracted for that instance. In addition, because there is a great deal of interest in using MOFs for ion and gas separation61,62, we extract guest species, which are chemical species that have been incorporated, stored, or adsorbed in the MOF. We extract applications the MOF is being studied for as a list of strings (e.g., "['gasseparation']" or "['heterogeneous catalyst', 'Diels- Alder reactions']") as well as a relevant description for the MOF, such as its morphology or processing history, similar to the general information extraction schema. Entries in the list are generally added inthe order the material names/formulae appear in the text. The MOF extraction model is labeled MOF-JSON, and an example is shown in Fig. 6 (b). - -Comparison baselines and evaluation To compare our model with other sequence-to-sequence approaches to information extraction, we perform a benchmark of two methods on the doping task to compare to the LLM-NERRE models. The first employs the seq2rel method of Giorgi et al.41 for the host-dopant task. -We formatted host-dopant relationships under tags labeled @DOPANT@ and @BASEMAT@ (base/host material), with their relationship signified by @DBR@ ("dopant-base material relationship"); these sequences were constructed from the same training data as the Doping-JSON and Doping-English models. We trained seq2rel to perform sentencelevel extraction with 30 epochs, batch size of 4, encoder learning rate 2 × 10−5, decoder learning rate 5 × 10−4, and pretrained BiomedNLP BERT tokenizer51 (further training details can be found in the Supplementary Note 4). Additionally, we compare against the previously - -published MatBERT doping-NER model5 combined with proximitybased heuristics for linking (see Supplementary Note 5). With this method, a MatBERT NER model pretrained on ~50 million materials science paragraphs and fine-tuned on 455 separate manually annotated abstracts first extracts hosts and dopants and then links them if they co-occur in the same sentence. - -Datasets Datasets were prepared from our database of more than 8 million research paper abstracts63. Annotations were performed by human annotators using a graphical user interface built using Jupyter64, although in principle annotations could be conducted via a simple text editor. To accelerate the collection of training data, new annotations are collected via a “human in the loop" approach where models are trained on small datasets and their outputs are used as starting points and corrected by human annotators (see Fig. 2.) This process of training and annotation is completed multiple times until a sufficiently large set of training data was achieved. Each dataset was annotated by a single domain expert annotator. Class support for each annotated dataset is provided in Supplementary Tables 1-3. - -Doping dataset. Training and evaluation data was gathered from our database of research paper abstracts using the keywords “n-type", “ptype", “-dop", “-codop", “doped", “doping", and “dopant" (with exclusions for common irrelevant keywords such as “-dopamine"), resulting in ~375k total abstracts. All doping tasks were trained on text from 162 randomly selected abstracts, comprising 1215 total sentences and filtered with regular expressions to only include 413 relevant (potentially including doping information) sentences. Doping tasks were tested on an additional 232 sentences (77 relevant by regex) from a separate holdout test set of 31 abstracts. - -General materials dataset. Training and evaluation data was gathered from our abstract database by using keywords for a variety of materials properties and applications (e.g., “magnetic", “laser", “space group", “ceramic", “fuel cell", “electrolytic", etc). For each keyword a materials science domain expert annotated ~10–50 abstracts, which resulted in ≈650 entries manually annotated according to the general materials information schema. Results were evaluated using a 10% random sample for validation, and this procedure was averaged over five trials using different random train/validation splits with no hyperparameter tuning. - -Metal–organic framework dataset. Training and evaluation data was selected from our database using the keywords “MOF", “MOFs", “metal-organic framework", “metal organic framework", “ZIF", “ZIFs", “porous coordination polymer", and “framework material", which produced approximately 6,000 results likely containing MOF-related information. From these, 507 abstracts were randomly selected and annotated by a MOF domain expert. Results were evaluated using the same repeated random split procedure as the general materials dataset in the previous section. - -GPT-3 fine tuning details For all tasks, we fine-tune GPT-3 (‘davinci’, 175B parameters)26 using the OpenAI API, which optimizes the cross-entropy loss on predicted tokens. Doping models were trained for 7 epochs at a batch size of 1, with inference temperature of 0 and output limited to a maximum length of 512 tokens (all doping models) or 1024 tokens (General-JSON, MOF-JSON). The intermediate models shown in Fig. 4 were trained with a number of epochs depending on the number of training samples t: 2 epochs for 20≤t < 26, 4 epochs for 26 < t ≤27, and 7 epochs for t ≥28. -Models for the MOF and general materials extraction tasks were trained for 4 epochs with a batch size of 1. We use a learning rate multiplier of 0.1 and a prompt loss weight of 0.01 but have not performed hyperparameter tuning for these hyperparameters. For all - ---- PAGE 11 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 11 - -tasks, the start and end tokens used were "\n\n\n###\n\n\n" and "\n\n\nEND\n\n\n". - -Llama-2 fine-tuning details Llama-231 fine-tunes were performed using a modified version of the Meta Research Llama-2 recipes repository; the modified repository can be found at https://github.com/lbnlp/nerre-llama. Llama-2 fine-tunes were performed using the 70 billion parameter version of Llama-2 (llama-2-70b-hf) with quantization (8 bit precision). The number of epochs was set to 7 for doping tasks and 4 for the MOF/general tasks. -Llama-2 fine-tunes used parameter efficient fine-tuning (PEFT) using low rank adaptation (LoRA)58 with LoRA r = 8, α = 32 and LoRA dropout of 0.05. Further hyperparameter tuning was not performed. Decoding was done without sampling using greedy decoding to be consistent with GPT-3 decoding setting of temperature = 0, with max tokens = 512 for doping task and 1024 for general and MOF task. More details on the fine tuning and inference parameters are available in the modified repository and Supplementary Note 3. All fine-tuning and inference was performed on a single A100 (Ampere) tensor core GPU with 80GB VRAM. -The fine-tuned weights for each model are provided in the NERRE- Llama repository (url above) along with code and instructions for downloading the weights, instantiating the models, and running inference. - -Evaluation criteria The fuzzy and complex natureof the entities and relationships detailed in the previous section necessitates the use of several metrics for scoring. We evaluate the performance of all models on two levels: - -1. -A relation F1 computed on a stringent exact word-match basis (i.e., how many words are correctly linked together exactly as they appear in the source text prompt). -2. -A holistic information extraction F1 based on manual inspection by a domain expert, which doesn’t require words to match exactly. - -We separately provide a sequence-level error analysis in Supplementary Note 7 and Supplementary Discussion 1. - -NERRE performance. We measure NERRE performance as the ability of the model to jointly recognize entities and the relationships between them. - -Exact word-match basis scoring We score named entity relationships on a word-basis by first converting an entity E into a set of constituent k whitespace-separated words E = {w1, w2, w3, …, wk}. When comparing two entities Etrue and Etest - -that do not contain chemical formulae, we count the number of exactly matching words in both sets as true positives (Etrue ∩Etest) and the mathematical set differences between the sets as false positives (Etest −Etrue) or false negatives (Etrue −Etest). For example, if the true entity is “Bi2Te3 thin film" and the predicted entity is “Bi2Te3 film sample", we record two true positive word exact matches ("Bi2Te3", “film"), one false negative ("thin"), and one false positive ("sample"). Formula-type entities are crucial for identifying materials, so in cases where entities contain chemical formulae, Etest must contain all wi that can be recognized as stoichiometries for any of wi ∈Etest to be considered correct. For example, if the true entity is “Bi2Te3 thin film", and the predicted entity is “thin film", we record three false negatives. Thus, any formula-type entity (Doping host, Doping dopant, General formula, and MOF mof_formula) containing a chemical composition is entirely incorrect if the composition is not an exact match. This choice of evaluation was made to avoid metrics measuring the performance of the model in a misleading way. For example, “Bi2Te3 nanoparticles" and “Bi2Se3 nanoparticles" have very high similarities via Jaro-Winkler (0.977) and character-level BLEU-4 (0.858), but these - -two phrases mean entirely different things—the material’s chemistry is wrong. Under our scoring system, they are recorded as entirely incorrect because the compositions do not match. - -We score relationships between entities on a word-by-word basis to determine the number of correct relation triplets. Triplets are 3-tuples relating word wn j of an entity En to word wm - -k of an entity Em by relationship r, represented as ðwn - -j ,wm - -k ,rÞ. The total set of correct relationships Ttrue for a text contains many of these triplets. A test set of relationships Ttest is evaluated by computing the number of triplets found in both sets (Ttrue ∩Ttest) as true positives and the differences between these sets as false positives (Ttest −Ttrue) or false negatives (Ttrue −Ttest). Entity triplets are also bound to the same requirement for composition correctness if either of the words in the triplet belong to an formula-type entity (host, dopant, formula, mof_formula), i.e., we count all triplets for two entities as incorrect if the formula is not an exact string match. With correct and incorrect triplets identified, F1 scores for each relation are calculated as: - -precision = No. of correct relations retrieved - -ð1Þ - -No. of relations retrieved - -recall = No. of correct relations retrieved - -ð2Þ - -No. of relations in test set - -F1 = 2ð precision  recallÞ - -precision + recall ð3Þ - -To compute triplet scores across entire test sets in practice, we first select a subset of relations to evaluate. We note that this is not a full evaluation of the task we are training the model to perform, which involves linking many interrelated entities simultaneously, but is rather provided to help give a general sense of its performance compared to other NERRE methods. For the doping task, we evaluate host-dopant relationships. For the general materials and MOF tasks, we evaluate relationships between the formula field (formula for general materials, mof_formula for MOFs) and all other remaining fields. For description, structure_or_phase, and applications fields, all of which may contain multiple values, all of the possible formula-value pairs are evaluated. - -Manual evaluation The metrics provided in prior sections demonstrate automatic and relatively strict methods for scoring NERRE tasks, but the underlying capabilities of the LLM models are best shown with manual evaluation. This is most apparent in the case of the General-JSON model, where exact boundaries on entities are fuzzier, precise definitions are difficult to universally define, and annotations include some implicit entity normalization. For example, the text “Pd ions were intercalated into mesoporous silica" may have equivalently have a correct description field for the material “silica" including “Pdintercalated", “Pd ion-intercalated", “intercalated with Pd ions", etc.; -the exact choice of which particular string is used as the “correct" answer is arbitrary. - -To better address scoring of these fuzzy tasks, we introduce an adjusted score based on a domain expert’s manual evaluation of whether the information extracted is a valid representation of the information actually contained in the passage. We term this adjusted score “manual score"; it constitutes a basis for precision, recall, and F1 that quantifies the quality of overall information capture for cases where there may be equivalent or multiple ways of representing the same concept. This score was constructed to better estimate the performance of our model for practical materials information extraction tasks. - -We score entities extracted by annotators but not present in the model’s output as false negatives, except when reasonable variations are present. The criteria for a true positive are as follows: - ---- PAGE 12 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 12 - -1. -The entity comes from the original passage or is a reasonable variation of the entity in the passage (e.g., “silicon" ⟶“Si"). It is not invented by the model. -2. -The entity is a root entity or is grouped with a valid root entity. For the General-JSON model, a root entity is either a material’s formula or name. If both are present, the formula is used at the root. -3. -The entity is in the correct field in the correct root entity’s group (JSON object). - -Manual scores are reported per-entity as if they were NER scores. -However, the requirements for a true positive implicitly include relational information, since an entity is only correct if is grouped with the correct root entity. - -Reporting summary Further information on research design is available in the Nature Portfolio Reporting Summary linked to this article. - -Data availability All data used for this study are available at https://github.com/ LBNLP/NERREand via Zenodo65, which contains the annotated datasets and test and train splits. Intermediate files for each step of the pipeline reported in this method are stored in this repository with corresponding documentation. Data for running Llama-2 models are available in the supplementary repository https://github.com/lbnlp/ nerre-llama66; LoRA weights for all Llama-2 models reported in this paper can be downloaded directly from Figshare (https://doi.org/10. -6084/m9.figshare.24501331.v1)67. Source data are provided with this paper. - -Code availability The code used for this study is available at https://github.com/LBNLP/ NERREand via Zenodo65 alongside the data. This code includes Jupyter notebooks for annotation as well as Python scripts for annotation, preprocessing, model training, and model evaluation on the train and test sets presented in this publication. The supplementary repository https://github.com/lbnlp/nerre-llama66 contains code and data for fine-tuning and inference with Llama-2 models trained in this study, including access to the complete sets of weights via a script. - -References 1. -Saal, J. E., Oliynyk, A. O. & Meredig, B. Machine learning in materials discovery: confirmed predictions and their underlying approaches. -Annu. Rev. Mater. Res. 50, 49–69 (2020). -2. -Choudhary, K. et al. Recent advances and applications of deep learning methods in materials science. npj Comput. Mater. 8, 59 (2022). -3. -Oliveira, O. N. & Oliveira, M. C. F. Materials discovery with machine learning and knowledge discovery. Front. Chem. 10, 930369 (2022). -4. -Weston, L. et al. Named entity recognition and normalization applied to large-scale information extraction from the materials science literature. J. Chem. Inform. Modeling 59, 3692–3702 (2019). -5. -Trewartha, A. et al. Quantifying the advantage of domain-specific pre-training on named entity recognition tasks in materials science. -Patterns 3, 100488 (2022). -6. -Isazawa, T. & Cole, J. M. Single model for organic and inorganic chemical named entity recognition in ChemDataExtractor. J. Chem. -Inform. Modeling 62, 1207–1213 (2022). -7. -Zhao, X., Greenberg, J., An, Y. & Hu, X. T. Fine-tuning BERT model for materials named entity recognition. In: 2021 IEEE International Conference on Big Data (Big Data) (IEEE, 2021). https://doi.org/10. -1109/bigdata52589.2021.9671697. - -8. -Sierepeklis, O. & Cole, J. M. A thermoelectric materials database auto-generated from the scientific literature using chemdataextractor. Sci. data 9, 648 (2022). -9. -Beard, E. J. & Cole, J. M. Perovskite- and dye-sensitized solar-cell device databases auto-generated using chemdataextractor. Sci. -Data 9, 329 (2022). -10. -Kumar, P., Kabra, S. & Cole, J. M. Auto-generating databases of yield strength and grain size using chemdataextractor. Sci. Data 9, 292 (2022). -11. -Huang, S. & Cole, J. M. BatteryBERT: A pretrained language model for battery database enhancement. J. Chem. Inform. Modeling 62, 6365–6377 (2022). -12. -Dong, Q. & Cole, J. M. Auto-generated database of semiconductor band gaps using chemdataextractor. Sci. Data 9, 193 (2022). -13. -Kononova, O. et al. Text-mined dataset of inorganic materials synthesis recipes. Sci. Data 6, 203 (2019). -14. -Huo, H. et al. Machine-learning rationalization and prediction of solid-state synthesis conditions. Chem. Mater. 34, 7323–7336 (2022). -15. -He, T. et al. Similarity of precursors in solid-state synthesis as textmined from scientific literature. Chem. of Mater. 32, 7861–7873 (2020). -16. -Wang, Z. et al. Dataset of solution-based inorganic materials synthesis procedures extracted from the scientific literature. Sci. -Data 9, 231 (2022). -17. -Huang, S. & Cole, J. M. A database of battery materials autogenerated using ChemDataExtractor. Sci. Data 7, 260 (2020). -18. -Beard, E. J., Sivaraman, G., Vázquez-Mayagoitia, Á., Vishwanath, V.& Cole, J. M. Comparative dataset of experimental and computational attributes of UV/vis absorption spectra. Sci. Data 6, 307 (2019). -19. -Zhao, J. & Cole, J. M. A database of refractive indices and dielectric constants auto-generated using ChemDataExtractor. Sci. Data 9, 192 (2022). -20. Bekoulis, G., Deleu, J., Demeester, T. & Develder, C. Joint entity recognition and relation extraction as a multi-head selection problem. Expert Syst. Appl. 114, 34–45 (2018). -21. -Han, X. et al. More data, more relations, more context and more openness: a review and outlook for relation extraction. In: Proceedings of the 1st Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics and the 10th International Joint Conference on Natural Language Processing, 745–758. -(Association for Computational Linguistics, 2020). https:// aclanthology.org/2020.aacl-main.75. -22. Yao, Y. et al. DocRED: A large-scale document-level relation extraction dataset. In: Proc. 57th Annual Meeting of the Association for Computational Linguistics, 764–777 (Association for Computational Linguistics, 2019). https://aclanthology.org/P19-1074. -23. Li, J. et al. Biocreative v cdr task corpus: a resource for chemical disease relation extraction. Database 2016 (2016). -24. Bravo, Á., Piñero, J., Queralt-Rosinach, N., Rautschka, M. & Furlong, L. I. Extraction of relations between genes and diseases from text and large-scale data analysis: Implications for translational research. BMC Bioinformatics 16, 1–17 (2015). -25. Yang, X. et al. Pcmsp: A dataset for scientific action graphs extraction from polycrystalline materials synthesis procedure text. -In: Findings of the Association for Computational Linguistics: EMNLP 2022, 6033–6046 (Association for Computational Linguistics, 2022). https://aclanthology.org/2022.findings-emnlp.446. -26. Brown, T. B. et al. Language models are few-shot learners. Preprint at https://browse.arxiv.org/abs/2005.14165 (2020). -27. -OpenAI. Gpt-4 technical report. Preprint at https://browse.arxiv. -org/abs/2303.08774 (2023). -28. Chowdhery, A. et al. Palm: Scaling language modeling with pathways. Journal of Machine Learning Research 24, 1–113 (2023). - ---- PAGE 13 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 13 - -29. Smith, S. et al. Using deepspeed and megatron to train megatronturing nlg 530b, a large-scale generative language model. Preprint at https://arxiv.org/abs/2201.11990 (2022). -30. Touvron, H. et al. Llama: Open and efficient foundation language models. Preprint at https://arxiv.org/abs/2302.13971 (2023). -31. -Touvron, H. et al. Llama 2: Open foundation and fine-tuned chat models. Preprint at https://arxiv.org/abs/2307.09288 (2023). -32. Zhang, S. et al. Opt: Open pre-trained transformer language models. Preprint at https://browse.arxiv.org/abs/2205.01068 (2022). -33. Hoffmann, J. et al. Training compute-optimal large language models. Preprint at http://arxiv.org/abs/2203.15556 (2022). -34. Wei, J. et al. Finetuned language models are zero-shot learners. In: -International Conference on Learning Representations (2022). -https://openreview.net/forum?id=gEZrGCozdqR. -35. BIG-bench collaboration. Beyond the imitation game: measuring and extrapolating the capabilities of language models. In preparation. https://github.com/google/BIG-bench/ (2021). -36. Dabre, R., Chu, C. & Kunchukuttan, A. A survey of multilingual neural machine translation. ACM Comput. Surv. 53, 1–38 (2020). -37. Petroni, F. et al. Language models as knowledge bases? In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), 2463–2473 (Association for Computational Linguistics, 2019). https://aclanthology. -org/D19-1250. -38. Han, J. M. et al. Unsupervised neural machine translation with generative language models only. https://openreview.net/forum? -id=SVwbKmEg7M (2022). -39. Zhang, H., Xu, J. & Wang, J. Pretraining-based natural language generation for text summarization. In: Proceedings of the 23rd Conference on Computational Natural Language Learning (CoNLL), 789–797 (Association for Computational Linguistics, 2019). https:// aclanthology.org/K19-1074. -40. Liu, Z. et al. Multi-stage prompting for knowledgeable dialogue generation. In: Findings of the Association for Computational Linguistics: ACL 2022, 1317–1337 (Association for Computational Linguistics, 2022). https://aclanthology.org/2022.findings-acl.104. -41. -Giorgi, J., Bader, G. & Wang, B. A sequence-to-sequence approach for document-level relation extraction. In: Proc. 21st Workshop on Biomedical Language Processing, 10–25 (Association for Computational Linguistics, 2022). https://aclanthology.org/2022.bionlp- 1.2. -42. Cabot, P.-L. H. & Navigli, R. REBEL: Relation extraction by end-toend language generation. In: Findings of the Association for Computational Linguistics: EMNLP 2021 (Association for Computational Linguistics, 2021). https://doi.org/10.18653/v1/2021.findingsemnlp.204. -43. Townsend, B., Ito-Fisher, E., Zhang, L. & May, M. Doc2dict: Information extraction as text generation. Preprint at http://arxiv.org/ abs/2105.07510 (2021). -44. Zheng, Z., Zhang, O., Borgs, C., Chayes, J. T. & Yaghi, O. M. ChatGPT chemistry assistant for text mining and the prediction of MOF synthesis. J. Am. Chem. Soc. 145, 18048–18062 (2023). -45. OpenAI et al. Introducing chatgpt https://openai.com/blog/ chatgpt (2022). -46. Castro Nascimento, C. M. & Pimentel, A. S. Do large language models understand chemistry? a conversation with chatgpt. J. -Chem. Inform. Modeling 63, 1649–1655 (2023). -47. White, A. D. et al. Assessment of chemistry knowledge in large language models that generate code. Digital Discov. 2, 368–376 (2023). -48. Xie, T. et al. Darwin series: Domain specific large language models for natural science. Preprint at https://arxiv.org/abs/2308. -13565 (2023). - -49. Jablonka, K. M., Schwaller, P., Ortega-Guerrero, A. & Smit, B. Is GPT all you need for low-data discovery in chemistry? https://doi.org/10. -26434/chemrxiv-2023-fw8n4-v2 (2023). -50. Jablonka, K. M. et al. 14 examples of how llms can transform materials science and chemistry: a reflection on a large language model hackathon. Digital Discov. 2, 1233–1250 (2023). -51. -Gu, Y. et al. Domain-specific language model pretraining for biomedical natural language processing. ACM Trans. Comput. -Healthcare 3, 1 (2021). -52. Mosqueira-Rey, E., Hernández-Pereira, E., Alonso-Ríos, D., Bobes- Bascarán, J. & Fernández-Leal, Á. Human-in-the-loop machine learning: a state of the art. Artif. Intel. Rev. 56, 1–50 (2022). -53. Maynez, J., Narayan, S., Bohnet, B. & McDonald, R. On faithfulness and factuality in abstractive summarization. In: Proc. 58th Annual Meeting of the Association for Computational Linguistics (Association for Computational Linguistics, 2020). https://doi.org/10. -18653/v1/2020.acl-main.173. -54. Ji, Z. et al. Survey of hallucination in natural language generation. -ACM Comput. Surv. 55, 12 (2023). -55. Black, S. et al. Gpt-neox-20b: An open-source autoregressive language model. Preprint at https://browse.arxiv.org/abs/2204. -06745 (2022). -56. Frantar, E. & Alistarh, D. Sparsegpt: Massive language models can be accurately pruned in one-shot. In: Proceedings of the 40th International Conference on Machine Learning, 10323–10337 (JLMR.org, 2023). https://proceedings.mlr.press/v202/frantar23a/ frantar23a.pdf. -57. Sun, M., Liu, Z., Bair, A. & Kolter, J. Z. A simple and effective pruning approach for large language models. In: Workshop on Efficient Systems for Foundation Models @ ICML2023. https://openreview. -net/forum?id=tz9JV2PRSv (2023). -58. Hu, E. J. et al. Lora: Low-rank adaptation of large language models. -Preprint at https://arxiv.org/abs/2106.09685 (2021). -59. Ma, X., Fang, G. & Wang, X. Llm-pruner: On the structural pruning of large language models. In: Thirty-seventh Conference on Neural Information Processing Systems (2023). https://openreview.net/ forum?id=J8Ajf9WfXP. -60. Bucior, B. J. et al. Identification schemes for metal–organic frameworks to enable rapid search and cheminformatics analysis. Cryst. -Growth Des. 19, 6682–6697 (2019). -61. -Li, X., Hill, M. R., Wang, H. & Zhang, H. Metal–organic frameworkbased ion-selective membranes. Adv. Mater. Technol. 6, 2000790 (2021). -62. Qian, Q. et al. Mof-based membranes for gas separations. Chem. -Rev. 120, 8161–8266 (2020). -63. Tshitoyan, V. et al. Unsupervised word embeddings capture latent knowledge from materials science literature. Nature 571, 95–98 (2019). -64. Kluyver, T. et al. Jupyter notebooks—a publishing format for reproducible computational workflows. In: Positioning and Power in Academic Publishing: Players, Agents and Agendas (eds. Loizides, F. -& Schmidt, B.) 87–90 (IOS Press, 2016). -65. Dagdelen, J. et al. Structured information extraction from scientific text with large language models https://doi.org/10.5281/zenodo. -10421174 (2023). -66. Dagdelen, J. et al. Llama 2 code for structured information extraction from scientific text with large language models. https://doi. -org/10.5281/zenodo.10421187 (2023). -67. Dagdelen, J. et al. Llama 2 weights for structured information extraction from scientific text with large language models. https://doi.org/10.6084/m9.figshare.24501331.v1 (2023). - ---- PAGE 14 --- - -Article https://doi.org/10.1038/s41467-024-45563-x - -Nature Communications| (2024) 15:1418 14 - -Acknowledgements This work was supported by Toyota Research Institute through the Accelerated Materials Design and Discovery program. A.S.R. -acknowledges support via a Miller Research Fellowship from the Miller Institute for Basic Research in Science, University of California, Berkeley. Funding for training and evaluating the Llama-2 model was provided by the U.S. Department of Energy, Office of Science, Office of Basic Energy Sciences, Materials Sciences and Engineering Division under Contract No. DE-AC02-05CH11231 (D2S2 program KCD2S2). This research used resources of the National Energy Research Scientific Computing Center (NERSC), a U.S. Department of Energy Office of Science User Facility located at Lawrence Berkeley National Laboratory, operated under Contract No. DE-AC02-05CH11231 using NERSC award BES- ERCAP0024004. We thank Anna Sackmann (Science Data and Engineering Librarian at UC Berkeley) for helping us to obtain Text and Data Mining agreements with the specified publishers and we also thank J. Montoya and A. Trewartha for helpful discussions. - -Author contributions J.D., A.D., N.W., and A.J. developed the information extraction method presented in this paper and J.D. and A.D. collected the abstract dataset used. J.D originated and performed supporting experiments to justify the sequence-to-sequence approach and use of GPT-3 for documentlevel information extraction from materials science text. A.D. expanded on JD’s initial experiments to further develop the approach. N.W. and J.D. -developed the sequence-to-JSON method. A.D. created the doping schemas, developed the sequence-to-sentence method for doping, and annotated the doping dataset. J.D. created the general materials information schema, annotated the general materials dataset, trained the general materials information extraction model, and manually scored the information extraction results for the General-JSON model validation set results. A.D. trained all Doping-* LLM-NERRE models and implemented the MatBERT + Proximity doping model. S.L. trained and collected data from the Llama-2 and seq2rel models. A.D. performed the learning curve experiments, and collected data on annotation times. -A.S.R. and J.D. co-created the MOF schema and annotated the MOF dataset while J.D. trained the MOF-JSON model. In addition to intellectual leadership and overall research directions, A.J. also contributed to - -task design and task scoring metrics. G.C., K.P., and A.J. supervised this work. All authors contributed to writing the manuscript. - -Competing interests The authors declare no competing interests. - -Additional information Supplementary information The online version contains supplementary material available at https://doi.org/10.1038/s41467-024-45563-x. - -Correspondence and requests for materials should be addressed to Anubhav Jain. - -Peer review information Nature Communications thanks the anonymous, reviewers for their contribution to the peer review of this work. A peer review file is available. - -Reprints and permissions information is available at http://www.nature.com/reprints - -Publisher’s note Springer Nature remains neutral with regard to jurisdictional claims in published maps and institutional affiliations. - -Open Access This article is licensed under a Creative Commons Attribution 4.0 International License, which permits use, sharing, adaptation, distribution and reproduction in any medium or format, as long as you give appropriate credit to the original author(s) and the source, provide a link to the Creative Commons license, and indicate if changes were made. The images or other third party material in this article are included in the article’s Creative Commons license, unless indicated otherwise in a credit line to the material. If material is not included in the article’s Creative Commons license and your intended use is not permitted by statutory regulation or exceeds the permitted use, you will need to obtain permission directly from the copyright holder. To view a copy of this license, visit http://creativecommons.org/ licenses/by/4.0/. - -© The Author(s) 2024 \ No newline at end of file diff --git a/experiments/literature_review/reviewer/README.md b/experiments/literature_review/reviewer/README.md new file mode 100644 index 0000000..e04977b --- /dev/null +++ b/experiments/literature_review/reviewer/README.md @@ -0,0 +1,109 @@ +# 论文筛选工具(2505+) + +本地浏览 + 分类 + 备注的网页工具,用于快速筛查论文。仅用 Python 标准库,无需安装额外依赖。 + +## 启动 + +```powershell +cd experiments\literature_review\reviewer +python server.py +``` + +默认在 `http://127.0.0.1:8765/` 启动并自动打开浏览器。 + +常用参数: + +```powershell +python server.py --port 9000 # 换端口 +python server.py --no-browser # 不自动打开浏览器 +``` + +## 范围 + +只纳入 `arxiv_id >= 2505`(2025 年 5 月及以后)的论文,共 **490 篇**(其中 488 篇有本地 PDF)。 + +## 操作 + +- 左侧列表选择论文,右侧内嵌 PDF 全文 + 标题/摘要/分类信息。 +- 顶部可切换筛选:全部 / 未筛 / 科研风 / 生动活泼 / 坏,支持搜索标题、作者、ID、分类。 +- 分类: + - **好 ▾** → 下拉选择「科研风」或「生动活泼」 + - **坏** + - **清除**(撤销当前分类) +- 每篇论文底部有**标记 / 备注**文本框,自动保存。 +- 键盘快捷键: + - `1` 科研风,`2` 生动活泼,`B` 坏,`U` 清除 + - `←` / `→` 上一篇 / 下一篇 + - 在「未筛」视图里标注后会自动跳到下一篇未筛论文。 + +## 数据保存 + +分类与备注实时写入: + +``` +experiments\literature_review\papers\review_2505.json +``` + +每条格式: + +```json +{ + "arxiv_id": { + "label": "good" | "bad", + "style": "research" | "lively", + "note": "自由文本标记", + "updated_at": "…" + } +} +``` + +刷新页面或重开后进度保留,可断点续标。清空重来:删除该 JSON 文件,或调用 `POST /api/reset`。 + +## 筛选完后:分类摆放 PDF + +```powershell +cd experiments\literature_review\reviewer +python sync_classified.py # 全量重建 + 生成 CSV +python sync_classified.py --dry-run # 先预览统计,不写盘 +``` + +分类来源(合并): +1. `review_2505.json` —— 网站上筛的 2505+ 论文(好/坏/未筛 + 风格 + 备注/优); +2. Downloads 两个文件夹 —— 2301~2504 手动筛出的「科研风 / 生动活泼」: + - `C:\Users\Lenovo\Downloads\1.科研风\1.科研风` + - `C:\Users\Lenovo\Downloads\2.生动型\2.生动型` + +把 PDF **复制**(不移动原文)到: + +``` +experiments\literature_review\papers\classified\ + 优\ # 好论文里备注含「优」的(跨风格汇总) + 好\ + 科研风\xxxx.pdf + 生动活泼\xxxx.pdf + 坏\xxxx.pdf + 未筛\xxxx.pdf + 分类清单.csv # UTF-8 BOM,Excel 可直接打开 +``` + +可反复重跑:会自动同步(新增/更新文件、清理已改分类的过期副本)。缺本地 PDF 的论文(如 `2507.23261`、`2510.18515`)只进清单、不复制文件。 + +## 推送到 GitHub + +```powershell +cd experiments\literature_review\reviewer +python push_to_github.py # 把 classified/ 推送到 Sen-illion/papers-dataset +python push_to_github.py --dry-run # 只预览,不推送 +``` + +- 免 token,走本机 SSH 密钥(`~/.ssh/id_ed25519`,需已授权访问 GitHub)。 +- 复用仓库已有的 PDF blob,实际只上传目录结构 + 分类清单 CSV,一个 commit 搞定。 +- 筛完新的、重新 `sync_classified.py` 后,再跑一次即可增量更新。 + +## 接口 + +- `GET /api/papers` — 论文列表(含 label / style / note)与统计 +- `POST /api/classify` — `{"arxiv_id": "...", "label": "good"|"bad"|null, "style": "research"|"lively"}` +- `POST /api/note` — `{"arxiv_id": "...", "note": "..."}` +- `POST /api/reset` — 清空全部分类与备注 +- `GET /pdf/.pdf` — 内嵌 PDF diff --git a/experiments/literature_review/reviewer/organize.py b/experiments/literature_review/reviewer/organize.py new file mode 100644 index 0000000..6662103 --- /dev/null +++ b/experiments/literature_review/reviewer/organize.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""按 review_2505.json 的分类结果,把 PDF 复制到分类文件夹。 + +结构(复制,不移动原文): + classified/ + 优/ ← 好论文里备注含“优”的(跨风格汇总,另外复制一份) + 好/ + 科研风/xxxx.pdf + 生动活泼/xxxx.pdf + 坏/xxxx.pdf + 未筛/xxxx.pdf + 分类清单.csv + +用法: + python organize.py # 实际复制 + 生成 CSV + python organize.py --dry-run # 只预览统计,不写盘 +""" +from __future__ import annotations + +import argparse +import csv +import json +import os +import shutil + +HERE = os.path.dirname(os.path.abspath(__file__)) +PAPERS_DIR = os.path.normpath(os.path.join(HERE, "..", "papers")) +PAPERS_JSON = os.path.join(PAPERS_DIR, "papers.json") +PDF_DIR = os.path.join(PAPERS_DIR, "pdf") +REVIEW_JSON = os.path.join(PAPERS_DIR, "review_2505.json") +OUT_DIR = os.path.join(PAPERS_DIR, "classified") +CSV_PATH = os.path.join(OUT_DIR, "分类清单.csv") + +ARXIV_FROM = "2505" +YOU_DIR = "优" + + +def category_of(label, style): + """返回 (顶级分类, 子文件夹或 None)。""" + if label == "good": + return ("好", "生动活泼" if style == "lively" else "科研风") + if label == "bad": + return ("坏", None) + return ("未筛", None) + + +def load_review(): + if not os.path.isfile(REVIEW_JSON): + return {} + with open(REVIEW_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + + +def copy_pdf(src, dst): + if not os.path.isfile(dst) or os.path.getsize(dst) != os.path.getsize(src): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + + +def main(): + parser = argparse.ArgumentParser(description="按筛选结果整理 PDF 到分类文件夹") + parser.add_argument("--dry-run", action="store_true", help="只预览,不实际复制/删除/写 CSV") + args = parser.parse_args() + + with open(PAPERS_JSON, "r", encoding="utf-8") as f: + all_papers = json.load(f) + review = load_review() + + papers = [p for p in all_papers if (p.get("arxiv_id") or "") >= ARXIV_FROM] + papers.sort(key=lambda p: p["arxiv_id"]) + + meta = {} # aid -> dict(top, sub, note, you, has_pdf) + rows = [] # CSV 行 + for p in papers: + aid = p["arxiv_id"] + entry = review.get(aid) + entry = entry if isinstance(entry, dict) else {} + label = entry.get("label") + style = entry.get("style") + note = (entry.get("note") or "").strip() + top, sub = category_of(label, style) + you = (label == "good") and ("优" in note) + meta[aid] = {"top": top, "sub": sub, "note": note, "you": you} + rows.append({ + "分类": top, + "风格": sub or "", + "优": "优" if you else "", + "arxiv_id": aid, + "标题": p.get("title", "") or "", + "备注": note, + "被引": p.get("cited_by_count", 0) or 0, + "arXiv": p.get("arxiv_url", "") or "", + }) + + # 统计 + stat = {} + you_count = 0 + for r in rows: + key = r["分类"] if not r["风格"] else f'{r["分类"]}/{r["风格"]}' + stat[key] = stat.get(key, 0) + 1 + if r["优"]: + you_count += 1 + stat[YOU_DIR] = you_count + + copied = 0 + you_copied = 0 + missing = [] + expected = set() # 期望存在的相对 .pdf 路径(用于清理过期文件) + + for p in papers: + aid = p["arxiv_id"] + src = os.path.join(PDF_DIR, aid + ".pdf") + m = meta[aid] + rel = os.path.join(m["top"], m["sub"]) if m["sub"] else m["top"] + rel_pdf = os.path.join(rel, aid + ".pdf") + expected.add(rel_pdf) + if m["you"]: + expected.add(os.path.join(YOU_DIR, aid + ".pdf")) + + if not os.path.isfile(src): + missing.append(aid) + continue + + if args.dry_run: + copied += 1 + if m["you"]: + you_copied += 1 + continue + + copy_pdf(src, os.path.join(OUT_DIR, rel, aid + ".pdf")) + if m["you"]: + copy_pdf(src, os.path.join(OUT_DIR, YOU_DIR, aid + ".pdf")) + you_copied += 1 + copied += 1 + + removed = 0 + if not args.dry_run and os.path.isdir(OUT_DIR): + for root, _dirs, files in os.walk(OUT_DIR): + for fn in files: + if not fn.lower().endswith(".pdf"): + continue + full = os.path.join(root, fn) + rel_pdf = os.path.relpath(full, OUT_DIR) + if rel_pdf not in expected: + os.remove(full) + removed += 1 + + if not args.dry_run: + os.makedirs(OUT_DIR, exist_ok=True) + with open(CSV_PATH, "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=["分类", "风格", "优", "arxiv_id", "标题", "备注", "被引", "arXiv"]) + writer.writeheader() + writer.writerows(rows) + + print("=== 分类整理结果" + ("(dry-run 预览)" if args.dry_run else "") + " ===") + for key in sorted(stat): + print(f" {key}: {stat[key]} 篇") + print(f" 合计: {len(papers)} 篇 · 已复制 PDF: {copied} · 其中“优”: {you_copied} · 缺 PDF: {len(missing)}" + (f" · 清理过期文件: {removed}" if removed else "")) + if missing: + print(" 缺本地 PDF 的 arxiv_id:", ", ".join(missing)) + if not args.dry_run: + print(f" 输出目录: {OUT_DIR}") + print(f" 清单文件: {CSV_PATH}") + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/reviewer/push_to_github.py b/experiments/literature_review/reviewer/push_to_github.py new file mode 100644 index 0000000..f50718f --- /dev/null +++ b/experiments/literature_review/reviewer/push_to_github.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- +"""把 classified/ 目录推送到 GitHub(免 token、免 REST API,走本机 SSH 密钥)。 + +用法(先确保已运行 sync_classified.py 生成最新 classified/): + python push_to_github.py # 推送 + python push_to_github.py --dry-run # 只构建对象、不推送 + +原理: + - classified/ 里的 PDF 与仓库 pdf/ 字节一致,直接复用已有 blob SHA(本地计算)。 + - 根目录里固定的文件(README/_meta/index/papers.csv/papers.json/_raw)从 + raw.githubusercontent.com 取内容算 SHA;pdf/ 树用本地 pdf/ 计算。 + - head 从 SSH git-receive-pack 的 ref 广告里读取。 + 因此整个流程不调用 api.github.com,不受未认证 API 限流影响。 +依赖:本机已配置好可访问 github.com 的 SSH 密钥(~/.ssh/id_ed25519)。 +""" +import hashlib +import os +import struct +import subprocess +import sys +import time +import urllib.request +import zlib + +REPO = "Sen-illion/papers-dataset" +RAW = f"https://raw.githubusercontent.com/{REPO}/main" +LOCAL_CLASSIFIED = os.path.join("experiments", "literature_review", "papers", "classified") +LOCAL_PDF_DIR = os.path.join("experiments", "literature_review", "papers", "pdf") +SSH = r"C:\Windows\System32\OpenSSH\ssh.exe" + +# 根目录里固定的条目(不含 classified,classified 每次重建) +ROOT_FILES = ["README.md", "_meta.json", "index.md", "papers.csv", "papers.json"] +RAW_SUBDIR_FILES = ["arxiv_metadata.json", "papers_enriched.json", "pdf_progress.json"] # _raw/ + +pack_objects = [] # (type_int, payload) + + +# ---------- git 对象 ---------- +def sha1(b): + return hashlib.sha1(b).hexdigest() + + +def blob_sha_from_bytes(data): + return sha1(b"blob %d\x00" % len(data) + data) + + +def obj_sha(typ, payload): + return sha1(typ.encode() + b" " + str(len(payload)).encode() + b"\x00" + payload) + + +def add_obj(typ, payload): + pack_objects.append((typ, payload)) + + +def make_blob(payload): + add_obj(3, payload) + return obj_sha("blob", payload) + + +def make_tree(entries): + payload = _tree_payload(entries) + add_obj(2, payload) + return obj_sha("tree", payload) + + +def make_commit(tree_sha, parent_sha, message): + now = int(time.time()) + ident = "Sen-illion " + payload = (f"tree {tree_sha}\nparent {parent_sha}\n" + f"author {ident} {now} +0800\ncommitter {ident} {now} +0800\n\n{message}\n").encode("utf-8") + add_obj(1, payload) + return obj_sha("commit", payload) + + +def _tree_payload(entries): + norm = [] + for mode, name, sha in entries: + m = mode.lstrip("0") or "0" + norm.append((m, name, sha)) + + def key(e): + m, name, _ = e + nb = name.encode("utf-8") + return nb + b"/" if m == "40000" else nb + + norm.sort(key=key) + payload = b"" + for m, name, sha in norm: + payload += m.encode() + b" " + name.encode("utf-8") + b"\x00" + bytes.fromhex(sha) + return payload + + +# ---------- 目录 -> git 树 ---------- +def build_tree_for_dir(dirpath, pdf_map): + entries = [] + for name in os.listdir(dirpath): + full = os.path.join(dirpath, name) + if os.path.isdir(full): + child = build_tree_for_dir(full, pdf_map) + entries.append(("40000", name, child)) + else: + if name.lower().endswith(".pdf"): + sha = pdf_map[name] + if sha is None: + raise SystemExit(f"pdf_map 缺少: {name}") + else: + data = open(full, "rb").read() + sha = make_blob(data) + entries.append(("100644", name, sha)) + return make_tree(entries) + + +# ---------- pack ---------- +def encode_header(typ, size): + out = bytearray() + byte = (typ << 4) | (size & 0x0F) + size >>= 4 + if size: + byte |= 0x80 + out.append(byte) + while size: + byte = size & 0x7F + size >>= 7 + if size: + byte |= 0x80 + out.append(byte) + return bytes(out) + + +def build_pack(objects): + body = b"PACK" + struct.pack(">II", 2, len(objects)) + for typ, payload in objects: + body += encode_header(typ, len(payload)) + zlib.compress(payload) + body += hashlib.sha1(body).digest() + return body + + +# ---------- 远端状态(raw CDN + 本地,不用 api.github.com) ---------- +def raw_fetch(path): + url = f"{RAW}/{path}" + req = urllib.request.Request(url, headers={"User-Agent": "dsh"}) + with urllib.request.urlopen(req, timeout=60) as r: + return r.read() + + +def local_pdf_map(): + m = {} + for fn in os.listdir(LOCAL_PDF_DIR): + if fn.lower().endswith(".pdf"): + with open(os.path.join(LOCAL_PDF_DIR, fn), "rb") as f: + m[fn] = blob_sha_from_bytes(f.read()) + return m + + +def root_nonclassified_entries(pdf_map): + """返回根树里除 classified 之外的条目(mode, name, sha)。""" + entries = [] + for name in ROOT_FILES: + data = raw_fetch(name) + entries.append(("100644", name, blob_sha_from_bytes(data))) + + # _raw/ 子树(树对象已在远端,本地算出相同 sha,不加入 pack) + raw_entries = [] + for name in RAW_SUBDIR_FILES: + data = raw_fetch(f"_raw/{name}") + raw_entries.append(("100644", name, blob_sha_from_bytes(data))) + raw_tree_sha = obj_sha("tree", _tree_payload(raw_entries)) + entries.append(("40000", "_raw", raw_tree_sha)) + + # pdf/ 子树(本地计算,与远端一致) + pdf_entries = [("100644", fn, sha) for fn, sha in pdf_map.items()] + pdf_tree_sha = obj_sha("tree", _tree_payload(pdf_entries)) + entries.append(("40000", "pdf", pdf_tree_sha)) + + return entries + + +# ---------- SSH ---------- +def read_advert(p): + """读取 receive-pack 的 v0 ref 广告,返回 {refname: sha}。""" + refs = {} + while True: + b = p.stdout.read(4) + if not b: + break + ln = int(b, 16) + if ln == 0: + break + payload = p.stdout.read(ln - 4) + if b"\x00" in payload: + payload = payload.split(b"\x00", 1)[0] + parts = payload.split(b" ") + if len(parts) >= 2: + refs[parts[1].decode()] = parts[0].decode() + return refs + + +def probe_head(): + cmd = [SSH, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", + "-T", "git@github.com", "git-receive-pack", f"{REPO}.git"] + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + refs = read_advert(p) + p.stdin.close() + p.terminate() + head = refs.get("refs/heads/main") + if not head: + raise SystemExit("未读到 main 的 head") + return head + + +def push(old_head, new_commit, pack_bytes): + cmd = [SSH, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", + "-T", "git@github.com", "git-receive-pack", f"{REPO}.git"] + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + refs = read_advert(p) + if refs.get("refs/heads/main") != old_head: + print(" [警告] 远端 head 已变化,以广告为准") + old_head = refs.get("refs/heads/main") + + cmdline = f"{old_head} {new_commit} refs/heads/main\x00report-status".encode("utf-8") + p.stdin.write(b"%04x" % (len(cmdline) + 4) + cmdline) + p.stdin.write(b"0000") + p.stdin.write(pack_bytes) + p.stdin.close() + + out = p.stdout.read() + err = p.stderr.read() + p.wait() + return out, err, p.returncode + + +def parse_report(out): + lines = [] + i = 0 + while i + 4 <= len(out): + ln = int(out[i:i + 4], 16) + if ln == 0: + break + if ln > 4: + lines.append(out[i + 4:i + ln].decode("utf-8", "replace").rstrip("\n")) + i += ln + return lines + + +def main(): + print("计算本地 pdf blob map ...") + pdf_map = local_pdf_map() + print(f" pdf_map: {len(pdf_map)} 个") + + print("读取远端根目录固定条目 ...") + root_entries = root_nonclassified_entries(pdf_map) + + print("构建 classified 树 ...") + classified_sha = build_tree_for_dir(LOCAL_CLASSIFIED, pdf_map) + print(" classified tree:", classified_sha) + + root_entries.append(("40000", "classified", classified_sha)) + new_root_sha = make_tree(root_entries) + print(" new root tree:", new_root_sha) + + head = probe_head() + print(" remote head:", head) + + new_commit = make_commit(new_root_sha, head, "Add classified papers (优/好/坏/未筛)") + print(" new commit:", new_commit) + + pack_bytes = build_pack(pack_objects) + print(f"pack objects: {len(pack_objects)} · pack size: {len(pack_bytes)} bytes") + if "--dry-run" in sys.argv: + from collections import Counter + print(" [dry-run] 对象分布:", dict(Counter({1: 'commit', 2: 'tree', 3: 'blob'}[t] for t, _ in pack_objects))) + return + + out, err, rc = push(head, new_commit, pack_bytes) + print("ssh rc:", rc) + if err.strip(): + print("stderr:", err.decode("utf-8", "replace")[:500]) + print("report:") + for line in parse_report(out): + print(" ", line) + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/reviewer/server.py b/experiments/literature_review/reviewer/server.py new file mode 100644 index 0000000..d9fe729 --- /dev/null +++ b/experiments/literature_review/reviewer/server.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +"""论文筛选工具 —— 本地 Web 服务(仅用 Python 标准库,无第三方依赖)。 + +用法: + python server.py [--port 8765] [--no-browser] + +行为: + - 从 experiments/literature_review/papers/papers.json 加载元数据; + - 只纳入 arxiv_id >= "2505"(2025 年 5 月及以后)的论文; + - 通过 http://127.0.0.1: 提供浏览界面; + - 好 / 坏分类结果写入 papers/review_2505.json,可断点续标。 +""" +from __future__ import annotations + +import argparse +import json +import os +import threading +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +HERE = os.path.dirname(os.path.abspath(__file__)) +PAPERS_DIR = os.path.normpath(os.path.join(HERE, "..", "papers")) +PAPERS_JSON = os.path.join(PAPERS_DIR, "papers.json") +PDF_DIR = os.path.join(PAPERS_DIR, "pdf") +STATIC_DIR = os.path.join(HERE, "static") +REVIEW_JSON = os.path.join(PAPERS_DIR, "review_2505.json") + +ARXIV_FROM = "2505" # 只纳入 arxiv_id >= 该值 +LABELS = ("good", "bad") # 可选分类 +STYLES = ("research", "lively") # “好”的两种风格:科研风 / 生动活泼 +DEFAULT_PORT = 8765 + +_lock = threading.Lock() +_papers: list[dict] = [] +_review: dict = {} + + +def load_papers() -> list[dict]: + """加载并筛选 papers.json,按 arxiv_id 升序返回。""" + with open(PAPERS_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + + out: list[dict] = [] + for p in data: + aid = (p.get("arxiv_id") or "").strip() + if aid < ARXIV_FROM: + continue + pdf_path = (p.get("pdf_path") or "").strip() + has_pdf = bool(pdf_path) and os.path.isfile(os.path.join(PAPERS_DIR, pdf_path)) + out.append({ + "arxiv_id": aid, + "title": p.get("title", "") or "", + "authors": p.get("authors", []) or [], + "categories": p.get("categories", []) or [], + "primary_category": p.get("primary_category", "") or "", + "year": p.get("year"), + "published": p.get("published", "") or "", + "cited_by_count": p.get("cited_by_count", 0) or 0, + "arxiv_url": p.get("arxiv_url", "") or "", + "abstract": p.get("abstract", "") or "", + "has_pdf": has_pdf, + }) + out.sort(key=lambda x: x["arxiv_id"]) + return out + + +def load_review() -> dict: + if not os.path.isfile(REVIEW_JSON): + return {} + try: + with open(REVIEW_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except (json.JSONDecodeError, OSError): + return {} + + +def save_review(review: dict) -> None: + tmp = REVIEW_JSON + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(review, f, ensure_ascii=False, indent=2) + os.replace(tmp, REVIEW_JSON) + + +def _entry(review: dict, aid: str) -> dict: + """返回某篇论文的标注条目(无则返回空 dict)。""" + e = review.get(aid) + return e if isinstance(e, dict) else {} + + +def counts(review: dict) -> dict: + c = { + "total": len(_papers), + "good_research": 0, + "good_lively": 0, + "bad": 0, + "unreviewed": 0, + } + for p in _papers: + entry = _entry(review, p["arxiv_id"]) + label = entry.get("label") + if label == "good": + if entry.get("style") == "lively": + c["good_lively"] += 1 + else: + c["good_research"] += 1 + elif label == "bad": + c["bad"] += 1 + else: + c["unreviewed"] += 1 + return c + + +def papers_with_label(review: dict) -> list[dict]: + result = [] + for p in _papers: + entry = _entry(review, p["arxiv_id"]) + label = entry.get("label") if entry.get("label") in LABELS else None + if label == "good": + style = entry.get("style") if entry.get("style") in STYLES else "research" + else: + style = None + note = (entry.get("note") or "").strip() + result.append({**p, "label": label, "style": style, "note": note}) + return result + + +class Handler(BaseHTTPRequestHandler): + server_version = "PaperReviewer/1.0" + + # ---- helpers --------------------------------------------------------- + def _send(self, code: int, body: bytes, ctype: str = "application/octet-stream", + extra: dict | None = None) -> None: + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + if extra: + for k, v in extra.items(): + self.send_header(k, v) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _send_json(self, code: int, obj) -> None: + body = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self._send(code, body, "application/json; charset=utf-8") + + def _read_body(self) -> dict: + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + return {} + raw = self.rfile.read(length) + try: + return json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return {} + + # ---- routing --------------------------------------------------------- + def do_HEAD(self) -> None: # noqa: N802 + # 复用 GET 路由(_send/_serve_pdf 内部已对 HEAD 跳过响应体) + self.do_GET() + + def do_GET(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + path = parsed.path + + if path in ("/", "/index.html"): + return self._serve_static("index.html") + + if path.startswith("/static/"): + name = os.path.basename(path[len("/static/"):]) + return self._serve_static(name) + + if path == "/api/papers": + with _lock: + return self._send_json(200, { + "papers": papers_with_label(_review), + "counts": counts(_review), + }) + + if path == "/favicon.ico": + return self._send(204, b"") + + if path.startswith("/pdf/"): + return self._serve_pdf(path[len("/pdf/"):]) + + return self._send_json(404, {"error": "not found"}) + + def do_POST(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + path = parsed.path + + if path == "/api/classify": + body = self._read_body() + aid = (body.get("arxiv_id") or "").strip() + label = body.get("label") + style = body.get("style") + + if aid not in {p["arxiv_id"] for p in _papers}: + return self._send_json(404, {"error": "unknown arxiv_id", "ok": False}) + + if label not in LABELS and label is not None: + return self._send_json(400, {"error": "label must be 'good', 'bad' or null", "ok": False}) + + with _lock: + if label is None: + _review.pop(aid, None) + else: + entry = _entry(_review, aid) + entry["label"] = label + if label == "good": + entry["style"] = style if style in STYLES else "research" + else: + entry.pop("style", None) + entry["updated_at"] = _now_iso() + _review[aid] = entry + save_review(_review) + return self._send_json(200, {"ok": True, "counts": counts(_review)}) + + if path == "/api/note": + body = self._read_body() + aid = (body.get("arxiv_id") or "").strip() + note = body.get("note") + + if aid not in {p["arxiv_id"] for p in _papers}: + return self._send_json(404, {"error": "unknown arxiv_id", "ok": False}) + + note = note.strip() if isinstance(note, str) else "" + with _lock: + entry = _entry(_review, aid) + if note: + entry["note"] = note + entry["updated_at"] = _now_iso() + _review[aid] = entry + else: + entry.pop("note", None) + if entry: + _review[aid] = entry + else: + _review.pop(aid, None) + save_review(_review) + return self._send_json(200, {"ok": True}) + + if path == "/api/reset": + with _lock: + _review.clear() + save_review(_review) + return self._send_json(200, {"ok": True, "counts": counts(_review)}) + + return self._send_json(404, {"error": "not found"}) + + # ---- static / pdf ---------------------------------------------------- + def _serve_static(self, name: str) -> None: + safe = os.path.basename(name) + full = os.path.join(STATIC_DIR, safe) + if not os.path.isfile(full): + return self._send_json(404, {"error": "not found"}) + ctype = MIME.get(os.path.splitext(safe)[1].lower(), "application/octet-stream") + with open(full, "rb") as f: + return self._send(200, f.read(), ctype) + + def _serve_pdf(self, suffix: str) -> None: + aid = os.path.basename(suffix) + # 接受 "" 或 ".pdf" + if aid.lower().endswith(".pdf"): + aid = aid[:-4] + if aid not in {p["arxiv_id"] for p in _papers}: + return self._send_json(404, {"error": "unknown arxiv_id"}) + + pdf_path = os.path.join(PDF_DIR, aid + ".pdf") + if not os.path.isfile(pdf_path): + return self._send_json(404, {"error": "pdf missing"}) + + size = os.path.getsize(pdf_path) + self.send_response(200) + self.send_header("Content-Type", "application/pdf") + self.send_header("Content-Length", str(size)) + self.send_header("Content-Disposition", f'inline; filename="{aid}.pdf"') + self.end_headers() + if self.command != "HEAD": + with open(pdf_path, "rb") as f: + while True: + chunk = f.read(65536) + if not chunk: + break + self.wfile.write(chunk) + + def log_message(self, fmt, *args): # noqa: A003 + # 简洁日志,避免刷屏 + print("[%s] %s" % (self.address_string(), fmt % args)) + + +def _now_iso() -> str: + import datetime + return datetime.datetime.now().astimezone().isoformat(timespec="seconds") + + +MIME = { + ".html": "text/html; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".png": "image/png", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", +} + + +def main() -> None: + global _papers, _review + parser = argparse.ArgumentParser(description="本地论文筛选服务") + parser.add_argument("--port", type=int, default=DEFAULT_PORT) + parser.add_argument("--no-browser", action="store_true", help="不自动打开浏览器") + args = parser.parse_args() + + _papers = load_papers() + _review = load_review() + + host = "127.0.0.1" + httpd = ThreadingHTTPServer((host, args.port), Handler) + url = f"http://{host}:{args.port}/" + + print(f"论文筛选服务已启动:{url}") + print(f" 共 {len(_papers)} 篇(arxiv_id >= {ARXIV_FROM})," + f"其中 {sum(1 for p in _papers if p['has_pdf'])} 篇有 PDF") + c = counts(_review) + print(f" 已标注:科研风 {c['good_research']} · 生动活泼 {c['good_lively']} · 坏 {c['bad']} · 未筛 {c['unreviewed']}") + print(f" 分类结果保存到:{REVIEW_JSON}") + print(" 按 Ctrl+C 退出。") + + if not args.no_browser: + try: + webbrowser.open(url) + except Exception as exc: # noqa: BLE001 + print(f" (自动打开浏览器失败: {exc})") + + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\n已退出。") + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/reviewer/static/app.js b/experiments/literature_review/reviewer/static/app.js new file mode 100644 index 0000000..00995a8 --- /dev/null +++ b/experiments/literature_review/reviewer/static/app.js @@ -0,0 +1,355 @@ +/* 论文筛选前端逻辑 */ +(function () { + "use strict"; + + const $ = (s) => document.querySelector(s); + + const state = { + papers: [], // 每篇: {arxiv_id, title, authors, categories, ..., label, style, note} + counts: { total: 0, good_research: 0, good_lively: 0, bad: 0, unreviewed: 0 }, + filter: "all", // all | unreviewed | research | lively | bad + query: "", + currentId: null, + }; + + const els = { + filters: $("#filters"), + search: $("#search"), + list: $("#list"), + listCount: $("#list-count"), + detail: $("#detail"), + statReviewed: $("#stat-reviewed"), + statTotal: $("#stat-total"), + statResearch: $("#stat-research"), + statLively: $("#stat-lively"), + statBad: $("#stat-bad"), + }; + + /* ---------- 数据 ---------- */ + async function load() { + const res = await fetch("/api/papers"); + const data = await res.json(); + state.papers = data.papers; + state.counts = data.counts; + renderStats(); + renderList(); + const first = filtered()[0]; + if (first) select(first.arxiv_id); + } + + function filtered() { + const q = state.query.trim().toLowerCase(); + return state.papers.filter((p) => { + if (state.filter === "unreviewed" && p.label != null) return false; + if (state.filter === "research" && !(p.label === "good" && p.style === "research")) return false; + if (state.filter === "lively" && !(p.label === "good" && p.style === "lively")) return false; + if (state.filter === "bad" && p.label !== "bad") return false; + if (q) { + const hay = [ + p.title, p.arxiv_id, + (p.authors || []).join(" "), + (p.categories || []).join(" "), + ].join(" ").toLowerCase(); + if (!hay.includes(q)) return false; + } + return true; + }); + } + + function byId(id) { + return state.papers.find((p) => p.arxiv_id === id) || null; + } + + /* ---------- 渲染 ---------- */ + function renderStats() { + const c = state.counts; + els.statTotal.textContent = c.total; + els.statReviewed.textContent = c.good_research + c.good_lively + c.bad; + els.statResearch.textContent = "科研风 " + c.good_research; + els.statLively.textContent = "生动 " + c.good_lively; + els.statBad.textContent = "坏 " + c.bad; + } + + function labelText(p) { + if (p.label === "good") return p.style === "lively" ? "生动活泼" : "科研风"; + if (p.label === "bad") return "坏"; + return null; + } + + function flagClass(p) { + if (p.label === "good") return p.style === "lively" ? "flag-lively" : "flag-research"; + if (p.label === "bad") return "flag-bad"; + return ""; + } + + function flagHtml(p) { + const t = labelText(p); + let html = ""; + if (t) html += '' + t + ""; + if (p.note) html += '📝'; + return html ? '
' + html + "
" : ""; + } + + function itemHtml(p) { + const cats = (p.categories || []).slice(0, 3).join(" · "); + const cls = p.arxiv_id === state.currentId ? "item is-current" : "item"; + return ( + '
' + + '
' + p.arxiv_id + "
" + + '
' + + '
' + esc(p.title) + "
" + + '
' + esc(cats || p.primary_category || "") + + (p.year ? " · " + p.year : "") + + (p.cited_by_count ? " · 被引 " + p.cited_by_count : "") + + "
" + + "
" + flagHtml(p) + + "
" + ); + } + + function renderList() { + const items = filtered(); + els.listCount.textContent = "(" + items.length + ")"; + + if (!items.length) { + els.list.innerHTML = '
没有符合条件的论文
'; + return; + } + els.list.innerHTML = items.map(itemHtml).join(""); + } + + function renderDetail(p) { + if (!p) { + els.detail.innerHTML = '
没有可显示的论文
'; + return; + } + const authors = (p.authors || []).join(", "); + const cats = (p.categories || []).map((c) => '' + esc(c) + "").join(" "); + const meta = []; + meta.push('' + esc(p.primary_category || "") + ""); + if (p.year) meta.push('' + p.year + ""); + meta.push('被引 ' + (p.cited_by_count || 0) + ""); + + const goodText = p.label === "good" + ? (p.style === "lively" ? "好 · 生动活泼" : "好 · 科研风") + : "好"; + const goodOn = p.label === "good" ? " is-on" : ""; + const badOn = p.label === "bad" ? " is-on" : ""; + const researchOn = (p.label === "good" && p.style === "research") ? " is-current" : ""; + const livelyOn = (p.label === "good" && p.style === "lively") ? " is-current" : ""; + + let pdfHtml; + if (p.has_pdf) { + pdfHtml = ''; + } else { + pdfHtml = '
此篇缺少本地 PDF(可点 arXiv 链接在线查看)
'; + } + + els.detail.innerHTML = + '
' + + '
' + + '' + + '
' + + '' + + '' + + "
" + + "
" + + '' + + '' + + (p.arxiv_url ? 'arXiv ↗' : "") + + "
" + + '

' + esc(p.title) + "

" + + '
' + esc(authors) + "
" + + '
' + meta.join("") + '' + esc(p.arxiv_id) + "
" + + '
' + pdfHtml + "
" + + (p.abstract + ? '
摘要' + esc(p.abstract) + "
" + : "") + + '
' + + '' + + '" + + '' + + "
"; + + bindNoteInput(p.arxiv_id); + } + + function select(id) { + state.currentId = id; + renderList(); + renderDetail(byId(id)); + } + + function move(delta) { + const items = filtered(); + if (!items.length) return; + let idx = items.findIndex((p) => p.arxiv_id === state.currentId); + if (idx === -1) idx = 0; + idx = Math.max(0, Math.min(items.length - 1, idx + delta)); + select(items[idx].arxiv_id); + const el = els.list.querySelector('[data-id="' + items[idx].arxiv_id + '"]'); + if (el && el.scrollIntoView) el.scrollIntoView({ block: "nearest" }); + } + + /* ---------- 分类 ---------- */ + async function classify(label, style) { + const p = byId(state.currentId); + if (!p) return; + + const oldItems = filtered(); + const oldIdx = oldItems.findIndex((x) => x.arxiv_id === state.currentId); + + const res = await fetch("/api/classify", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ arxiv_id: p.arxiv_id, label: label, style: style }), + }); + const data = await res.json(); + if (!data.ok) return; + + p.label = label; + p.style = label === "good" ? (style || "research") : null; + state.counts = data.counts; + + renderStats(); + renderList(); + + // 自动跳到下一篇(在"未筛"等会移除当前项的视图里,oldIdx 即下一条) + const items = filtered(); + let idx = oldIdx === -1 ? 0 : oldIdx; + if (state.filter === "all") idx += 1; + idx = Math.max(0, Math.min(items.length - 1, idx)); + const next = items.length ? items[idx] : null; + + if (next) { + state.currentId = next.arxiv_id; + renderList(); + renderDetail(byId(state.currentId)); + } else { + state.currentId = null; + renderList(); + renderDetail(null); + } + } + + /* ---------- 备注 ---------- */ + let noteTimer = null; + + function bindNoteInput(id) { + const ta = els.detail.querySelector(".note-input"); + if (!ta) return; + ta.addEventListener("input", () => { + clearTimeout(noteTimer); + noteTimer = setTimeout(() => saveNote(id, ta.value), 600); + }); + ta.addEventListener("blur", () => { + clearTimeout(noteTimer); + saveNote(id, ta.value); + }); + } + + async function saveNote(id, value) { + const res = await fetch("/api/note", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ arxiv_id: id, note: value }), + }); + const data = await res.json(); + if (!data.ok) return; + + const p = byId(id); + if (p) p.note = value.trim(); + + // 就地更新列表里的备注标记,不整列表重绘(避免滚动跳动) + const item = els.list.querySelector('[data-id="' + id + '"]'); + if (item && p) item.outerHTML = itemHtml(p); + + if (state.currentId === id) { + const st = els.detail.querySelector("[data-note-status]"); + if (st) { + st.textContent = "已保存"; + st.classList.add("saved"); + setTimeout(() => { if (st.isConnected) { st.textContent = ""; st.classList.remove("saved"); } }, 1500); + } + } + } + + /* ---------- 事件 ---------- */ + function closeMenus() { + els.detail.querySelectorAll(".good-wrap.open").forEach((w) => w.classList.remove("open")); + } + + els.filters.addEventListener("click", (e) => { + const btn = e.target.closest("[data-filter]"); + if (!btn) return; + state.filter = btn.dataset.filter; + els.filters.querySelectorAll(".chip").forEach((c) => c.classList.remove("is-active")); + btn.classList.add("is-active"); + renderList(); + const first = filtered()[0]; + if (first) select(first.arxiv_id); + else renderDetail(null); + }); + + els.search.addEventListener("input", (e) => { + state.query = e.target.value; + renderList(); + const first = filtered()[0]; + if (first) select(first.arxiv_id); + else renderDetail(null); + }); + + els.list.addEventListener("click", (e) => { + const item = e.target.closest("[data-id]"); + if (item) select(item.dataset.id); + }); + + els.detail.addEventListener("click", (e) => { + const btn = e.target.closest("[data-act]"); + if (btn && btn.dataset.act === "good") { + closeMenus(); + btn.closest(".good-wrap").classList.toggle("open"); + return; + } + const mi = e.target.closest("[data-style]"); + if (mi) { + closeMenus(); + classify("good", mi.dataset.style); + return; + } + if (btn) { + if (btn.dataset.act === "bad") classify("bad"); + else if (btn.dataset.act === "clear") classify(null); + } + }); + + document.addEventListener("click", (e) => { + if (!e.target.closest(".good-wrap")) closeMenus(); + }); + + document.addEventListener("keydown", (e) => { + const tag = (e.target.tagName || "").toLowerCase(); + if (tag === "input" || tag === "textarea" || tag === "select") return; + if (e.metaKey || e.ctrlKey || e.altKey) return; + + switch (e.key) { + case "1": classify("good", "research"); e.preventDefault(); break; + case "2": classify("good", "lively"); e.preventDefault(); break; + case "b": case "B": classify("bad"); e.preventDefault(); break; + case "u": case "U": classify(null); e.preventDefault(); break; + case "ArrowLeft": move(-1); e.preventDefault(); break; + case "ArrowRight": move(1); e.preventDefault(); break; + } + }); + + /* ---------- 工具 ---------- */ + function esc(s) { + return String(s == null ? "" : s) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); + } + + load(); +})(); diff --git a/experiments/literature_review/reviewer/static/index.html b/experiments/literature_review/reviewer/static/index.html new file mode 100644 index 0000000..b4df511 --- /dev/null +++ b/experiments/literature_review/reviewer/static/index.html @@ -0,0 +1,52 @@ + + + + + + 论文筛选 · 2505+ + + + +
+
+ 论文筛选 + arxiv_id ≥ 2505 +
+ +
+ + + + + +
+ + + +
+ 0/0 已筛 + 科研风 0 + 生动 0 + 坏 0 +
+
+ +
+ + +
+
← 从左侧选择一篇论文开始筛选
+
+
+ +
+ 1 科研风   2 生动活泼   B 坏   U 清除   + ←→ 上一篇/下一篇 +
+ + + + diff --git a/experiments/literature_review/reviewer/static/style.css b/experiments/literature_review/reviewer/static/style.css new file mode 100644 index 0000000..93c5317 --- /dev/null +++ b/experiments/literature_review/reviewer/static/style.css @@ -0,0 +1,266 @@ +:root { + --bg: #f6f7f9; + --panel: #ffffff; + --border: #e4e7ec; + --text: #1f2733; + --muted: #7a8598; + --accent: #2f6fed; + --research: #2563eb; + --research-bg: #e8eefc; + --lively: #d97706; + --lively-bg: #fdf1e3; + --bad: #d64545; + --bad-bg: #fdeeee; + --radius: 10px; +} + +* { box-sizing: border-box; } + +html, body { + margin: 0; + height: 100%; + font-family: -apple-system, "Segoe UI", "Microsoft YaHei", Roboto, sans-serif; + color: var(--text); + background: var(--bg); + font-size: 14px; +} + +/* ---- top bar ---- */ +.topbar { + display: flex; + align-items: center; + gap: 14px; + padding: 10px 16px; + background: var(--panel); + border-bottom: 1px solid var(--border); + flex-wrap: wrap; +} +.brand-title { font-size: 17px; font-weight: 700; } +.brand-sub { font-size: 12px; color: var(--muted); margin-left: 6px; } + +.filters { display: flex; gap: 6px; } +.chip { + border: 1px solid var(--border); + background: #fff; + color: var(--text); + border-radius: 999px; + padding: 5px 13px; + cursor: pointer; + font-size: 13px; +} +.chip:hover { border-color: var(--accent); } +.chip.is-active { background: var(--accent); border-color: var(--accent); color: #fff; } +.chip-research.is-active { background: var(--research); border-color: var(--research); } +.chip-lively.is-active { background: var(--lively); border-color: var(--lively); } +.chip-bad.is-active { background: var(--bad); border-color: var(--bad); } + +.search { + flex: 1 1 200px; + min-width: 160px; + padding: 7px 12px; + border: 1px solid var(--border); + border-radius: 999px; + outline: none; +} +.search:focus { border-color: var(--accent); } + +.stats { display: flex; gap: 12px; margin-left: auto; font-size: 13px; } +.stat { color: var(--muted); } +.stat b { color: var(--text); } +.stat-research { color: var(--research); font-weight: 600; } +.stat-lively { color: var(--lively); font-weight: 600; } +.stat-bad { color: var(--bad); font-weight: 600; } + +/* ---- layout ---- */ +.layout { + display: grid; + grid-template-columns: 380px 1fr; + gap: 0; + height: calc(100vh - 57px); +} +@media (max-width: 900px) { + .layout { grid-template-columns: 1fr; grid-template-rows: 45% 1fr; } +} + +.list-pane { + border-right: 1px solid var(--border); + background: var(--panel); + display: flex; + flex-direction: column; + min-height: 0; +} +.list-head { + padding: 10px 14px; + font-weight: 600; + border-bottom: 1px solid var(--border); + font-size: 13px; +} +.muted { color: var(--muted); font-weight: 400; } + +.list { overflow-y: auto; flex: 1; } +.item { + padding: 10px 14px; + border-bottom: 1px solid var(--border); + cursor: pointer; + display: flex; + gap: 10px; + align-items: flex-start; +} +.item:hover { background: #f0f4fb; } +.item.is-current { background: #e8f0fe; } +.item .id { font-family: ui-monospace, Consolas, monospace; font-size: 12px; color: var(--muted); white-space: nowrap; } +.item .body { flex: 1; min-width: 0; } +.item .title { font-weight: 600; line-height: 1.35; margin: 2px 0 3px; } +.item .meta { font-size: 12px; color: var(--muted); } +.flags { flex: 0 0 auto; display: flex; flex-direction: column; gap: 3px; align-items: flex-end; margin-left: auto; } +.flag { font-size: 11px; padding: 2px 8px; border-radius: 999px; white-space: nowrap; } +.flag-research { background: var(--research-bg); color: var(--research); } +.flag-lively { background: var(--lively-bg); color: var(--lively); } +.flag-bad { background: var(--bad-bg); color: var(--bad); } +.note-dot { font-size: 12px; } + +/* ---- detail ---- */ +.detail-pane { min-width: 0; overflow-y: auto; padding: 16px 20px 120px; } +.empty { color: var(--muted); text-align: center; margin-top: 80px; } + +.d-head { display: flex; gap: 16px; align-items: flex-start; margin-bottom: 14px; } +.d-title { font-size: 20px; font-weight: 700; line-height: 1.3; margin: 0; } +.d-authors { color: var(--muted); margin: 8px 0 4px; } +.d-meta { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px; } +.tag { + font-size: 12px; + background: #eef1f6; + color: #445; + padding: 2px 9px; + border-radius: 6px; +} +.tag-cat { background: #e8f0fe; color: #2f6fed; } +.tag-cited { background: #fff4e0; color: #b07a00; } + +.d-actions { + position: sticky; + top: 0; + z-index: 5; + display: flex; + gap: 10px; + align-items: center; + background: var(--bg); + padding: 10px 0; + margin-bottom: 8px; +} +.btn { + border: none; + border-radius: 8px; + padding: 9px 20px; + font-size: 14px; + font-weight: 600; + cursor: pointer; + color: #fff; +} +.btn-good { background: var(--research); } +.btn-bad { background: var(--bad); } +.btn-clear { background: #e7eaf0; color: #445; } +.btn:hover { filter: brightness(0.95); } +.btn.is-on { box-shadow: 0 0 0 3px rgba(0,0,0,.15) inset; } + +/* 好 → 子类下拉 */ +.good-wrap { position: relative; display: inline-block; } +.good-menu { + position: absolute; + top: calc(100% + 4px); + left: 0; + background: #fff; + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: 0 6px 18px rgba(0,0,0,.12); + padding: 4px; + display: none; + z-index: 10; + min-width: 130px; +} +.good-wrap.open .good-menu { display: block; } +.menu-item { + display: block; + width: 100%; + text-align: left; + border: none; + background: none; + padding: 8px 12px; + border-radius: 6px; + cursor: pointer; + font-size: 14px; +} +.menu-item:hover { background: #f0f4fb; } +.menu-item.is-current { background: var(--research-bg); color: var(--research); font-weight: 600; } + +.pdf-wrap { + border: 1px solid var(--border); + border-radius: var(--radius); + overflow: hidden; + background: #fafbfc; + height: 70vh; + margin-bottom: 16px; +} +.pdf-wrap iframe { width: 100%; height: 100%; border: none; } +.pdf-missing { + display: flex; + align-items: center; + justify-content: center; + height: 100%; + color: var(--muted); +} + +.abs { + background: var(--panel); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 14px 16px; + line-height: 1.6; + color: #37404d; + margin-bottom: 16px; +} +.abs summary { cursor: pointer; font-weight: 600; margin-bottom: 8px; } + +/* 备注 */ +.note-block { + background: var(--panel); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 12px 14px; +} +.note-label { font-weight: 600; display: block; margin-bottom: 6px; } +.note-input { + width: 100%; + min-height: 72px; + border: 1px solid var(--border); + border-radius: 8px; + padding: 8px 10px; + font-family: inherit; + font-size: 13px; + resize: vertical; + outline: none; +} +.note-input:focus { border-color: var(--accent); } +.note-status { font-size: 12px; color: var(--muted); } +.note-status.saved { color: #1a9e5c; } + +/* ---- hint ---- */ +.kb-hint { + position: fixed; + right: 14px; + bottom: 12px; + font-size: 12px; + color: var(--muted); + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + padding: 6px 10px; + box-shadow: 0 2px 8px rgba(0,0,0,.06); +} +kbd { + background: #eef1f6; + border: 1px solid #d8dee8; + border-radius: 4px; + padding: 0 5px; + font-family: ui-monospace, Consolas, monospace; +} diff --git a/experiments/literature_review/reviewer/sync_classified.py b/experiments/literature_review/reviewer/sync_classified.py new file mode 100644 index 0000000..88fbbde --- /dev/null +++ b/experiments/literature_review/reviewer/sync_classified.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +"""重建 classified/ 并生成清单 CSV。 + +分类来源(合并): + 1. review_2505.json —— 2505+ 论文(好/坏/未筛 + 风格 + 备注/优) + 2. Downloads 三个文件夹 —— 2301~2504 手动筛出的「科研风 / 生动活泼 / 反面教材(坏)」 + +用法: + python sync_classified.py # 重建(复制 + 清理过期 + 写 CSV) + python sync_classified.py --dry-run # 只打印统计,不写盘 +""" +import argparse +import csv +import json +import os +import shutil + +HERE = os.path.dirname(os.path.abspath(__file__)) +PAPERS_DIR = os.path.normpath(os.path.join(HERE, "..", "papers")) +PAPERS_JSON = os.path.join(PAPERS_DIR, "papers.json") +PDF_DIR = os.path.join(PAPERS_DIR, "pdf") +REVIEW_JSON = os.path.join(PAPERS_DIR, "review_2505.json") +CLASSIFIED = os.path.join(PAPERS_DIR, "classified") +CSV_PATH = os.path.join(CLASSIFIED, "分类清单.csv") + +DOWN = { + "research": r"C:\Users\Lenovo\Downloads\1.科研风\1.科研风", + "lively": r"C:\Users\Lenovo\Downloads\2.生动型\2.生动型", + "bad": r"C:\Users\Lenovo\Downloads\3.反面教材", +} +YOU_DIR = "优" + + +def copy_pdf(src, dst): + if not os.path.isfile(dst) or os.path.getsize(dst) != os.path.getsize(src): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + papers = {p["arxiv_id"]: p for p in json.load(open(PAPERS_JSON, encoding="utf-8"))} + review = json.load(open(REVIEW_JSON, encoding="utf-8")) + + # aid -> (top, sub, note, you) + cmap = {} + + # 1) 2505+ 从 review + for aid, p in papers.items(): + if aid < "2505": + continue + e = review.get(aid) or {} + label = e.get("label") + style = e.get("style") + note = (e.get("note") or "").strip() + if label == "good": + top, sub = "好", ("生动活泼" if style == "lively" else "科研风") + elif label == "bad": + top, sub = "坏", "" + else: + top, sub = "未筛", "" + you = (label == "good" and "优" in note) + cmap[aid] = (top, sub, note, you) + + # 2) Downloads(2301~2504) + down_ids = {cat: set() for cat in DOWN} + for cat, d in DOWN.items(): + if not os.path.isdir(d): + print(f"[警告] 目录不存在:{d}") + continue + for f in os.listdir(d): + if f.lower().endswith(".pdf"): + down_ids[cat].add(f[:-4]) + for aid in down_ids["research"]: + cmap[aid] = ("好", "科研风", "", False) + for aid in down_ids["lively"]: + cmap[aid] = ("好", "生动活泼", "", False) + for aid in down_ids["bad"]: + cmap[aid] = ("坏", "", "", False) + + # 复制 + 清理 + expected = set() + copied = you_copied = 0 + missing = [] + for aid, (top, sub, note, you) in cmap.items(): + rel = os.path.join(top, sub) if sub else top + src = os.path.join(PDF_DIR, aid + ".pdf") + rel_pdf = os.path.join(rel, aid + ".pdf") + if not os.path.isfile(src): + missing.append(aid) + continue + expected.add(rel_pdf) + if you: + expected.add(os.path.join(YOU_DIR, aid + ".pdf")) + if not args.dry_run: + copy_pdf(src, os.path.join(CLASSIFIED, rel, aid + ".pdf")) + if you: + copy_pdf(src, os.path.join(CLASSIFIED, YOU_DIR, aid + ".pdf")) + you_copied += 1 + copied += 1 + + removed = 0 + if not args.dry_run and os.path.isdir(CLASSIFIED): + for root, _dirs, files in os.walk(CLASSIFIED): + for fn in files: + if not fn.lower().endswith(".pdf"): + continue + full = os.path.join(root, fn) + if os.path.relpath(full, CLASSIFIED) not in expected: + os.remove(full) + removed += 1 + + # CSV + rows = [] + for aid in sorted(cmap): + top, sub, note, you = cmap[aid] + p = papers.get(aid, {}) + rows.append([top, sub, "优" if you else "", aid, + p.get("title", "") or "", note, + p.get("cited_by_count", 0) or 0, p.get("arxiv_url", "") or ""]) + if not args.dry_run: + with open(CSV_PATH, "w", newline="", encoding="utf-8-sig") as f: + w = csv.writer(f) + w.writerow(["分类", "风格", "优", "arxiv_id", "标题", "备注", "被引", "arXiv"]) + w.writerows(rows) + + # 统计 + stat = {} + for aid, (top, sub, note, you) in cmap.items(): + key = f"{top}/{sub}" if sub else top + stat[key] = stat.get(key, 0) + 1 + stat[YOU_DIR] = sum(1 for _, _, _, you in cmap.values() if you) + + print("=== 分类重建" + ("(dry-run)" if args.dry_run else "") + " ===") + for k in sorted(stat): + print(f" {k}: {stat[k]}") + print(f" 合计: {len(cmap)} 篇 · 有 PDF: {copied} · 其中优: {you_copied} · 缺 PDF: {len(missing)}" + (f" · 清理过期: {removed}" if removed else "")) + if missing: + print(" 缺 PDF:", ", ".join(sorted(missing))) + if not args.dry_run: + print(f" CSV: {CSV_PATH} ({len(rows)} 行)") + + +if __name__ == "__main__": + main() diff --git a/experiments/literature_review/scripts/extract_layout_text.py b/experiments/literature_review/scripts/extract_layout_text.py deleted file mode 100644 index 391c4e8..0000000 --- a/experiments/literature_review/scripts/extract_layout_text.py +++ /dev/null @@ -1,55 +0,0 @@ -from pathlib import Path -import re - -import fitz - - -FILES = [ - ("scidoc", Path(r"D:\Downloads\2024.findings-emnlp.780.pdf")), - ("structured_ie", Path(r"D:\Downloads\s41467-024-45563-x.pdf")), - ("scirex", Path(r"D:\Downloads\2020.acl-main.670.pdf")), - ("diagrammergpt", Path(r"C:\Users\zhang\Downloads\2310.12128v2.pdf")), -] - -OUT = Path(r"D:\ResearchFigureStudio\tmp\pdfs") - - -def block_text(block: tuple) -> str: - text = block[4].strip() - text = re.sub(r"([A-Za-z])\-\n([a-z])", r"\1\2", text) - text = re.sub(r"(? str: - width = page.rect.width - blocks = [b for b in page.get_text("blocks") if b[4].strip()] - full, left, right = [], [], [] - for block in blocks: - x0, y0, x1, _y1 = block[:4] - block_width = x1 - x0 - if block_width > width * 0.60 or (x0 < width * 0.22 and x1 > width * 0.78): - full.append(block) - elif (x0 + x1) / 2 < width / 2: - left.append(block) - else: - right.append(block) - - # Full-width title/section blocks precede the two reading columns. This is - # imperfect for wide bottom tables but keeps academic two-column prose in - # substantially better order than a flat y/x sort. - ordered = ( - sorted(full, key=lambda b: (b[1], b[0])) - + sorted(left, key=lambda b: (b[1], b[0])) - + sorted(right, key=lambda b: (b[1], b[0])) - ) - texts = [block_text(b) for b in ordered] - return "\n\n".join(t for t in texts if t) - - -for name, pdf_path in FILES: - doc = fitz.open(pdf_path) - pages = [f"\n\n--- PAGE {i + 1} ---\n\n{page_text(page)}" for i, page in enumerate(doc)] - output = OUT / f"{name}.txt" - output.write_text("".join(pages), encoding="utf-8") - print(name, len(doc), output.stat().st_size) diff --git a/experiments/literature_review/scripts/translate_papers.py b/experiments/literature_review/scripts/translate_papers.py deleted file mode 100644 index ee337d1..0000000 --- a/experiments/literature_review/scripts/translate_papers.py +++ /dev/null @@ -1,191 +0,0 @@ -from __future__ import annotations - -import re -import sys -from pathlib import Path - -from argostranslate import translate -from ftfy import fix_text - - -ROOT = Path(r"D:\ResearchFigureStudio") -SOURCE_DIR = ROOT / "tmp" / "pdfs" -OUTPUT_DIR = ROOT / "output" / "translations" -OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - -PAPERS = [ - { - "source": SOURCE_DIR / "scidoc.txt", - "output": OUTPUT_DIR / "01_SciDoc2Diagrammer-MAF_中文全译初稿.md", - "title_en": "SciDoc2Diagrammer-MAF: Towards Generation of Scientific Diagrams from Documents guided by Multi-Aspect Feedback Refinement", - "title_zh": "SciDoc2Diagrammer-MAF:基于多维反馈精炼的科研文档科学图生成方法", - "authors": "Ishani Mondal, Zongxia Li, Yufang Hou, Anandhavelu Natarajan, Aparna Garimella, Jordan Boyd-Graber", - }, - { - "source": SOURCE_DIR / "structured_ie.txt", - "output": OUTPUT_DIR / "02_Structured_Information_Extraction_中文全译初稿.md", - "title_en": "Structured information extraction from scientific text with large language models", - "title_zh": "使用大语言模型从科学文本中抽取结构化信息", - "authors": "论文作者及单位信息见原始 PDF", - }, - { - "source": SOURCE_DIR / "scirex.txt", - "output": OUTPUT_DIR / "03_SciREX_中文全译初稿.md", - "title_en": "SciREX: A Challenge Dataset for Document-Level Information Extraction", - "title_zh": "SciREX:面向文档级信息抽取的挑战数据集", - "authors": "论文作者及单位信息见原始 PDF", - }, - { - "source": SOURCE_DIR / "diagrammergpt.txt", - "output": OUTPUT_DIR / "04_DiagrammerGPT_中文全译初稿.md", - "title_en": "DiagrammerGPT: Generating Open-Domain, Open-Platform Diagrams via LLM Planning", - "title_zh": "DiagrammerGPT:通过大语言模型规划生成开放领域、开放平台图示", - "authors": "论文作者及单位信息见原始 PDF", - }, -] - - -SECTION_RE = re.compile( - r"^(?:Abstract|Introduction|Background|Methodology|Methods?|Results?|Discussion|Conclusion|" - r"Conclusions|Limitations|Ethics Statement|Acknowledgements?|References|Appendix|" - r"\d+(?:\.\d+)*\s+\S.+|[A-Z]\.\s+\S.+)$", - re.IGNORECASE, -) - - -def clean_lines(raw: str) -> list[str]: - raw = fix_text(raw) - raw = re.sub(r"([A-Za-z])\-\n([a-z])", r"\1\2", raw) - lines: list[str] = [] - for original in raw.splitlines(): - line = original.strip() - if not line: - lines.append("") - continue - if re.fullmatch(r"--- PAGE \d+ ---", line): - continue - if re.fullmatch(r"\d{4,6}", line): - continue - if "Findings of the Association for Computational Linguistics" in line: - continue - if re.match(r"^(?:November|Proceedings of|©|Copyright)", line): - continue - lines.append(line) - return lines - - -def is_heading(line: str) -> bool: - if len(line) > 150: - return False - return bool(SECTION_RE.match(line)) - - -def make_blocks(lines: list[str]) -> list[tuple[str, str]]: - blocks: list[tuple[str, str]] = [] - buffer: list[str] = [] - in_references = False - - def flush() -> None: - if buffer: - text = " ".join(x for x in buffer if x).strip() - if text: - blocks.append(("reference" if in_references else "text", text)) - buffer.clear() - - for line in lines: - if is_heading(line): - flush() - normalized = line.lower() - if normalized == "references": - in_references = True - elif in_references and ( - normalized.startswith("appendix") - or re.match(r"^(?:[9-9]|1[0-9])(?:\.\d+)*\s+", line) - or re.match(r"^[A-Z]\.\s+", line) - ): - in_references = False - blocks.append(("heading", line)) - elif not line: - flush() - else: - buffer.append(line) - if sum(len(x) for x in buffer) > 3200: - flush() - flush() - return blocks - - -HEADING_MAP = { - "abstract": "摘要", - "introduction": "引言", - "background": "背景", - "method": "方法", - "methods": "方法", - "methodology": "方法", - "results": "结果", - "result": "结果", - "discussion": "讨论", - "conclusion": "结论", - "conclusions": "结论", - "limitations": "局限性", - "ethics statement": "伦理声明", - "acknowledgement": "致谢", - "acknowledgements": "致谢", - "references": "参考文献(保留英文原文)", - "appendix": "附录", -} - - -def translate_heading(heading: str) -> str: - key = heading.strip().lower() - if key in HEADING_MAP: - return HEADING_MAP[key] - match = re.match(r"^(\d+(?:\.\d+)*|[A-Z]\.)\s+(.+)$", heading) - if match: - number, text = match.groups() - return f"{number} {translate.translate(text, 'en', 'zh')}" - return translate.translate(heading, "en", "zh") - - -def translate_paper(paper: dict[str, object]) -> None: - source = Path(paper["source"]) - output = Path(paper["output"]) - lines = clean_lines(source.read_text(encoding="utf-8")) - blocks = make_blocks(lines) - rendered = [ - f"# {paper['title_zh']}", - "", - f"**英文标题:** {paper['title_en']}", - "", - f"**作者:** {paper['authors']}", - "", - "> 说明:本文档为离线机器翻译形成的中文全译初稿,已保留公式、数值、图表编号和参考文献引用。参考文献条目保留英文。关键术语与复杂句仍建议结合原 PDF 校对。", - "", - ] - - total = len(blocks) - for index, (kind, content) in enumerate(blocks, start=1): - if kind == "heading": - rendered.extend([f"## {translate_heading(content)}", ""]) - elif kind == "reference": - rendered.extend([content, ""]) - else: - translated = translate.translate(content, "en", "zh") - rendered.extend([translated.strip(), ""]) - if index % 10 == 0 or index == total: - print(f"{output.name}: {index}/{total}", flush=True) - - output.write_text("\n".join(rendered).strip() + "\n", encoding="utf-8") - print(f"WROTE {output}", flush=True) - - -def main() -> None: - requested = set(sys.argv[1:]) - for index, paper in enumerate(PAPERS, start=1): - if requested and str(index) not in requested: - continue - translate_paper(paper) - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index cc3c97a..bbc58df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "PyMuPDF", "pypdf", "pdfplumber", + "flask", "requests", "opencv-python-headless", "wordninja", @@ -32,4 +33,4 @@ rfs = "rfs.cli:main" [tool.setuptools.packages.find] where = ["."] -include = ["rfs*"] +include = ["rfs*", "webapp*"] diff --git a/reference-figures/README.md b/reference-figures/README.md new file mode 100644 index 0000000..0132143 --- /dev/null +++ b/reference-figures/README.md @@ -0,0 +1,50 @@ +# Reference Figures + +Put local paper framework figures here for the paper-to-image reference learning workflow. + +Suggested use: + +- `excellent/`: strong framework figures that Paperimage should learn from. +- `negative/`: weak or unwanted styles that Paperimage should avoid. +- `user/`: reserved for future user-provided style learning. + +Do not commit real paper figures, screenshots, PDFs, or private data. The workflow will write structured analysis records to an output directory; those records are safer to review and curate than the original images. + +Stage 1 command: + +```powershell +rfs analyze-reference-figures ` + --references reference-figures ` + --out output/reference_learning_stage1 ` + --mode heuristic ` + --json +``` + +Stage 2 API-backed library build: + +```powershell +Get-Content .env | ForEach-Object { + if ($_ -match '^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$') { + [Environment]::SetEnvironmentVariable($matches[1], $matches[2].Trim("'"""), "Process") + } +} + +rfs analyze-reference-figures ` + --references reference-figures ` + --out output/reference_learning_stage2 ` + --mode vlm ` + --model gpt-5.6-sol ` + --json +``` + +Stage 3 exemplar selection for a paper run: + +```powershell +rfs select-reference-exemplars ` + --paper-review output/paper_to_image/paper_review.json ` + --library output/reference_learning_stage2/paper_to_image_reference_library.jsonl ` + --out output/paper_to_image/reference_exemplars ` + --selected-template output/paper_to_image/selected_template.json ` + --top-k 5 ` + --json +``` diff --git a/reference-figures/excellent/.gitkeep b/reference-figures/excellent/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference-figures/excellent/.gitkeep @@ -0,0 +1 @@ + diff --git a/reference-figures/negative/.gitkeep b/reference-figures/negative/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference-figures/negative/.gitkeep @@ -0,0 +1 @@ + diff --git a/reference-figures/user/.gitkeep b/reference-figures/user/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/reference-figures/user/.gitkeep @@ -0,0 +1 @@ + diff --git a/rfs/analysis/__init__.py b/rfs/analysis/__init__.py index c0522fe..f181c12 100644 --- a/rfs/analysis/__init__.py +++ b/rfs/analysis/__init__.py @@ -3,8 +3,14 @@ Implementations remain in their existing modules while interfaces stabilize. """ -from ..layout_planner import plan_reference_layout from ..paper_to_image.analyzer import parse_paper -from ..reference_text_extractor import extract_reference_text +from .rebuild import estimate_background, extract_reference_text, group_text_regions, group_text_regions_heuristic, plan_reference_layout -__all__ = ["extract_reference_text", "parse_paper", "plan_reference_layout"] +__all__ = [ + "estimate_background", + "extract_reference_text", + "group_text_regions", + "group_text_regions_heuristic", + "parse_paper", + "plan_reference_layout", +] diff --git a/rfs/analysis/rebuild/__init__.py b/rfs/analysis/rebuild/__init__.py new file mode 100644 index 0000000..79efd3e --- /dev/null +++ b/rfs/analysis/rebuild/__init__.py @@ -0,0 +1,16 @@ +"""Canonical analysis APIs for reference-image editable rebuilds.""" + +from .controls import localize_reference_controls +from .layout import dominant_palette, estimate_background, plan_reference_layout +from .reference_text import extract_reference_text +from ...text_grouping import group_text_regions, group_text_regions_heuristic + +__all__ = [ + "estimate_background", + "dominant_palette", + "extract_reference_text", + "group_text_regions", + "group_text_regions_heuristic", + "localize_reference_controls", + "plan_reference_layout", +] diff --git a/rfs/analysis/rebuild/controls.py b/rfs/analysis/rebuild/controls.py new file mode 100644 index 0000000..fae3bd2 --- /dev/null +++ b/rfs/analysis/rebuild/controls.py @@ -0,0 +1,832 @@ +from __future__ import annotations + +import math +import inspect +from pathlib import Path +from typing import Callable + +from PIL import Image, ImageDraw + +from ...utils import read_json, write_json +from ...cv_compat import hough_line_coordinates + +VALID_RENDER_STYLES = { + "filled_block_arrow", + "line_connector", + "elbow_connector", + "branch_line_connector", + "dashed_loop_connector", +} +VALID_ROUTE_INTENTS = {"straight", "orthogonal", "branch", "loop"} +VALID_VISUAL_WEIGHTS = {"chunky", "normal", "thin"} +VALID_PREFERRED_AXES = {"horizontal", "vertical", "horizontal_first", "vertical_first"} +VALID_LINE_PATTERNS = {"solid", "dash", "dashed"} + + +def _center(box: dict) -> tuple[float, float]: + return float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2 + + +def _nearest_slot(point: tuple[float, float], slots: list[dict]) -> dict | None: + if not slots: + return None + px, py = point + return min(slots, key=lambda slot: (_center(slot["bbox_percent"])[0] - px) ** 2 + (_center(slot["bbox_percent"])[1] - py) ** 2) + + +def _distance_to_bbox(point: tuple[float, float], obj: dict | None) -> float: + if not obj or not isinstance(obj.get("bbox_percent"), dict): + return 1.0 + box = obj["bbox_percent"] + px, py = point + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + dx = max(x - px, 0.0, px - (x + w)) + dy = max(y - py, 0.0, py - (y + h)) + return math.hypot(dx, dy) + + +def _coerce_choice(value, allowed: set[str], default: str = "") -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "block_arrow": "filled_block_arrow", + "chunky_arrow": "filled_block_arrow", + "thick_arrow": "filled_block_arrow", + "straight_arrow": "line_connector", + "elbow": "elbow_connector", + "orthogonal_connector": "elbow_connector", + "branch_connector": "branch_line_connector", + "dashed_loop": "dashed_loop_connector", + "loop_connector": "dashed_loop_connector", + "dashed": "dash", + } + text = aliases.get(text, text) + return text if text in allowed else default + + +def _slot_by_id(slots: list[dict]) -> dict[str, dict]: + return {str(slot.get("id")): slot for slot in slots if isinstance(slot, dict) and slot.get("id")} + + +def _path_bbox(points: list[list[float]]) -> dict[str, float]: + xs = [float(point[0]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + ys = [float(point[1]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + if not xs or not ys: + return {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0} + x0 = min(xs) + y0 = min(ys) + return {"x": x0, "y": y0, "w": max(xs) - x0, "h": max(ys) - y0} + + +def _bbox_iou(a: dict, b: dict) -> float: + ax0, ay0 = float(a.get("x", 0.0)), float(a.get("y", 0.0)) + ax1, ay1 = ax0 + float(a.get("w", 0.0)), ay0 + float(a.get("h", 0.0)) + bx0, by0 = float(b.get("x", 0.0)), float(b.get("y", 0.0)) + bx1, by1 = bx0 + float(b.get("w", 0.0)), by0 + float(b.get("h", 0.0)) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + inter = ix * iy + union = float(a.get("w", 0.0)) * float(a.get("h", 0.0)) + float(b.get("w", 0.0)) * float(b.get("h", 0.0)) - inter + return inter / max(union, 0.000001) + + +def _point_distance(a: list[float], b: list[float]) -> float: + return math.hypot(float(a[0]) - float(b[0]), float(a[1]) - float(b[1])) + + +def _path_distance(a: list[list[float]], b: list[list[float]]) -> float: + if len(a) < 2 or len(b) < 2: + return 1.0 + return (_point_distance(a[0], b[0]) + _point_distance(a[-1], b[-1])) / 2 + + +def _control_signature(control: dict) -> tuple[str, str]: + return str(control.get("source_id") or control.get("source") or ""), str(control.get("target_id") or control.get("target") or "") + + +def _controls_are_duplicate(a: dict, b: dict) -> bool: + ap = a.get("path_percent") or [] + bp = b.get("path_percent") or [] + same_binding = _control_signature(a) == _control_signature(b) + if same_binding and _path_distance(ap, bp) <= 0.08: + return True + return _bbox_iou(_path_bbox(ap), _path_bbox(bp)) >= 0.55 and _path_distance(ap, bp) <= 0.12 + + +def _controls_match_for_disagreement(heuristic: dict, vlm: dict) -> bool: + if _controls_are_duplicate(heuristic, vlm): + return True + hp = heuristic.get("path_percent") or [] + vp = vlm.get("path_percent") or [] + return _control_signature(heuristic) == _control_signature(vlm) or _path_distance(hp, vp) <= 0.08 or _bbox_iou(_path_bbox(hp), _path_bbox(vp)) >= 0.35 + + +def _bindable_objects(out_dir: Path, slots: list[dict]) -> list[dict]: + bindable = [dict(slot) for slot in slots if isinstance(slot, dict) and isinstance(slot.get("bbox_percent"), dict)] + seen = {str(item.get("id")) for item in bindable} + try: + geometry = read_json(out_dir / "reference_geometry.json") + except Exception: + geometry = {} + for card in geometry.get("cards", []) if isinstance(geometry, dict) else []: + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + card_id = str(card.get("id") or "") + if not card_id or card_id in seen: + continue + enriched = dict(card) + enriched.setdefault("display_label", card.get("title") or card_id) + enriched.setdefault("paper_concept", card.get("title") or card_id) + enriched.setdefault("semantic_role", "reference_card") + enriched.setdefault("asset_type", "editable_card") + bindable.append(enriched) + seen.add(card_id) + return bindable + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _center(sbox) + tx, ty = _center(tbox) + if abs(tx - sx) >= abs(ty - sy): + start = (float(sbox["x"]) + (float(sbox["w"]) if tx >= sx else 0.0), sy) + end = (float(tbox["x"]) + (0.0 if tx >= sx else float(tbox["w"])), ty) + else: + start = (sx, float(sbox["y"]) + (float(sbox["h"]) if ty >= sy else 0.0)) + end = (tx, float(tbox["y"]) + (0.0 if ty >= sy else float(tbox["h"]))) + return [[round(start[0], 4), round(start[1], 4)], [round(end[0], 4), round(end[1], 4)]] + + +def _sequence_controls(slots: list[dict], palette: list[str]) -> list[dict]: + ordered = sorted(slots, key=lambda slot: (float(slot["bbox_percent"]["y"]), float(slot["bbox_percent"]["x"]))) + arrows = [] + for idx, (src, dst) in enumerate(zip(ordered[:-1], ordered[1:]), start=1): + sbox = src["bbox_percent"] + dbox = dst["bbox_percent"] + sx = float(sbox["x"]) + float(sbox["w"]) + sy = float(sbox["y"]) + float(sbox["h"]) / 2 + tx = float(dbox["x"]) + ty = float(dbox["y"]) + float(dbox["h"]) / 2 + dx = tx - sx + dy = ty - sy + arrows.append({ + "id": f"arrow_{idx:02d}", + "source": src["id"], + "target": dst["id"], + "source_id": src["id"], + "target_id": dst["id"], + "control_kind": "straight_arrow", + "path_percent": [[round(sx, 4), round(sy, 4)], [round(tx, 4), round(ty, 4)]], + "arrowhead_direction": round(math.degrees(math.atan2(dy, dx)), 2), + "stroke_color": palette[2] if len(palette) > 2 else "#4A6FA5", + "stroke_width_pt": 1.5, + "dash_style": "solid", + "line_pattern": "solid", + "confidence": 0.35, + "detected_by": "sequence_fallback", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + }) + return arrows + + +def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str]) -> list[dict]: + try: + import cv2 + except Exception: + return [] + image = cv2.imread(str(reference_path)) + if image is None: + return [] + height, width = image.shape[:2] + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + edges = cv2.Canny(gray, 50, 150, apertureSize=3) + lines = cv2.HoughLinesP(edges, 1, math.pi / 180, threshold=40, minLineLength=max(24, min(width, height) // 14), maxLineGap=12) + if lines is None: + return [] + candidates = [] + for line in lines[:80]: + coordinates = hough_line_coordinates(line) + if coordinates is None: + values = line.reshape(-1) if hasattr(line, "reshape") else line + if len(values) < 4: + continue + coordinates = [int(v) for v in values[:4]] + x1, y1, x2, y2 = coordinates + length = math.hypot(x2 - x1, y2 - y1) + if length < min(width, height) * 0.06: + continue + p1 = (x1 / width, y1 / height) + p2 = (x2 / width, y2 / height) + source = _nearest_slot(p1, slots) + target = _nearest_slot(p2, slots) + if not source or not target or source["id"] == target["id"]: + continue + if _distance_to_bbox(p1, source) > 0.075 or _distance_to_bbox(p2, target) > 0.075: + continue + direction = math.degrees(math.atan2(p2[1] - p1[1], p2[0] - p1[0])) + candidates.append({ + "line": [p1, p2], + "length": length, + "source": source, + "target": target, + "direction": direction, + }) + candidates.sort(key=lambda item: item["length"], reverse=True) + arrows = [] + seen = set() + for candidate in candidates: + source = candidate["source"]["id"] + target = candidate["target"]["id"] + key = (source, target) + if key in seen: + continue + seen.add(key) + p1, p2 = candidate["line"] + arrows.append({ + "id": f"arrow_{len(arrows) + 1:02d}", + "source": source, + "target": target, + "source_id": source, + "target_id": target, + "control_kind": "straight_arrow", + "path_percent": [[round(p1[0], 4), round(p1[1], 4)], [round(p2[0], 4), round(p2[1], 4)]], + "arrowhead_direction": round(candidate["direction"], 2), + "stroke_color": palette[2] if len(palette) > 2 else "#4A6FA5", + "stroke_width_pt": 1.4, + "dash_style": "solid", + "line_pattern": "solid", + "confidence": 0.55, + "detected_by": "cv_hough_line", + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + }) + if len(arrows) >= max(1, min(16, len(slots) + 2)): + break + return arrows + + +def _normalize_controls(raw: list[dict], slots: list[dict], palette: list[str]) -> list[dict]: + slot_ids = {str(slot.get("id")) for slot in slots} + objects = _slot_by_id(slots) + normalized = [] + for idx, item in enumerate(raw or [], start=1): + if not isinstance(item, dict): + continue + source = str(item.get("source_id") or item.get("source") or "") + target = str(item.get("target_id") or item.get("target") or "") + path = item.get("path_percent") + if source not in slot_ids or target not in slot_ids: + continue + points = [] + if isinstance(path, list): + for point in path: + if not isinstance(point, (list, tuple)) or len(point) < 2: + continue + points.append([round(max(0.0, min(1.0, float(point[0]))), 4), round(max(0.0, min(1.0, float(point[1]))), 4)]) + has_reference_path = len(points) >= 2 + if len(points) < 2: + points = _edge_path(objects.get(source), objects.get(target)) + if len(points) < 2: + continue + dx = points[-1][0] - points[0][0] + dy = points[-1][1] - points[0][1] + dash_style = str(item.get("dash_style") or item.get("line_pattern") or "solid") + line_pattern = _coerce_choice(dash_style, VALID_LINE_PATTERNS, "solid") + render_style = _coerce_choice(item.get("render_style"), VALID_RENDER_STYLES, "") + route_intent = _coerce_choice(item.get("route_intent"), VALID_ROUTE_INTENTS, "") + visual_weight = _coerce_choice(item.get("visual_weight"), VALID_VISUAL_WEIGHTS, "") + preferred_axis = _coerce_choice(item.get("preferred_axis"), VALID_PREFERRED_AXES, "") + explicit_route_policy = str(item.get("route_policy") or "").strip() + route_policy = explicit_route_policy or ("preserve_reference_path" if has_reference_path else "bbox_route_from_hint") + route_style = str(item.get("route_style") or ("dashed_loop" if str(item.get("control_kind") or "").lower() == "dashed_loop" else "")) + if line_pattern in {"dash", "dashed"} and len(points) >= 3 and not route_style: + route_style = "dashed_spline_like" + record = { + "id": str(item.get("id") or f"arrow_{idx:02d}"), + "source": source, + "target": target, + "source_id": source, + "target_id": target, + "target_ids": [str(target_id) for target_id in item.get("target_ids", []) if str(target_id) in slot_ids] if isinstance(item.get("target_ids"), list) else [], + "control_kind": str(item.get("control_kind") or "straight_arrow"), + "path_percent": points, + "arrowhead_direction": float(item.get("arrowhead_direction") if item.get("arrowhead_direction") is not None else math.degrees(math.atan2(dy, dx))), + "stroke_color": str(item.get("stroke_color") or (palette[2] if len(palette) > 2 else "#4A6FA5")), + "stroke_width_pt": float(item.get("stroke_width_pt") or 1.5), + "dash_style": dash_style, + "line_pattern": "dash" if line_pattern in {"dash", "dashed"} else "solid", + "route_style": route_style, + "render_style": render_style, + "route_intent": route_intent, + "visual_weight": visual_weight, + "preferred_axis": preferred_axis, + "bend_side": str(item.get("bend_side") or ""), + "fill_color": str(item.get("fill_color") or ""), + "outline_color": str(item.get("outline_color") or ""), + "outline_width_pt": item.get("outline_width_pt"), + "block_arrow_thickness_percent": item.get("block_arrow_thickness_percent"), + "line_cap": str(item.get("line_cap") or ""), + "arrowhead_size": str(item.get("arrowhead_size") or ""), + "binding_source": str(item.get("binding_source") or ("heuristic" if str(item.get("detected_by") or "").startswith(("cv_", "sequence_")) else "vlm")), + "route_policy": route_policy, + "confidence": float(item.get("confidence") or 0.8), + "detected_by": str(item.get("detected_by") or "vlm"), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + } + normalized.append({key: value for key, value in record.items() if value not in ("", [], None)}) + return normalized + + +def _disagreement_brief(item: dict | None) -> dict | None: + if not isinstance(item, dict): + return None + return { + key: item.get(key) + for key in ( + "id", + "source_id", + "target_id", + "control_kind", + "render_style", + "route_intent", + "visual_weight", + "path_percent", + "stroke_color", + "stroke_width_pt", + "dash_style", + "line_pattern", + "confidence", + "detected_by", + ) + if key in item + } + + +def _control_disagreements(heuristic: list[dict], vlm: list[dict]) -> dict: + matched_heuristic: set[int] = set() + matched_vlm: set[int] = set() + disagreements: list[dict] = [] + duplicates: list[dict] = [] + for v_idx, vlm_item in enumerate(vlm): + best_idx = None + best_score = -1.0 + for h_idx, heuristic_item in enumerate(heuristic): + if h_idx in matched_heuristic: + continue + if not _controls_match_for_disagreement(heuristic_item, vlm_item): + continue + score = _bbox_iou(_path_bbox(heuristic_item.get("path_percent") or []), _path_bbox(vlm_item.get("path_percent") or [])) + if _control_signature(heuristic_item) == _control_signature(vlm_item): + score += 1.0 + score -= _path_distance(heuristic_item.get("path_percent") or [], vlm_item.get("path_percent") or []) + if score > best_score: + best_idx = h_idx + best_score = score + if best_idx is None: + continue + heuristic_item = heuristic[best_idx] + matched_heuristic.add(best_idx) + matched_vlm.add(v_idx) + same_style = ( + str(heuristic_item.get("render_style") or "") == str(vlm_item.get("render_style") or "") + or not heuristic_item.get("render_style") + or not vlm_item.get("render_style") + ) + same_route = ( + str(heuristic_item.get("route_intent") or "") == str(vlm_item.get("route_intent") or "") + or not heuristic_item.get("route_intent") + or not vlm_item.get("route_intent") + ) + if _controls_are_duplicate(heuristic_item, vlm_item) and same_style and same_route: + duplicates.append({ + "id": f"duplicate_{len(duplicates)+1:02d}", + "kind": "duplicate", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + else: + disagreements.append({ + "id": f"control_mismatch_{len(disagreements)+1:02d}", + "kind": "mismatch", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + for idx, item in enumerate(heuristic): + if idx not in matched_heuristic: + disagreements.append({ + "id": f"heuristic_only_{len(disagreements)+1:02d}", + "kind": "heuristic_only", + "heuristic": _disagreement_brief(item), + "vlm": None, + "default_decision": "keep_heuristic", + }) + for idx, item in enumerate(vlm): + if idx not in matched_vlm: + disagreements.append({ + "id": f"vlm_only_{len(disagreements)+1:02d}", + "kind": "vlm_only", + "heuristic": None, + "vlm": _disagreement_brief(item), + "default_decision": "keep_vlm", + }) + return { + "summary": "Control disagreements between heuristic/CV and VLM arrow detection.", + "heuristic_count": len(heuristic), + "vlm_count": len(vlm), + "duplicate_count": len(duplicates), + "disagreement_count": len(disagreements), + "duplicates": duplicates, + "disagreements": disagreements, + } + + +def _call_adjudicator(adapter: Callable, reference: Path, bindable: list[dict], disagreements: dict, flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, disagreements, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, disagreements) + + +def _decision_by_id(raw: dict | list | None) -> dict[str, dict]: + decisions = raw.get("decisions", []) if isinstance(raw, dict) else raw + result = {} + for item in decisions or []: + if not isinstance(item, dict): + continue + disagreement_id = str(item.get("disagreement_id") or item.get("id") or "") + if disagreement_id: + result[disagreement_id] = item + return result + + +def _append_unique(arrows: list[dict], candidate: dict) -> bool: + if any(_controls_are_duplicate(existing, candidate) for existing in arrows): + return False + candidate_id = str(candidate.get("id") or f"arrow_{len(arrows)+1:02d}") + used_ids = {str(item.get("id") or "") for item in arrows} + if candidate_id in used_ids: + stem = candidate_id + suffix = 2 + while f"{stem}_{suffix}" in used_ids: + suffix += 1 + candidate = dict(candidate) + candidate["id"] = f"{stem}_{suffix}" + arrows.append(candidate) + return True + + +def _merge_with_adjudication( + heuristic: list[dict], + vlm: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + adjudicator: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict, dict]: + disagreements = _control_disagreements(heuristic, vlm) + write_json(out / "reference_control_disagreements.json", disagreements) + if mode == "off": + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(vlm), + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return vlm, disagreements, adjudication_doc + raw_adjudication: dict | list | None = None + decisions: dict[str, dict] = {} + status = "skipped" + if mode == "vlm" and adjudicator and disagreements["disagreement_count"]: + try: + raw_adjudication = _call_adjudicator(adjudicator, reference, bindable, disagreements, flow_graph) + decisions = _decision_by_id(raw_adjudication) + status = "used" + except Exception as exc: + status = "fallback" + warnings.append(f"control_adjudication_failed:{exc}") + elif mode == "vlm" and disagreements["disagreement_count"]: + status = "unavailable" + merged: list[dict] = [] + for duplicate in disagreements["duplicates"]: + vlm_item = duplicate.get("vlm") + normalized = _normalize_controls([vlm_item], bindable, palette) + if normalized: + _append_unique(merged, normalized[0]) + for item in disagreements["disagreements"]: + decision = decisions.get(str(item.get("id") or ""), {}) + action = str(decision.get("decision") or item.get("default_decision") or "").lower() + selected: list[dict] = [] + if action == "keep_heuristic": + if item.get("heuristic"): + selected = [item["heuristic"]] + elif action == "keep_both": + selected = [candidate for candidate in [item.get("vlm"), item.get("heuristic")] if candidate] + elif action == "drop_both": + selected = [] + elif action == "merge_or_adjust": + selected = [decision.get("merged_control") or decision.get("control") or item.get("vlm") or item.get("heuristic")] + else: + selected = [item.get("vlm") or item.get("heuristic")] + normalized = _normalize_controls(selected, bindable, palette) + if len(normalized) < len([candidate for candidate in selected if candidate]): + warnings.append(f"invalid_control_adjudication_item_dropped:{item.get('id')}") + for candidate in normalized: + _append_unique(merged, candidate) + if not merged: + merged = vlm or heuristic + adjudication_doc = { + "summary": "Control disagreement adjudication report.", + "mode": mode, + "status": status, + "decision_count": len(decisions), + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(merged), + "raw_adjudication": raw_adjudication, + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return merged, disagreements, adjudication_doc + + +def _call_coverage_audit(adapter: Callable, reference: Path, bindable: list[dict], arrows: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, arrows, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, arrows) + + +def _apply_coverage_audit( + arrows: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + coverage_audit_adapter: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict]: + audit_doc: dict = { + "summary": "Control coverage audit report.", + "mode": mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } + if mode != "vlm": + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + if not coverage_audit_adapter: + audit_doc["status"] = "unavailable" + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + try: + raw = _call_coverage_audit(coverage_audit_adapter, reference, bindable, arrows, flow_graph) + raw_items = raw.get("missing_controls") if isinstance(raw, dict) else raw + if raw_items is None and isinstance(raw, dict): + raw_items = raw.get("arrows", []) + normalized = _normalize_controls(raw_items or [], bindable, palette) + appended = 0 + for candidate in normalized: + candidate = dict(candidate) + candidate["detected_by"] = "coverage_audit_vlm" + candidate["binding_source"] = "coverage_audit_vlm" + if _append_unique(arrows, candidate): + appended += 1 + audit_doc.update({ + "status": "used", + "raw_audit": raw, + "missing_control_count": len(raw_items or []), + "valid_missing_control_count": len(normalized), + "appended_control_count": appended, + }) + invalid_count = max(0, len(raw_items or []) - len(normalized)) + if invalid_count: + warnings.append(f"{invalid_count}_invalid_coverage_audit_control(s)_dropped") + except Exception as exc: + audit_doc["status"] = "fallback" + audit_doc["fallback_reason"] = f"control_coverage_audit_failed:{exc}" + warnings.append(str(audit_doc["fallback_reason"])) + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + + +def _draw_overlay(reference_path: Path, out_path: Path, arrows: list[dict]) -> None: + with Image.open(reference_path).convert("RGB") as image: + draw = ImageDraw.Draw(image) + width, height = image.size + for arrow in arrows: + points = arrow.get("path_percent") or [] + if len(points) < 2: + continue + xy = [(float(p[0]) * width, float(p[1]) * height) for p in points] + color = str(arrow.get("stroke_color") or "#D94141") + draw.line(xy, fill=color, width=4) + end = xy[-1] + prev = xy[-2] + angle = math.atan2(end[1] - prev[1], end[0] - prev[0]) + head = 12 + left = (end[0] - head * math.cos(angle - 0.45), end[1] - head * math.sin(angle - 0.45)) + right = (end[0] - head * math.cos(angle + 0.45), end[1] - head * math.sin(angle + 0.45)) + draw.polygon([end, left, right], fill=color) + draw.text((end[0] + 3, end[1] + 3), str(arrow.get("id") or ""), fill=color) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.save(out_path) + + +def _flow_edges(flow_graph: dict | None) -> list[dict]: + if not isinstance(flow_graph, dict): + return [] + return [item for item in flow_graph.get("edges", []) or [] if isinstance(item, dict)] + + +def _flow_graph_stats(flow_graph: dict | None, objects: list[dict]) -> dict: + object_ids = {str(item.get("id") or "") for item in objects if isinstance(item, dict)} + edges = _flow_edges(flow_graph) + invalid = [] + for edge in edges: + source = str(edge.get("source_id") or edge.get("source") or "") + target = str(edge.get("target_id") or edge.get("target") or "") + if source not in object_ids or target not in object_ids: + invalid.append(str(edge.get("id") or f"{source}->{target}")) + return { + "flow_graph_used": bool(edges), + "flow_graph_edge_count": len(edges), + "invalid_flow_edge_count": len(invalid), + "invalid_flow_edge_ids": invalid, + } + + +def _call_control_adapter(adapter: Callable, reference: Path, bindable: list[dict], heuristic: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, heuristic, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, heuristic) + + +def localize_reference_controls( + reference_path: str | Path, + slots: list[dict], + palette: list[str], + out_dir: str | Path, + mode: str = "hybrid", + control_adapter: Callable[[str | Path, list[dict], list[dict]], dict | list[dict]] | None = None, + flow_graph: dict | None = None, + control_adjudication_mode: str = "off", + control_adjudicator: Callable | None = None, + control_coverage_audit_mode: str = "off", + coverage_audit_adapter: Callable | None = None, +) -> dict: + reference = Path(reference_path) + out = Path(out_dir) + warnings = [] + bindable = _bindable_objects(out, slots) + flow_stats = _flow_graph_stats(flow_graph, bindable) + if flow_stats["invalid_flow_edge_count"]: + warnings.append(f"{flow_stats['invalid_flow_edge_count']}_invalid_flow_graph_edge(s)") + cv_heuristic = _cv_line_controls(reference, bindable, palette) + heuristic = cv_heuristic or _sequence_controls(slots, palette) + _draw_overlay(reference, out / "reference_controls_candidates_overlay.png", heuristic) + arrows = heuristic + vlm_status = "not_requested" + normalized: list[dict] = [] + raw: dict | list | None = None + disagreements = { + "summary": "Control disagreement detection skipped.", + "heuristic_count": len(heuristic), + "vlm_count": 0, + "duplicate_count": 0, + "disagreement_count": 0, + "duplicates": [], + "disagreements": [], + } + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": control_adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": 0, + "input_duplicate_count": 0, + "merged_arrow_count": len(arrows), + } + audit_doc = { + "summary": "Control coverage audit skipped.", + "mode": control_coverage_audit_mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } + if mode in {"vlm", "hybrid"}: + if control_adapter: + try: + raw = _call_control_adapter(control_adapter, reference, bindable, heuristic, flow_graph) + raw_arrows = raw.get("arrows", []) if isinstance(raw, dict) else raw + normalized = _normalize_controls(raw_arrows, bindable, palette) + if normalized: + vlm_status = "used" + if mode == "hybrid": + arrows, disagreements, adjudication_doc = _merge_with_adjudication( + heuristic, + normalized, + bindable, + palette, + reference, + out, + flow_graph, + control_adjudication_mode, + control_adjudicator, + warnings, + ) + else: + arrows = normalized + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + invalid_count = max(0, len(raw_arrows or []) - len(normalized)) + if invalid_count: + warnings.append(f"{invalid_count}_invalid_vlm_control_arrow(s)_dropped") + else: + vlm_status = "fallback" + warnings.append("control_adapter_returned_no_valid_arrows") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + except Exception as exc: + vlm_status = "fallback" + warnings.append(f"vlm_control_failed:{exc}") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + else: + vlm_status = "unavailable_fallback_to_heuristic" + if mode == "vlm": + warnings.append("control_mode_vlm_requested_without_adapter") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + else: + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + if mode == "manual": + arrows = [] + vlm_status = "manual_empty" + _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) + if mode in {"vlm", "hybrid"}: + arrows, audit_doc = _apply_coverage_audit( + arrows, + bindable, + palette, + reference, + out, + flow_graph, + control_coverage_audit_mode, + coverage_audit_adapter, + warnings, + ) + _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) + return { + "summary": "Reference control localization report.", + "mode": mode, + "status": "ok", + "vlm_status": vlm_status, + "vlm_model": str(raw.get("_vlm_model")) if mode in {"vlm", "hybrid"} and isinstance(raw, dict) and raw.get("_vlm_model") else None, + "bindable_object_count": len(bindable), + "heuristic_arrow_count": len(heuristic), + "cv_heuristic_arrow_count": len(cv_heuristic), + "vlm_arrow_count": len(normalized), + "merged_arrow_count": len(arrows), + "control_adjudication_mode": control_adjudication_mode, + "control_adjudication_status": adjudication_doc.get("status"), + "control_disagreement_count": disagreements.get("disagreement_count", 0), + "control_duplicate_count": disagreements.get("duplicate_count", 0), + "control_coverage_audit_mode": control_coverage_audit_mode, + "control_coverage_audit_status": audit_doc.get("status"), + "control_coverage_appended_count": audit_doc.get("appended_control_count", 0), + "warnings": warnings, + "arrows": arrows, + **flow_stats, + } diff --git a/rfs/analysis/rebuild/layout.py b/rfs/analysis/rebuild/layout.py new file mode 100644 index 0000000..b4dd4cd --- /dev/null +++ b/rfs/analysis/rebuild/layout.py @@ -0,0 +1,1017 @@ +from __future__ import annotations + +import inspect +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Callable + +from PIL import Image, ImageDraw + +from ...utils import write_json + + +PANELWISE_LAYOUT_MAX_PANELS = 8 +PANELWISE_LAYOUT_MAX_WORKERS = 3 +PANELWISE_MIN_PANEL_AREA = 0.01 + + +def clamp_bbox(bbox: dict) -> dict[str, float]: + x = max(0.0, min(0.995, float(bbox.get("x", 0.0)))) + y = max(0.0, min(0.995, float(bbox.get("y", 0.0)))) + w = max(0.001, min(float(bbox.get("w", 0.001)), 1.0 - x)) + h = max(0.001, min(float(bbox.get("h", 0.001)), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: + ratio = width_px / max(height_px, 1) + width = max(10.0, min(15.6, 7.5 * ratio)) + return round(width, 3), round(width / ratio, 3) + + +def rgb_hex(rgb: tuple[int, int, int]) -> str: + return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" + + +def estimate_background(image: Image.Image) -> str: + rgb = image.convert("RGB") + w, h = rgb.size + samples = [] + edge = max(1, min(w, h) // 12) + for box in [(0, 0, w, edge), (0, h - edge, w, h), (0, 0, edge, h), (w - edge, 0, w, h)]: + crop = rgb.crop(box).resize((1, 1)) + samples.append(crop.getpixel((0, 0))) + avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) + return rgb_hex(avg) + + +def dominant_palette(image: Image.Image, count: int = 6) -> list[str]: + small = image.convert("RGB").resize((96, 96)) + colors = small.quantize(colors=count).convert("RGB").getcolors(96 * 96) or [] + colors = sorted(colors, key=lambda item: item[0], reverse=True) + return [rgb_hex(rgb) for _n, rgb in colors[:count]] + + +def _bbox_iou(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + inter = ix * iy + union = float(a["w"]) * float(a["h"]) + float(b["w"]) * float(b["h"]) - inter + return inter / max(union, 0.000001) + + +def _bbox_center(bbox: dict) -> tuple[float, float]: + return float(bbox["x"]) + float(bbox["w"]) / 2.0, float(bbox["y"]) + float(bbox["h"]) / 2.0 + + +def _bbox_center_inside(inner: dict, outer: dict) -> bool: + cx, cy = _bbox_center(inner) + return ( + float(outer["x"]) <= cx <= float(outer["x"]) + float(outer["w"]) + and float(outer["y"]) <= cy <= float(outer["y"]) + float(outer["h"]) + ) + + +def _global_to_local_bbox(bbox: dict, panel_bbox: dict) -> dict[str, float]: + panel = clamp_bbox(panel_bbox) + item = clamp_bbox(bbox) + return clamp_bbox({ + "x": (float(item["x"]) - float(panel["x"])) / max(float(panel["w"]), 0.000001), + "y": (float(item["y"]) - float(panel["y"])) / max(float(panel["h"]), 0.000001), + "w": float(item["w"]) / max(float(panel["w"]), 0.000001), + "h": float(item["h"]) / max(float(panel["h"]), 0.000001), + }) + + +def _local_to_global_bbox(bbox: dict, panel_bbox: dict) -> dict[str, float]: + panel = clamp_bbox(panel_bbox) + local = clamp_bbox(bbox) + return clamp_bbox({ + "x": float(panel["x"]) + float(local["x"]) * float(panel["w"]), + "y": float(panel["y"]) + float(local["y"]) * float(panel["h"]), + "w": float(local["w"]) * float(panel["w"]), + "h": float(local["h"]) * float(panel["h"]), + }) + + +def _safe_panel_id(value: str) -> str: + cleaned = re.sub(r"[^a-zA-Z0-9_]+", "_", str(value or "panel").strip().lower()).strip("_") + return cleaned or "panel" + + +def _is_same_frame_box(box: dict, frame: dict) -> bool: + fb = frame.get("bbox_percent") if isinstance(frame, dict) else None + if not isinstance(fb, dict): + return False + if _bbox_iou(box, fb) > 0.55: + return True + tolerance = 0.012 + return ( + abs(float(box["x"]) - float(fb["x"])) <= tolerance + and abs(float(box["y"]) - float(fb["y"])) <= tolerance + and abs(float(box["w"]) - float(fb["w"])) <= tolerance * 2 + and abs(float(box["h"]) - float(fb["h"])) <= tolerance * 2 + ) + + +def _hex_from_rgb_array(values) -> str: + try: + import numpy as np + except Exception: + return "#59AFCB" + if values is None or len(values) == 0: + return "#59AFCB" + rgb = np.median(values, axis=0).astype(int) + return rgb_hex((int(rgb[0]), int(rgb[1]), int(rgb[2]))) + + +def _dash_style_from_edge_runs(edge_crop, thickness: int) -> str: + try: + import numpy as np + except Exception: + return "solid" + if edge_crop.size == 0: + return "solid" + h, w = edge_crop.shape[:2] + bands = [ + edge_crop[:thickness, :].max(axis=0), + edge_crop[max(0, h - thickness):, :].max(axis=0), + edge_crop[:, :thickness].max(axis=1), + edge_crop[:, max(0, w - thickness):].max(axis=1), + ] + dashed_votes = 0 + for band in bands: + values = np.asarray(band) > 0 + if values.size < 16: + continue + # Close tiny gaps caused by anti-aliasing, then count meaningful on/off runs. + closed = np.convolve(values.astype(int), np.ones(3, dtype=int), mode="same") > 0 + runs = [] + current = bool(closed[0]) + length = 1 + for value in closed[1:]: + if bool(value) == current: + length += 1 + else: + runs.append((current, length)) + current = bool(value) + length = 1 + runs.append((current, length)) + on_runs = [length for is_on, length in runs if is_on and length >= 3] + off_runs = [length for is_on, length in runs if not is_on and length >= 4] + on_ratio = float(closed.mean()) + if len(on_runs) >= 3 and len(off_runs) >= 2 and 0.10 <= on_ratio <= 0.82: + dashed_votes += 1 + return "dash" if dashed_votes >= 2 else "solid" + + +def _find_frame_components(reference_path: Path, canvas_w: int, canvas_h: int) -> list[dict]: + try: + import cv2 + import numpy as np + except Exception: + return [] + image = cv2.imread(str(reference_path)) + if image is None: + return [] + rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + blur = cv2.GaussianBlur(gray, (3, 3), 0) + edges = cv2.Canny(blur, 35, 110) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) + candidate_edges = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel, iterations=1) + candidate_edges = cv2.dilate(candidate_edges, kernel, iterations=1) + contours, hierarchy = cv2.findContours(candidate_edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + if hierarchy is None: + return [] + image_area = canvas_w * canvas_h + candidates = [] + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + area = w * h + if area < image_area * 0.006 or area > image_area * 0.82: + continue + if w < 36 or h < 28: + continue + aspect = w / max(h, 1) + if aspect < 0.25 or aspect > 5.8: + continue + perimeter = cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, 0.025 * perimeter, True) + if len(approx) < 4 or len(approx) > 18: + continue + + thickness = max(3, min(14, int(min(w, h) * 0.06))) + crop_edges = edges[y:y + h, x:x + w] + crop_rgb = rgb[y:y + h, x:x + w] + if crop_edges.size == 0 or crop_rgb.size == 0: + continue + + ring_mask = np.zeros((h, w), dtype=bool) + ring_mask[:thickness, :] = True + ring_mask[max(0, h - thickness):, :] = True + ring_mask[:, :thickness] = True + ring_mask[:, max(0, w - thickness):] = True + center_mask = ~ring_mask + if not center_mask.any(): + continue + ring_edge_density = float((crop_edges[ring_mask] > 0).mean()) + center_edge_density = float((crop_edges[center_mask] > 0).mean()) + center_pixels = crop_rgb[center_mask] + ring_pixels = crop_rgb[ring_mask] + center_median = np.median(center_pixels, axis=0) + color_distance = np.linalg.norm(ring_pixels.astype(float) - center_median.astype(float), axis=1) + stroke_pixels = ring_pixels[color_distance > 18] + stroke_coverage = float(len(stroke_pixels) / max(len(ring_pixels), 1)) + if ring_edge_density < 0.045 and stroke_coverage < 0.12: + continue + if center_edge_density > max(0.30, ring_edge_density * 3.2): + continue + + bbox = clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}) + candidates.append({ + "id": f"card_{len(candidates)+1:02d}", + "semantic_role": "outer_group_boundary" if area > image_area * 0.16 else "subcard_frame", + "bbox_percent": bbox, + "shape_kind": "rounded_rect" if len(approx) > 4 else "rect", + "fill_color": _hex_from_rgb_array(center_pixels), + "fill_transparency": 1.0, + "stroke_color": _hex_from_rgb_array(stroke_pixels if len(stroke_pixels) else ring_pixels), + "color_status": "sampled_reference_pixels", + "color_source": "cv_frame_ring_pixels", + "stroke_width_pt": 1.5, + "dash_style": _dash_style_from_edge_runs(crop_edges, thickness), + "corner_radius": 0.08 if len(approx) > 4 else 0.0, + "editable_in": "pptx", + "z_index": 12, + "confidence": round(min(0.9, 0.45 + ring_edge_density + stroke_coverage), 3), + "detected_by": "cv_hollow_frame", + }) + + candidates.sort(key=lambda item: (float(item["bbox_percent"]["y"]), float(item["bbox_percent"]["x"]))) + kept: list[dict] = [] + for item in candidates: + if any(_bbox_iou(item["bbox_percent"], other["bbox_percent"]) > 0.72 for other in kept): + continue + item["id"] = f"card_{len(kept)+1:02d}" + kept.append(item) + if len(kept) >= 24: + break + return kept + + +def _find_visual_components(reference_path: Path, canvas_w: int, canvas_h: int, excluded_frames: list[dict] | None = None) -> list[dict]: + try: + import cv2 + except Exception: + return [] + image = cv2.imread(str(reference_path)) + if image is None: + return [] + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + blur = cv2.GaussianBlur(gray, (5, 5), 0) + edges = cv2.Canny(blur, 45, 120) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) + dilated = cv2.dilate(edges, kernel, iterations=2) + contours, _hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + boxes = [] + image_area = canvas_w * canvas_h + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + area = w * h + if area < image_area * 0.008 or area > image_area * 0.35: + continue + if w < 24 or h < 24: + continue + bbox = clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}) + if any(_is_same_frame_box(bbox, frame) for frame in excluded_frames or []): + continue + boxes.append((x, y, w, h)) + boxes.sort(key=lambda item: (item[1], item[0])) + kept: list[tuple[int, int, int, int]] = [] + for box in boxes: + x, y, w, h = box + duplicate = False + for kx, ky, kw, kh in kept: + ix = max(0, min(x + w, kx + kw) - max(x, kx)) + iy = max(0, min(y + h, ky + kh) - max(y, ky)) + if ix * iy / max(w * h, 1) > 0.55: + duplicate = True + break + if not duplicate: + kept.append(box) + if len(kept) >= 16: + break + slots = [] + for idx, (x, y, w, h) in enumerate(kept, start=1): + slots.append({ + "id": f"slot_{idx:02d}", + "asset_id": f"slot_{idx:02d}", + "paper_concept": f"visual element {idx}", + "display_label": "", + "bbox_percent": clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}), + "composition_type": "full_frame_icon", + "show_slot_caption": False, + "z_index": 20 + idx, + "confidence": 0.55, + "detected_by": "cv_contour", + }) + return slots + + +def _fallback_slots(count: int = 6) -> list[dict]: + boxes = [ + (0.08, 0.26, 0.16, 0.22), + (0.31, 0.26, 0.16, 0.22), + (0.54, 0.26, 0.16, 0.22), + (0.77, 0.26, 0.16, 0.22), + (0.25, 0.58, 0.18, 0.22), + (0.57, 0.58, 0.18, 0.22), + ][:count] + return [{ + "id": f"slot_{idx:02d}", + "asset_id": f"slot_{idx:02d}", + "paper_concept": f"visual element {idx}", + "display_label": "", + "bbox_percent": clamp_bbox({"x": x, "y": y, "w": w, "h": h}), + "composition_type": "full_frame_icon", + "show_slot_caption": False, + "z_index": 20 + idx, + "confidence": 0.25, + "detected_by": "fallback_grid", + } for idx, (x, y, w, h) in enumerate(boxes, start=1)] + + +def _normalize_items(items: list, prefix: str, canvas_id: str = "reference_canvas") -> list[dict]: + normalized = [] + for idx, item in enumerate(items or [], start=1): + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + item_id = str(item.get("id") or f"{prefix}_{idx:02d}") + record = dict(item) + record["id"] = item_id + if prefix == "slot": + record.setdefault("asset_id", item_id) + record.setdefault("paper_concept", item.get("prompt_subject") or item.get("label") or f"visual element {idx}") + record.setdefault("composition_type", "full_frame_icon") + record.setdefault("show_slot_caption", False) + record.setdefault("z_index", 20 + idx) + record.setdefault("panel_id", str(item.get("panel_id") or canvas_id)) + else: + record.setdefault("title", item.get("label") or item_id) + record.setdefault("editable_in", "pptx") + if prefix == "card": + record.setdefault("semantic_role", item.get("semantic_role") or "subcard_frame") + record.setdefault("shape_kind", item.get("shape_kind") or "rounded_rect") + stroke_color = str(item.get("stroke_color") or "").strip() + if stroke_color: + record["stroke_color"] = stroke_color + if str(item.get("detected_by") or "") == "cv_hollow_frame": + record.setdefault("color_status", "sampled_reference_pixels") + record.setdefault("color_source", "cv_frame_ring_pixels") + else: + record.setdefault("color_status", "vlm_reference_estimated") + record.setdefault("color_source", "vlm") + else: + record["stroke_color"] = "#59AFCB" + record.setdefault("color_status", "fallback_missing_vlm_color") + record.setdefault("color_source", "local_default") + record.setdefault("stroke_width_pt", item.get("stroke_width_pt") or 1.5) + dash_style = str(item.get("dash_style") or "solid").lower() + record["dash_style"] = "dash" if dash_style == "dashed" else dash_style + record.setdefault("fill_color", item.get("fill_color") or "#FFFFFF") + record.setdefault("fill_transparency", item.get("fill_transparency", 1.0)) + record.setdefault("corner_radius", item.get("corner_radius", 0.08)) + record.setdefault("z_index", item.get("z_index", 12)) + raw_bbox = dict(record["bbox_percent"]) + record["bbox_percent"] = clamp_bbox(record["bbox_percent"]) + try: + raw_normalized = {key: round(float(raw_bbox[key]), 4) for key in ("x", "y", "w", "h")} + except Exception: + raw_normalized = {} + record["bbox_was_clamped"] = record["bbox_percent"] != raw_normalized + record.setdefault("confidence", 0.8) + record.setdefault("detected_by", "vlm") + normalized.append(record) + return normalized + + +def _layout_item_brief(item: dict | None) -> dict | None: + if not isinstance(item, dict): + return None + return { + key: item.get(key) + for key in ( + "id", + "asset_id", + "title", + "paper_concept", + "prompt_subject", + "semantic_role", + "shape_kind", + "bbox_percent", + "panel_id", + "confidence", + "detected_by", + ) + if key in item + } + + +def _layout_disagreements(base: dict, vlm: dict) -> dict: + disagreements: list[dict] = [] + duplicates: list[dict] = [] + for collection in ("panels", "cards", "slots", "legend_regions"): + heuristic_items = [item for item in base.get(collection, []) or [] if isinstance(item, dict)] + vlm_items = [item for item in vlm.get(collection, []) or [] if isinstance(item, dict)] + matched_h: set[int] = set() + matched_v: set[int] = set() + for v_idx, vlm_item in enumerate(vlm_items): + vbox = vlm_item.get("bbox_percent") if isinstance(vlm_item.get("bbox_percent"), dict) else None + best_idx = None + best_iou = 0.0 + for h_idx, heuristic_item in enumerate(heuristic_items): + if h_idx in matched_h: + continue + hbox = heuristic_item.get("bbox_percent") if isinstance(heuristic_item.get("bbox_percent"), dict) else None + if not hbox or not vbox: + continue + iou = _bbox_iou(hbox, vbox) + if iou > best_iou: + best_idx = h_idx + best_iou = iou + if best_idx is None or best_iou < 0.25: + continue + matched_h.add(best_idx) + matched_v.add(v_idx) + kind = "duplicate" if best_iou >= 0.72 else "bbox_mismatch" + record = { + "id": f"{collection}_{kind}_{len(disagreements)+len(duplicates)+1:02d}", + "collection": collection, + "kind": kind, + "heuristic": _layout_item_brief(heuristic_items[best_idx]), + "vlm": _layout_item_brief(vlm_item), + "iou": round(best_iou, 4), + "default_decision": "keep_vlm", + } + if kind == "duplicate": + duplicates.append(record) + else: + disagreements.append(record) + for h_idx, item in enumerate(heuristic_items): + if h_idx not in matched_h: + disagreements.append({ + "id": f"{collection}_heuristic_only_{len(disagreements)+1:02d}", + "collection": collection, + "kind": "heuristic_only", + "heuristic": _layout_item_brief(item), + "vlm": None, + "default_decision": "drop_heuristic", + }) + for v_idx, item in enumerate(vlm_items): + if v_idx not in matched_v: + disagreements.append({ + "id": f"{collection}_vlm_only_{len(disagreements)+1:02d}", + "collection": collection, + "kind": "vlm_only", + "heuristic": None, + "vlm": _layout_item_brief(item), + "default_decision": "keep_vlm", + }) + return { + "summary": "Layout disagreements between heuristic/CV and VLM object detection.", + "disagreement_count": len(disagreements), + "duplicate_count": len(duplicates), + "disagreements": disagreements, + "duplicates": duplicates, + } + + +def _call_layout_adjudicator(adapter: Callable, reference: Path, base: dict, vlm: dict, disagreements: dict): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, base, vlm, disagreements) + except (TypeError, ValueError): + pass + return adapter(reference, disagreements) + + +def _layout_decision_by_id(raw: dict | list | None) -> dict[str, dict]: + decisions = raw.get("decisions", []) if isinstance(raw, dict) else raw + result = {} + for item in decisions or []: + if not isinstance(item, dict): + continue + disagreement_id = str(item.get("disagreement_id") or item.get("id") or "") + if disagreement_id: + result[disagreement_id] = item + return result + + +def _dedupe_layout_items(items: list[dict]) -> list[dict]: + kept: list[dict] = [] + used_ids: set[str] = set() + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + if any(_bbox_iou(item["bbox_percent"], existing["bbox_percent"]) > 0.72 for existing in kept): + continue + record = dict(item) + item_id = str(record.get("id") or f"item_{len(kept)+1:02d}") + if item_id in used_ids: + stem = item_id + suffix = 2 + while f"{stem}_{suffix}" in used_ids: + suffix += 1 + item_id = f"{stem}_{suffix}" + record["id"] = item_id + used_ids.add(item_id) + kept.append(record) + return kept + + +def _merge_vlm_layout( + base: dict, + vlm: dict, + reference_path: str | Path | None = None, + out_dir: str | Path | None = None, + adjudicator: Callable | None = None, + adjudication_mode: str = "off", +) -> dict: + merged = dict(base) + panels = _normalize_items(vlm.get("panels", []), "panel") + cards = _normalize_items(vlm.get("cards", []), "card") + slots = _normalize_items(vlm.get("slots", []), "slot") + legends = _normalize_items(vlm.get("legend_regions", []), "legend") + vlm_normalized = {"panels": panels, "cards": cards, "slots": slots, "legend_regions": legends} + disagreements = _layout_disagreements(base, vlm_normalized) + adjudication_doc = { + "summary": "Layout disagreement adjudication report.", + "mode": adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + } + if out_dir: + out = Path(out_dir) + write_json(out / "reference_layout_disagreements.json", disagreements) + decisions: dict[str, dict] = {} + if adjudication_mode == "vlm" and adjudicator and disagreements["disagreement_count"] and reference_path: + try: + raw_adjudication = _call_layout_adjudicator(adjudicator, Path(reference_path), base, vlm_normalized, disagreements) + decisions = _layout_decision_by_id(raw_adjudication) + adjudication_doc.update({"status": "used", "decision_count": len(decisions), "raw_adjudication": raw_adjudication}) + except Exception as exc: + adjudication_doc.update({"status": "fallback", "fallback_reason": f"layout_adjudication_failed:{exc}"}) + merged.setdefault("warnings", []).append(str(adjudication_doc["fallback_reason"])) + elif adjudication_mode == "vlm" and disagreements["disagreement_count"]: + adjudication_doc["status"] = "unavailable" + if adjudication_mode == "vlm" and decisions: + selected: dict[str, list[dict]] = {"panels": [], "cards": [], "slots": [], "legend_regions": []} + for duplicate in disagreements["duplicates"]: + collection = str(duplicate.get("collection")) + item = duplicate.get("vlm") or duplicate.get("heuristic") + if collection in selected and item: + selected[collection].append(item) + for item in disagreements["disagreements"]: + collection = str(item.get("collection")) + if collection not in selected: + continue + decision = decisions.get(str(item.get("id") or ""), {}) + action = str(decision.get("decision") or item.get("default_decision") or "").lower() + if action == "keep_heuristic": + candidates = [item.get("heuristic")] + elif action == "keep_both": + candidates = [item.get("vlm"), item.get("heuristic")] + elif action == "drop_both": + candidates = [] + elif action == "merge_or_adjust": + candidates = [decision.get("merged_item") or decision.get("item") or item.get("vlm") or item.get("heuristic")] + elif action == "keep_vlm": + candidates = [item.get("vlm")] + else: + candidates = [item.get("vlm") or item.get("heuristic")] + selected[collection].extend([candidate for candidate in candidates if isinstance(candidate, dict)]) + panels = _normalize_items(selected["panels"], "panel") + cards = _normalize_items(selected["cards"], "card") + slots = _normalize_items(selected["slots"], "slot") + legends = _normalize_items(selected["legend_regions"], "legend") + if panels: + merged["panels"] = _dedupe_layout_items(panels) + if slots: + merged["slots"] = _dedupe_layout_items(slots) + if cards: + merged["cards"] = _dedupe_layout_items(cards) + merged["legend_regions"] = _dedupe_layout_items(legends) + merged["confidence"] = float(vlm.get("confidence") or 0.75) + merged["vlm_status"] = "used" + if vlm.get("_vlm_model"): + merged["vlm_model"] = str(vlm.get("_vlm_model")) + merged["layout_adjudication_mode"] = adjudication_mode + merged["layout_adjudication_status"] = adjudication_doc.get("status") + merged["layout_disagreement_count"] = disagreements.get("disagreement_count", 0) + merged["layout_duplicate_count"] = disagreements.get("duplicate_count", 0) + merged.setdefault("warnings", []) + if out_dir: + write_json(Path(out_dir) / "reference_layout_adjudication.json", adjudication_doc) + return merged + + +def _design_layers(design_plan: dict | None) -> list[dict]: + if not isinstance(design_plan, dict): + return [] + if isinstance(design_plan.get("layers"), list): + return [item for item in design_plan["layers"] if isinstance(item, dict)] + layer_plan = design_plan.get("layer_plan") if isinstance(design_plan.get("layer_plan"), dict) else {} + return [item for item in layer_plan.get("layers", []) or [] if isinstance(item, dict)] + + +def _apply_design_seed(layout: dict, design_plan: dict | None) -> dict: + layers = _design_layers(design_plan) + if not layers: + layout["design_seed_status"] = "not_available" + return layout + panels = _normalize_items([item for item in layers if str(item.get("kind") or "") == "panel"], "panel") + cards = _normalize_items([item for item in layers if str(item.get("kind") or "") == "card"], "card") + slots = _normalize_items([item for item in layers if str(item.get("kind") or "") == "visual_slot"], "slot") + legends = _normalize_items([item for item in layers if str(item.get("kind") or "") == "legend"], "legend") + ignored_count = sum(1 for item in layers if str(item.get("kind") or "") in {"background", "text", "connector", "ignore"}) + if panels: + layout["panels"] = panels + if cards: + layout["cards"] = cards + if slots: + layout["slots"] = slots + if legends: + layout["legend_regions"] = legends + layout["design_seed_status"] = "used" if panels or cards or slots or legends else "no_layout_layers" + layout["design_seed_layer_count"] = len(layers) + layout["design_seed_ignored_layer_count"] = ignored_count + layout["design_seed_slot_count"] = len(slots) + layout["design_seed_card_count"] = len(cards) + layout["design_seed_panel_count"] = len(panels) + layout.setdefault("warnings", []) + if ignored_count: + layout["warnings"].append(f"{ignored_count}_non_layout_design_layer(s)_ignored") + return layout + + +def _draw_overlay(reference_path: Path, out_path: Path, layout: dict) -> None: + with Image.open(reference_path).convert("RGB") as image: + draw = ImageDraw.Draw(image) + w, h = image.size + for collection, color, width in [("panels", "#2D6FB7", 4), ("cards", "#8A5CF6", 3), ("slots", "#E17721", 3), ("legend_regions", "#4B9B52", 3)]: + for item in layout.get(collection, []): + box = item.get("bbox_percent") + if not isinstance(box, dict): + continue + x0 = int(float(box["x"]) * w) + y0 = int(float(box["y"]) * h) + x1 = int((float(box["x"]) + float(box["w"])) * w) + y1 = int((float(box["y"]) + float(box["h"])) * h) + draw.rectangle((x0, y0, x1, y1), outline=color, width=width) + draw.text((x0 + 3, y0 + 3), str(item.get("id") or ""), fill=color) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.save(out_path) + + +def _real_layout_panels(layout: dict) -> list[dict]: + panels = [] + for panel in layout.get("panels", []) or []: + if not isinstance(panel, dict) or not isinstance(panel.get("bbox_percent"), dict): + continue + if str(panel.get("id") or "") == "reference_canvas" and str(panel.get("detected_by") or "") == "default_canvas": + continue + bbox = clamp_bbox(panel["bbox_percent"]) + if float(bbox["w"]) * float(bbox["h"]) < PANELWISE_MIN_PANEL_AREA: + continue + record = dict(panel) + record["bbox_percent"] = bbox + panels.append(record) + return panels + + +def _select_panelwise_panels(layout: dict, max_panels: int = PANELWISE_LAYOUT_MAX_PANELS) -> tuple[list[dict], list[dict]]: + panels = _real_layout_panels(layout) + panels.sort(key=lambda item: ( + float(item["bbox_percent"]["y"]), + float(item["bbox_percent"]["x"]), + -float(item["bbox_percent"]["w"]) * float(item["bbox_percent"]["h"]), + )) + if len(panels) <= max_panels: + return panels, [] + ranked = sorted(panels, key=lambda item: float(item["bbox_percent"]["w"]) * float(item["bbox_percent"]["h"]), reverse=True) + selected_ids = {str(item.get("id")) for item in ranked[:max_panels]} + selected = [item for item in panels if str(item.get("id")) in selected_ids] + skipped = [item for item in panels if str(item.get("id")) not in selected_ids] + return selected, skipped + + +def _items_inside_panel(layout: dict, collection: str, panel: dict, *, local: bool = False) -> list[dict]: + panel_bbox = panel.get("bbox_percent") + if not isinstance(panel_bbox, dict): + return [] + result = [] + for item in layout.get(collection, []) or []: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + bbox = clamp_bbox(item["bbox_percent"]) + if str(item.get("panel_id") or "") == str(panel.get("id") or "") or _bbox_center_inside(bbox, panel_bbox): + record = dict(item) + record["panel_id"] = str(panel.get("id") or record.get("panel_id") or "reference_canvas") + record["bbox_percent"] = _global_to_local_bbox(bbox, panel_bbox) if local else bbox + result.append(record) + return result + + +def _panel_seed_layout(layout: dict, panel: dict) -> dict: + return { + "panel": _layout_item_brief(panel), + "cards": _items_inside_panel(layout, "cards", panel, local=True), + "slots": _items_inside_panel(layout, "slots", panel, local=True), + "legend_regions": _items_inside_panel(layout, "legend_regions", panel, local=True), + } + + +def _panel_fallback_layout(layout: dict, panel: dict) -> dict[str, list[dict]]: + return { + "cards": _items_inside_panel(layout, "cards", panel), + "slots": _items_inside_panel(layout, "slots", panel), + "legend_regions": _items_inside_panel(layout, "legend_regions", panel), + } + + +def _panel_crop_path(out: Path, panel: dict, index: int) -> Path: + panel_id = _safe_panel_id(str(panel.get("id") or f"panel_{index:02d}")) + return out / "panel_layout_crops" / f"{index:02d}_{panel_id}.png" + + +def _save_panel_crop(reference: Path, out: Path, panel: dict, index: int) -> Path: + crop_path = _panel_crop_path(out, panel, index) + with Image.open(reference).convert("RGB") as image: + w, h = image.size + bbox = clamp_bbox(panel["bbox_percent"]) + left = max(0, min(w - 1, int(float(bbox["x"]) * w))) + top = max(0, min(h - 1, int(float(bbox["y"]) * h))) + right = max(left + 1, min(w, int((float(bbox["x"]) + float(bbox["w"])) * w))) + bottom = max(top + 1, min(h, int((float(bbox["y"]) + float(bbox["h"])) * h))) + crop_path.parent.mkdir(parents=True, exist_ok=True) + image.crop((left, top, right, bottom)).save(crop_path) + return crop_path + + +def _panelwise_item_id(panel: dict, item: dict, prefix: str, index: int) -> str: + panel_id = _safe_panel_id(str(panel.get("id") or "panel")) + raw_id = _safe_panel_id(str(item.get("id") or f"{prefix}_{index:02d}")) + if raw_id.startswith(f"{panel_id}_"): + return raw_id + return f"{panel_id}_{raw_id}" + + +def _normalize_panelwise_items(items: list, prefix: str, panel: dict) -> list[dict]: + normalized = [] + panel_id = str(panel.get("id") or "reference_canvas") + for idx, item in enumerate(items or [], start=1): + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + record = dict(item) + original_id = str(record.get("id") or f"{prefix}_{idx:02d}") + record["id"] = _panelwise_item_id(panel, record, prefix, idx) + record["bbox_percent"] = _local_to_global_bbox(record["bbox_percent"], panel["bbox_percent"]) + record["panel_id"] = panel_id + record["detected_by"] = str(record.get("detected_by") or "panel_layout_vlm") + if prefix == "slot": + if not record.get("asset_id") or str(record.get("asset_id")) == original_id: + record["asset_id"] = record["id"] + record.setdefault("paper_concept", record.get("prompt_subject") or record.get("label") or record["id"]) + record.setdefault("composition_type", "full_frame_icon") + record.setdefault("show_slot_caption", False) + normalized.append(record) + return _normalize_items(normalized, prefix) + + +def _panel_count_record(panel: dict, status: str, crop_path: Path | None, objects: dict[str, list[dict]], fallback_reason: str | None = None) -> dict: + record = { + "id": panel.get("id"), + "title": panel.get("title"), + "bbox_percent": panel.get("bbox_percent"), + "crop_path": str(crop_path) if crop_path else None, + "status": status, + "card_count": len(objects.get("cards", [])), + "slot_count": len(objects.get("slots", [])), + "legend_region_count": len(objects.get("legend_regions", [])), + } + if fallback_reason: + record["fallback_reason"] = fallback_reason + return record + + +def _write_panel_layout_artifacts(reference: Path, out: Path, layout: dict, plan_doc: dict | None = None) -> None: + doc = plan_doc or { + "summary": "Panel-wise layout planning skipped.", + "status": "skipped", + "panel_count": 0, + "processed_panel_count": 0, + "fallback_panel_count": 0, + "panels": [], + } + write_json(out / "reference_panel_layout_plan.json", doc) + _draw_overlay(reference, out / "reference_panel_layout_overlay.png", layout) + + +def _outside_panelwise_panels(layout: dict, panels: list[dict], collection: str) -> list[dict]: + result = [] + panel_boxes = [panel["bbox_percent"] for panel in panels if isinstance(panel.get("bbox_percent"), dict)] + for item in layout.get(collection, []) or []: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): + continue + bbox = clamp_bbox(item["bbox_percent"]) + if any(_bbox_center_inside(bbox, panel_box) for panel_box in panel_boxes): + continue + result.append(dict(item)) + return result + + +def _run_panelwise_layout( + reference: Path, + out: Path, + layout: dict, + panel_layout_adapter: Callable[[str | Path, dict, dict], dict] | None, + *, + max_panels: int = PANELWISE_LAYOUT_MAX_PANELS, + max_workers: int = PANELWISE_LAYOUT_MAX_WORKERS, +) -> dict | None: + selected, skipped = _select_panelwise_panels(layout, max_panels=max_panels) + if not selected and not skipped: + _write_panel_layout_artifacts(reference, out, layout) + return None + + all_panels = selected + skipped + panel_entries: list[dict] = [] + panel_objects: dict[str, list[dict]] = {"cards": [], "slots": [], "legend_regions": []} + + def process_panel(index_panel: tuple[int, dict]) -> tuple[int, dict, dict[str, list[dict]], str, str | None, Path | None, str | None]: + index, panel = index_panel + crop_path = _save_panel_crop(reference, out, panel, index) + if not panel_layout_adapter: + fallback = _panel_fallback_layout(layout, panel) + return index, panel, fallback, "fallback", "panel_layout_adapter_unavailable", crop_path, None + try: + seed = _panel_seed_layout(layout, panel) + raw = panel_layout_adapter(crop_path, panel, seed) + objects = { + "cards": _normalize_panelwise_items(raw.get("cards", []) if isinstance(raw, dict) else [], "card", panel), + "slots": _normalize_panelwise_items(raw.get("slots", []) if isinstance(raw, dict) else [], "slot", panel), + "legend_regions": _normalize_panelwise_items(raw.get("legend_regions", []) if isinstance(raw, dict) else [], "legend", panel), + } + model = str(raw.get("_vlm_model")) if isinstance(raw, dict) and raw.get("_vlm_model") else None + return index, panel, objects, "used", None, crop_path, model + except Exception as exc: + fallback = _panel_fallback_layout(layout, panel) + return index, panel, fallback, "fallback", f"panel_layout_failed:{exc}", crop_path, None + + results = [] + worker_count = max(1, min(max_workers, len(selected))) + if selected: + with ThreadPoolExecutor(max_workers=worker_count) as executor: + futures = [executor.submit(process_panel, item) for item in enumerate(selected, start=1)] + for future in as_completed(futures): + results.append(future.result()) + results.sort(key=lambda item: item[0]) + + models = [] + for _index, panel, objects, status, fallback_reason, crop_path, model in results: + for collection in ("cards", "slots", "legend_regions"): + panel_objects[collection].extend(objects.get(collection, [])) + panel_entries.append(_panel_count_record(panel, status, crop_path, objects, fallback_reason)) + if model and model not in models: + models.append(model) + + for panel in skipped: + fallback = _panel_fallback_layout(layout, panel) + for collection in ("cards", "slots", "legend_regions"): + panel_objects[collection].extend(fallback.get(collection, [])) + panel_entries.append(_panel_count_record(panel, "fallback", None, fallback, "panel_limit_exceeded")) + + merged = dict(layout) + merged["panels"] = layout.get("panels", []) + for collection in ("cards", "slots", "legend_regions"): + merged[collection] = _dedupe_layout_items(_outside_panelwise_panels(layout, all_panels, collection) + panel_objects[collection]) + merged["panel_layout_status"] = "used" if panel_layout_adapter else "fallback" + merged["panel_layout_processed_count"] = len(selected) + merged["panel_layout_fallback_count"] = sum(1 for item in panel_entries if item.get("status") == "fallback") + merged["layout_detection_strategy"] = "panelwise" + if models: + merged["vlm_model"] = ",".join(models) + plan_doc = { + "summary": "Panel-wise layout planning report.", + "status": merged["panel_layout_status"], + "max_panels": max_panels, + "max_workers": max_workers, + "panel_count": len(all_panels), + "processed_panel_count": len(selected), + "fallback_panel_count": merged["panel_layout_fallback_count"], + "panels": panel_entries, + } + _write_panel_layout_artifacts(reference, out, merged, plan_doc) + return merged + + +def plan_reference_layout( + reference_path: str | Path, + out_dir: str | Path, + mode: str = "hybrid", + vlm_adapter: Callable[[str | Path, dict], dict] | None = None, + panel_layout_adapter: Callable[[str | Path, dict, dict], dict] | None = None, + design_plan: dict | None = None, + layout_adjudication_mode: str = "off", + layout_adjudicator: Callable | None = None, +) -> dict: + reference = Path(reference_path) + out = Path(out_dir) + with Image.open(reference).convert("RGB") as image: + canvas_w, canvas_h = image.size + width_in, height_in = canvas_inches(canvas_w, canvas_h) + background = estimate_background(image) + palette = dominant_palette(image) + cards = _find_frame_components(reference, canvas_w, canvas_h) + slots = _find_visual_components(reference, canvas_w, canvas_h, excluded_frames=cards) or _fallback_slots() + panel = { + "id": "reference_canvas", + "title": "Editable Rebuild", + "bbox_percent": clamp_bbox({"x": 0.025, "y": 0.065, "w": 0.95, "h": 0.84}), + "editable_in": "pptx", + "confidence": 0.5, + "detected_by": "default_canvas", + } + for slot in slots: + slot["panel_id"] = panel["id"] + layout = { + "summary": "Reference geometry inferred by editable rebuild layout planner.", + "layout_mode": mode, + "status": "pass", + "canvas": {"width_px": canvas_w, "height_px": canvas_h, "width_in": width_in, "height_in": height_in, "background": background}, + "panels": [panel], + "cards": cards, + "slots": slots, + "legend_regions": [], + "palette": palette, + "confidence": 0.55 if slots else 0.25, + "vlm_status": "not_requested", + "warnings": [], + } + layout = _apply_design_seed(layout, design_plan) + write_json(out / "reference_layout_disagreements.json", { + "summary": "Layout disagreement detection skipped.", + "disagreement_count": 0, + "duplicate_count": 0, + "disagreements": [], + "duplicates": [], + }) + write_json(out / "reference_layout_adjudication.json", { + "summary": "Layout disagreement adjudication skipped.", + "mode": layout_adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": 0, + "input_duplicate_count": 0, + }) + _write_panel_layout_artifacts(reference, out, layout) + if mode in {"vlm", "hybrid"}: + panelwise_layout = _run_panelwise_layout(reference, out, layout, panel_layout_adapter) if panel_layout_adapter else None + if panelwise_layout is not None: + layout = panelwise_layout + layout["confidence"] = max(float(layout.get("confidence") or 0.0), 0.75) + layout["vlm_status"] = "used" + elif vlm_adapter: + try: + vlm_layout = vlm_adapter(reference, layout) + if isinstance(vlm_layout, dict): + layout = _merge_vlm_layout( + layout, + vlm_layout, + reference_path=reference, + out_dir=out, + adjudicator=layout_adjudicator, + adjudication_mode=layout_adjudication_mode, + ) + except Exception as exc: + layout["warnings"].append(f"vlm_layout_failed:{exc}") + layout["vlm_status"] = "fallback" + else: + layout["vlm_status"] = "unavailable_fallback_to_heuristic" + if mode == "vlm": + layout["warnings"].append("layout_mode_vlm_requested_without_adapter") + layout.setdefault("layout_adjudication_mode", layout_adjudication_mode) + layout.setdefault("layout_adjudication_status", "skipped") + layout.setdefault("layout_disagreement_count", 0) + layout.setdefault("layout_duplicate_count", 0) + _draw_overlay(reference, out / "reference_geometry_overlay.png", layout) + return layout diff --git a/rfs/analysis/rebuild/reference_text.py b/rfs/analysis/rebuild/reference_text.py new file mode 100644 index 0000000..990a122 --- /dev/null +++ b/rfs/analysis/rebuild/reference_text.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +import math +import os +import threading +from pathlib import Path +from typing import Any, Callable + +from PIL import Image + + +OCR_LOW_CONFIDENCE = 0.65 +_EASYOCR_READERS: dict[tuple[str, ...], Any] = {} +_RAPIDOCR_LOCAL = threading.local() + + +def _positive_ocr_setting(explicit: int | None, env_name: str, default: int) -> int: + try: + return max(1, int(explicit if explicit is not None else os.getenv(env_name) or default)) + except (TypeError, ValueError): + return max(1, int(default)) + + +def _rapidocr_detector_limit(explicit: int | None = None) -> int: + """Return a bounded detector resize limit suitable for RapidOCR.""" + value = _positive_ocr_setting(explicit, "RFS_RAPIDOCR_DET_LIMIT", 512) + return max(256, min(2048, value)) + + +def _round4(value: float) -> float: + return round(float(value), 4) + + +def _clamp_bbox(bbox: dict[str, float]) -> dict[str, float]: + x = max(0.0, min(0.995, float(bbox["x"]))) + y = max(0.0, min(0.995, float(bbox["y"]))) + w = max(0.001, min(float(bbox["w"]), 1.0 - x)) + h = max(0.001, min(float(bbox["h"]), 1.0 - y)) + return {"x": _round4(x), "y": _round4(y), "w": _round4(w), "h": _round4(h)} + + +def _bbox_center(bbox: dict[str, float]) -> dict[str, float]: + return {"x": _round4(float(bbox["x"]) + float(bbox["w"]) / 2), "y": _round4(float(bbox["y"]) + float(bbox["h"]) / 2)} + + +def _luminance(rgb: tuple[int, int, int]) -> float: + return 0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2] + + +def _hex(rgb: tuple[int, int, int]) -> str: + return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" + + +def _contrast(a: tuple[int, int, int], b: tuple[int, int, int]) -> float: + return abs(_luminance(a) - _luminance(b)) + + +def _sample_text_color(image: Image.Image, bbox: dict[str, float], fallback: str = "#263747") -> tuple[str, str]: + try: + rgb_image = image.convert("RGB") + iw, ih = rgb_image.size + pad_x = max(1, int(float(bbox["w"]) * iw * 0.04)) + pad_y = max(1, int(float(bbox["h"]) * ih * 0.10)) + x0 = max(0, min(iw - 1, int(float(bbox["x"]) * iw) - pad_x)) + y0 = max(0, min(ih - 1, int(float(bbox["y"]) * ih) - pad_y)) + x1 = max(x0 + 1, min(iw, int((float(bbox["x"]) + float(bbox["w"])) * iw) + pad_x)) + y1 = max(y0 + 1, min(ih, int((float(bbox["y"]) + float(bbox["h"])) * ih) + pad_y)) + crop = rgb_image.crop((x0, y0, x1, y1)) + pixels = list(crop.get_flattened_data() if hasattr(crop, "get_flattened_data") else crop.getdata()) + except Exception: + return fallback, "sample_failed" + if not pixels: + return fallback, "empty_crop" + pixels = sorted(pixels, key=_luminance) + dark = pixels[: max(1, len(pixels) // 5)] + light = pixels[-max(1, len(pixels) // 5):] + dark_avg = tuple(int(sum(px[i] for px in dark) / len(dark)) for i in range(3)) + light_avg = tuple(int(sum(px[i] for px in light) / len(light)) for i in range(3)) + if _contrast(dark_avg, light_avg) < 32: + return fallback, "low_contrast_fallback" + text_rgb = dark_avg if _luminance(dark_avg) < _luminance(light_avg) else light_avg + return _hex(text_rgb), "sampled_foreground" + + +def _bbox_overlap(a: dict[str, float], b: dict[str, float]) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + area = ix * iy + return area / max(float(a["w"]) * float(a["h"]), 0.000001) + + +def _nearest_target(bbox: dict[str, float], program: dict) -> tuple[str, str]: + candidates: list[tuple[float, str, str]] = [] + for panel in program.get("panels", []): + if isinstance(panel, dict) and isinstance(panel.get("bbox_percent"), dict): + overlap = _bbox_overlap(bbox, panel["bbox_percent"]) + if overlap > 0: + candidates.append((overlap, "panel_title", str(panel.get("id") or ""))) + for slot in program.get("slots", []): + if isinstance(slot, dict) and isinstance(slot.get("bbox_percent"), dict): + overlap = _bbox_overlap(bbox, slot["bbox_percent"]) + if overlap > 0: + candidates.append((overlap, "slot_caption", str(slot.get("id") or ""))) + if candidates: + _overlap, role, target_id = max(candidates, key=lambda item: item[0]) + return role, target_id + return "free_text", "canvas" + + +def _font_family_guess(text: str) -> str: + if any(ord(char) > 127 for char in text): + return "Microsoft YaHei" + return "Arial" + + +def _normalize_quad(quad: Any) -> list[list[float]] | None: + if hasattr(quad, "tolist"): + quad = quad.tolist() + if not isinstance(quad, (list, tuple)) or len(quad) < 4: + return None + points: list[list[float]] = [] + for point in quad[:4]: + if not isinstance(point, (list, tuple)) or len(point) < 2: + return None + points.append([float(point[0]), float(point[1])]) + return points + + +def _flatten_paddle_result(result: Any) -> list[Any]: + if not isinstance(result, list): + return [] + if result and all(isinstance(item, list) and len(item) >= 2 and _normalize_quad(item[0]) for item in result): + return result + flattened: list[Any] = [] + for page in result: + if isinstance(page, list): + flattened.extend(_flatten_paddle_result(page)) + return flattened + + +def _mapping_value(mapping: dict[str, Any], keys: tuple[str, ...]) -> Any: + for key in keys: + if key in mapping: + return mapping[key] + return None + + +def _parse_paddle_mapping(mapping: dict[str, Any]) -> list[dict[str, Any]]: + payload = mapping.get("res") if isinstance(mapping.get("res"), dict) else mapping + texts = _mapping_value(payload, ("rec_texts", "texts", "text")) + quads = _mapping_value(payload, ("rec_polys", "dt_polys", "det_polys", "polys", "boxes")) + scores = _mapping_value(payload, ("rec_scores", "scores", "confidences")) + if hasattr(texts, "tolist"): + texts = texts.tolist() + if hasattr(quads, "tolist"): + quads = quads.tolist() + if hasattr(scores, "tolist"): + scores = scores.tolist() + if isinstance(texts, str): + texts = [texts] + if not isinstance(texts, list) or not isinstance(quads, list): + return [] + records = [] + score_values = scores if isinstance(scores, list) else [] + for index, text in enumerate(texts): + if index >= len(quads): + break + quad = _normalize_quad(quads[index]) + raw_text = str(text or "").strip() + if not quad or not raw_text: + continue + confidence = 0.0 + if index < len(score_values): + try: + confidence = float(score_values[index]) + except Exception: + confidence = 0.0 + records.append({"text": raw_text, "confidence": confidence, "quad": quad}) + return records + + +def _parse_ocr_result(result: Any) -> list[dict[str, Any]]: + records = [] + if isinstance(result, dict): + return _parse_paddle_mapping(result) + if isinstance(result, list): + for page in result: + value = page + if hasattr(value, "json"): + value = value.json() if callable(value.json) else value.json + if isinstance(value, dict): + records.extend(_parse_paddle_mapping(value)) + if records: + return records + for item in _flatten_paddle_result(result): + if not isinstance(item, (list, tuple)) or len(item) < 2: + continue + quad = _normalize_quad(item[0]) + payload = item[1] + if not quad: + continue + text = "" + confidence = 0.0 + if isinstance(payload, (list, tuple)) and payload: + text = str(payload[0] or "").strip() + if len(payload) > 1: + try: + confidence = float(payload[1]) + except Exception: + confidence = 0.0 + else: + text = str(payload or "").strip() + if text: + records.append({"text": text, "confidence": confidence, "quad": quad}) + return records + + +def run_paddle_ocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: + from paddleocr import PaddleOCR # type: ignore + + lang_map = {"en_ch": "ch", "ch": "ch", "en": "en"} + mapped_lang = lang_map.get(str(lang), "ch") + init_options = [ + {"use_angle_cls": True, "lang": mapped_lang, "show_log": False}, + {"use_angle_cls": True, "lang": mapped_lang}, + {"use_textline_orientation": True, "lang": mapped_lang}, + {"lang": mapped_lang}, + ] + last_error: Exception | None = None + for options in init_options: + try: + ocr = PaddleOCR(**options) + break + except Exception as exc: + last_error = exc + else: + assert last_error is not None + raise last_error + try: + result = ocr.ocr(str(image_path), cls=True) + except Exception: + result = ocr.ocr(str(image_path)) + return _parse_ocr_result(result) + + +def run_easyocr(image_path: str | Path, lang: str) -> list[dict[str, Any]]: + import easyocr # type: ignore + + lang_map = {"en_ch": ["ch_sim", "en"], "ch": ["ch_sim"], "en": ["en"]} + languages = tuple(lang_map.get(str(lang), ["ch_sim", "en"])) + reader = _EASYOCR_READERS.get(languages) + if reader is None: + allow_download = str(os.getenv("RFS_OCR_ALLOW_DOWNLOAD") or "").strip().casefold() in {"1", "true", "yes", "on"} + try: + reader = easyocr.Reader(list(languages), gpu=False, download_enabled=allow_download, verbose=False) + except Exception as exc: + hint = " Set RFS_OCR_ALLOW_DOWNLOAD=1 for an explicit one-time model download." if not allow_download else "" + raise RuntimeError(f"EasyOCR model is not ready for languages {list(languages)}.{hint} Original error: {exc}") from exc + _EASYOCR_READERS[languages] = reader + records = [] + for quad, text, confidence in reader.readtext(str(image_path)): + points = _normalize_quad(quad) + if points and str(text).strip(): + records.append({"text": str(text).strip(), "confidence": float(confidence), "quad": points}) + return records + + +def run_rapidocr_detailed( + image_path: str | Path, + lang: str, + *, + threads: int | None = None, + batch_size: int | None = None, + detector_limit: int | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + del lang + thread_count = _positive_ocr_setting(threads, "RFS_RAPIDOCR_THREADS", 1) + recognition_batch = _positive_ocr_setting(batch_size, "RFS_RAPIDOCR_BATCH", 6) + detector_side_limit = _rapidocr_detector_limit(detector_limit) + engines = getattr(_RAPIDOCR_LOCAL, "engines", None) + if engines is None: + engines = {} + _RAPIDOCR_LOCAL.engines = engines + engine_key = (thread_count, recognition_batch, detector_side_limit) + if engine_key not in engines: + from rapidocr_onnxruntime import RapidOCR # type: ignore + + engines[engine_key] = RapidOCR( + intra_op_num_threads=thread_count, + inter_op_num_threads=1, + det_limit_side_len=detector_side_limit, + rec_batch_num=recognition_batch, + ) + result, timings = engines[engine_key](str(image_path)) + records = [] + for item in result or []: + if not isinstance(item, (list, tuple)) or len(item) < 3: + continue + quad = _normalize_quad(item[0]) + text = str(item[1] or "").strip() + if quad and text: + records.append({"text": text, "confidence": float(item[2] or 0.0), "quad": quad}) + timing_values = [float(value or 0.0) for value in (timings or [])] + while len(timing_values) < 3: + timing_values.append(0.0) + diagnostics = { + "detector_limit": detector_side_limit, + "thread_count": thread_count, + "recognition_batch": recognition_batch, + "detection_seconds": round(timing_values[0], 4), + "classification_seconds": round(timing_values[1], 4), + "recognition_seconds": round(timing_values[2], 4), + "inference_seconds": round(sum(timing_values[:3]), 4), + "region_count": len(records), + } + return records, diagnostics + + +def run_rapidocr(image_path: str | Path, lang: str, *, threads: int | None = None, batch_size: int | None = None, detector_limit: int | None = None) -> list[dict[str, Any]]: + records, _diagnostics = run_rapidocr_detailed( + image_path, + lang, + threads=threads, + batch_size=batch_size, + detector_limit=detector_limit, + ) + return records + + +def _record_to_region(record: dict[str, Any], image: Image.Image, program: dict, index: int, canvas_height_in: float, engine: str) -> dict: + width, height = image.size + xs = [point[0] for point in record["quad"]] + ys = [point[1] for point in record["quad"]] + bbox = _clamp_bbox({ + "x": min(xs) / max(width, 1), + "y": min(ys) / max(height, 1), + "w": (max(xs) - min(xs)) / max(width, 1), + "h": (max(ys) - min(ys)) / max(height, 1), + }) + text = str(record.get("text") or "").strip() + confidence = float(record.get("confidence") or 0.0) + color_hex, color_status = _sample_text_color(image, bbox) + role, target_id = _nearest_target(bbox, program) + font_size = max(1.0, float(bbox["h"]) * canvas_height_in * 72 * 0.72) + return { + "id": f"ref_text_ocr_{index:03d}", + "text": text, + "raw_text": text, + "role": role, + "target_id": target_id, + "bbox_percent": bbox, + "line_bbox_percent": bbox, + "word_bbox_percent": [bbox], + "center_percent": _bbox_center(bbox), + "width_percent": _round4(bbox["w"]), + "height_percent": _round4(bbox["h"]), + "estimated_font_ratio": _round4(float(bbox["h"]) * 0.72), + "font_size_pt": round(font_size, 2), + "font_family_guess": _font_family_guess(text), + "font_weight_guess": "bold" if role == "panel_title" and float(bbox["h"]) > 0.025 else "regular", + "color_hex": color_hex, + "color_sampling_status": color_status, + "confidence": round(confidence, 4), + "ocr_engine": engine, + "source": "reference_ocr_text_region", + "editable_in": "pptx", + } + + +def extract_reference_text( + reference_path: str | Path, + program: dict, + *, + mode: str = "ocr", + engine: str = "easyocr", + lang: str = "en_ch", + ocr_adapter: Callable[[str | Path, str], list[dict[str, Any]]] | None = None, +) -> tuple[list[dict], dict]: + requested_mode = str(mode or "ocr").lower() + requested_engine = str(engine or "easyocr").lower() + report = { + "summary": "OCR text extraction quality report.", + "mode": requested_mode, + "ocr_engine": requested_engine, + "ocr_engine_effective": requested_engine, + "ocr_lang": lang, + "status": "not_run", + "text_region_count": 0, + "low_confidence_count": 0, + "warnings": [], + "fallback_reason": None, + } + if requested_mode != "ocr" or requested_engine == "off": + report.update({"status": "fallback", "fallback_reason": "ocr_disabled"}) + return [], report + effective_engine = requested_engine + try: + if ocr_adapter: + raw_records = ocr_adapter(reference_path, lang) + elif requested_engine == "rapidocr": + raw_records = run_rapidocr(reference_path, lang) + elif requested_engine == "easyocr": + raw_records = run_easyocr(reference_path, lang) + else: + try: + raw_records = run_paddle_ocr(reference_path, lang) + except Exception as paddle_exc: + try: + raw_records = run_easyocr(reference_path, lang) + effective_engine = "easyocr" + report["ocr_engine_effective"] = effective_engine + report["warnings"].append(f"paddle_unavailable_used_easyocr:{paddle_exc}") + report["fallback_reason"] = f"paddle_unavailable_used_easyocr:{paddle_exc}" + except Exception as easy_exc: + raise RuntimeError(f"paddle_failed:{paddle_exc}; easyocr_failed:{easy_exc}") from easy_exc + except Exception as exc: + report.update({"status": "fallback", "fallback_reason": f"ocr_unavailable:{exc}"}) + report["warnings"].append(str(exc)) + return [], report + try: + with Image.open(reference_path) as image: + rgb_image = image.convert("RGB") + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + canvas_height_in = float(canvas.get("height_in") or 7.5) + regions = [_record_to_region(record, rgb_image, program, index, canvas_height_in, effective_engine) for index, record in enumerate(raw_records, start=1)] + except Exception as exc: + report.update({"status": "fallback", "fallback_reason": f"ocr_postprocess_failed:{exc}"}) + report["warnings"].append(str(exc)) + return [], report + regions = [region for region in regions if str(region.get("text", "")).strip()] + low_conf = [region for region in regions if float(region.get("confidence") or 0.0) < OCR_LOW_CONFIDENCE] + overlap_warnings = [] + for region in regions: + if region.get("role") == "slot_caption": + overlap_warnings.append(region["id"]) + if not regions: + report.update({"status": "fallback", "fallback_reason": "ocr_returned_no_text", "text_region_count": 0}) + return [], report + report.update({ + "status": "pass", + "ocr_engine_effective": effective_engine, + "text_region_count": len(regions), + "low_confidence_count": len(low_conf), + "low_confidence_text_ids": [region["id"] for region in low_conf], + "slot_overlap_text_ids": overlap_warnings, + "warnings": report["warnings"] + ([f"{len(low_conf)} low-confidence OCR text region(s)"] if low_conf else []), + }) + return regions, report diff --git a/rfs/arrow_router.py b/rfs/arrow_router.py index 91cd8c4..9ac35d3 100644 --- a/rfs/arrow_router.py +++ b/rfs/arrow_router.py @@ -1,707 +1,9 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from .utils import write_json - -ROUTER_VERSION = "reference-constrained-orthogonal-v1" -AESTHETIC_ROUTER_VERSION = "reference-tunnel-aesthetic-v2" -DEFAULT_REFERENCE_TUNNEL_PERCENT = 0.024 - - -def _bbox_center(bbox: dict[str, Any]) -> list[float]: - return [ - round(float(bbox["x"]) + float(bbox["w"]) / 2, 4), - round(float(bbox["y"]) + float(bbox["h"]) / 2, 4), - ] - - -def _anchor_point(obj: dict | None, anchor: str) -> list[float] | None: - if not obj or not isinstance(obj.get("bbox_percent"), dict): - return None - bbox = obj["bbox_percent"] - x = float(bbox["x"]) - y = float(bbox["y"]) - w = float(bbox["w"]) - h = float(bbox["h"]) - points = { - "left_mid": [x, y + h / 2], - "right_mid": [x + w, y + h / 2], - "top_mid": [x + w / 2, y], - "bottom_mid": [x + w / 2, y + h], - "center": [x + w / 2, y + h / 2], - } - point = points.get(anchor) or points["center"] - return [round(point[0], 4), round(point[1], 4)] - - -def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: - if not source or not target: - return [] - sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None - tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None - if not sbox or not tbox: - return [] - sx, sy = _bbox_center(sbox) - tx, ty = _bbox_center(tbox) - dx = tx - sx - dy = ty - sy - if abs(dx) >= abs(dy): - s_anchor = "right_mid" if dx >= 0 else "left_mid" - t_anchor = "left_mid" if dx >= 0 else "right_mid" - else: - s_anchor = "bottom_mid" if dy >= 0 else "top_mid" - t_anchor = "top_mid" if dy >= 0 else "bottom_mid" - s = _anchor_point(source, s_anchor) - t = _anchor_point(target, t_anchor) - return [s, t] if s and t else [] - - -def _clean_path(points: list[list[float] | None]) -> list[list[float]]: - cleaned: list[list[float]] = [] - for point in points: - if not point or len(point) < 2: - continue - x = min(1.0, max(0.0, float(point[0]))) - y = min(1.0, max(0.0, float(point[1]))) - rounded = [round(x, 4), round(y, 4)] - if not cleaned or abs(cleaned[-1][0] - rounded[0]) > 0.001 or abs(cleaned[-1][1] - rounded[1]) > 0.001: - cleaned.append(rounded) - if len(cleaned) <= 2: - return cleaned - simplified = [cleaned[0]] - for index in range(1, len(cleaned) - 1): - prev = simplified[-1] - cur = cleaned[index] - nxt = cleaned[index + 1] - same_x = abs(prev[0] - cur[0]) < 0.001 and abs(cur[0] - nxt[0]) < 0.001 - same_y = abs(prev[1] - cur[1]) < 0.001 and abs(cur[1] - nxt[1]) < 0.001 - if not (same_x or same_y): - simplified.append(cur) - simplified.append(cleaned[-1]) - return simplified - - -def _segments(points: list[list[float]]) -> list[tuple[list[float], list[float]]]: - return [(points[i], points[i + 1]) for i in range(max(0, len(points) - 1))] - - -def _path_length(points: list[list[float]]) -> float: - total = 0.0 - for a, b in _segments(points): - total += ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 - return round(total, 4) - - -def _path_normal(points: list[list[float]]) -> list[float]: - if len(points) < 2: - return [0.0, 0.0] - dx = float(points[-1][0]) - float(points[0][0]) - dy = float(points[-1][1]) - float(points[0][1]) - length = (dx * dx + dy * dy) ** 0.5 - if length < 0.0001: - return [0.0, 0.0] - return [-dy / length, dx / length] - - -def _point_distance(a: list[float], b: list[float]) -> float: - return ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 - - -def _point_segment_distance(point: list[float], a: list[float], b: list[float]) -> float: - px, py = float(point[0]), float(point[1]) - ax, ay = float(a[0]), float(a[1]) - bx, by = float(b[0]), float(b[1]) - dx = bx - ax - dy = by - ay - denom = dx * dx + dy * dy - if denom < 0.0000001: - return _point_distance(point, a) - t = max(0.0, min(1.0, ((px - ax) * dx + (py - ay) * dy) / denom)) - proj = [ax + t * dx, ay + t * dy] - return _point_distance(point, proj) - - -def _max_distance_to_polyline(points: list[list[float]], reference: list[list[float]]) -> float: - segments = _segments(reference) - if not segments: - return 0.0 - distances = [] - for point in points: - distances.append(min(_point_segment_distance(point, a, b) for a, b in segments)) - return round(max(distances, default=0.0), 4) - - -def _lane_offset(lane_index: int, lane_count: int, tunnel_percent: float) -> float: - if lane_count <= 1: - return 0.0 - centered = float(lane_index) - (float(lane_count) - 1.0) / 2.0 - step = min(0.006, max(0.002, tunnel_percent / max(lane_count, 1))) - return max(-tunnel_percent * 0.82, min(tunnel_percent * 0.82, centered * step)) - - -def _shift_point(point: list[float], normal: list[float], offset: float) -> list[float]: - return [ - round(min(1.0, max(0.0, float(point[0]) + normal[0] * offset)), 4), - round(min(1.0, max(0.0, float(point[1]) + normal[1] * offset)), 4), - ] - - -def _aesthetic_tunnel_path( - points: list[list[float]], - lane_index: int, - lane_count: int, - role: str, - tunnel_percent: float = DEFAULT_REFERENCE_TUNNEL_PERCENT, -) -> tuple[list[list[float]], dict[str, Any]]: - original = _clean_path(points) - if len(original) < 2: - return original, { - "routing_algorithm": AESTHETIC_ROUTER_VERSION, - "route_generation_status": "aesthetic_no_path", - "reference_tunnel_percent": tunnel_percent, - "reference_path_delta_max": 0.0, - "reference_tunnel_preserved": True, - } - normal = _path_normal(original) - offset = _lane_offset(lane_index, lane_count, tunnel_percent) - adjusted = [list(point) for point in original] - if abs(offset) > 0.0001 and len(original) == 2: - mid = [ - (float(original[0][0]) + float(original[1][0])) / 2.0, - (float(original[0][1]) + float(original[1][1])) / 2.0, - ] - adjusted = [original[0], _shift_point(mid, normal, offset), original[-1]] - elif role in {"main_flow", "module_flow"}: - # Single-lane arrows keep the exact reference geometry; the PPT renderer - # still softens them through curve connectors, halo, and line caps. - adjusted = original - adjusted = _clean_path(adjusted) - delta = _max_distance_to_polyline(adjusted, original) - return adjusted, { - "routing_algorithm": AESTHETIC_ROUTER_VERSION, - "route_generation_status": "aesthetic_tunnel_adjusted" if adjusted != original else "aesthetic_style_only", - "reference_tunnel_percent": tunnel_percent, - "reference_path_delta_max": delta, - "reference_tunnel_preserved": delta <= tunnel_percent, - "lane_offset_percent": round(offset, 4), - } - - -def _orientation(a: list[float], b: list[float], c: list[float]) -> float: - return (float(b[1]) - float(a[1])) * (float(c[0]) - float(b[0])) - (float(b[0]) - float(a[0])) * (float(c[1]) - float(b[1])) - - -def _share_endpoint(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: - pairs = ((a1, b1), (a1, b2), (a2, b1), (a2, b2)) - return any(abs(p[0] - q[0]) < 0.002 and abs(p[1] - q[1]) < 0.002 for p, q in pairs) +"""Compatibility alias for :mod:`rfs.composition.rebuild.arrows`.""" +from __future__ import annotations -def _segments_cross(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: - if _share_endpoint(a1, a2, b1, b2): - return False - # Fast reject by segment bounding boxes. - if max(min(a1[0], a2[0]), min(b1[0], b2[0])) > min(max(a1[0], a2[0]), max(b1[0], b2[0])): - return False - if max(min(a1[1], a2[1]), min(b1[1], b2[1])) > min(max(a1[1], a2[1]), max(b1[1], b2[1])): - return False - o1 = _orientation(a1, a2, b1) - o2 = _orientation(a1, a2, b2) - o3 = _orientation(b1, b2, a1) - o4 = _orientation(b1, b2, a2) - return o1 * o2 < 0 and o3 * o4 < 0 - - -def _point_inside_bbox(point: list[float], bbox: dict[str, Any], pad: float = 0.0) -> bool: - x = float(bbox["x"]) - pad - y = float(bbox["y"]) - pad - w = float(bbox["w"]) + pad * 2 - h = float(bbox["h"]) + pad * 2 - return x <= float(point[0]) <= x + w and y <= float(point[1]) <= y + h - - -def _segment_bbox_overlap(a: list[float], b: list[float], bbox: dict[str, Any], pad: float = 0.004) -> bool: - if _point_inside_bbox(a, bbox, pad=pad) or _point_inside_bbox(b, bbox, pad=pad): - return True - x0 = float(bbox["x"]) - pad - y0 = float(bbox["y"]) - pad - x1 = float(bbox["x"]) + float(bbox["w"]) + pad - y1 = float(bbox["y"]) + float(bbox["h"]) + pad - edges = [ - ([x0, y0], [x1, y0]), - ([x1, y0], [x1, y1]), - ([x1, y1], [x0, y1]), - ([x0, y1], [x0, y0]), - ] - return any(_segments_cross(a, b, e0, e1) for e0, e1 in edges) - - -def _candidate_lanes(source: dict, target: dict, obstacles: dict[str, dict], axis: str, limit: int = 18) -> list[float]: - sbox = source["bbox_percent"] - tbox = target["bbox_percent"] - if axis == "x": - base = [ - float(sbox["x"]), - float(sbox["x"]) + float(sbox["w"]), - float(sbox["x"]) + float(sbox["w"]) / 2, - float(tbox["x"]), - float(tbox["x"]) + float(tbox["w"]), - float(tbox["x"]) + float(tbox["w"]) / 2, - ] - for obj in obstacles.values(): - bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None - if not bbox: - continue - base.extend([float(bbox["x"]) - 0.014, float(bbox["x"]) + float(bbox["w"]) + 0.014]) - else: - base = [ - float(sbox["y"]), - float(sbox["y"]) + float(sbox["h"]), - float(sbox["y"]) + float(sbox["h"]) / 2, - float(tbox["y"]), - float(tbox["y"]) + float(tbox["h"]), - float(tbox["y"]) + float(tbox["h"]) / 2, - ] - for obj in obstacles.values(): - bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None - if not bbox: - continue - base.extend([float(bbox["y"]) - 0.014, float(bbox["y"]) + float(bbox["h"]) + 0.014]) - midpoint = (sum(base[:6]) / 6) if len(base) >= 6 else 0.5 - lanes = sorted({round(min(0.985, max(0.015, value)), 4) for value in base}, key=lambda value: abs(value - midpoint)) - return sorted(lanes[:limit]) - - -def _crossing_count(points: list[list[float]], existing_segments: list[tuple[list[float], list[float]]]) -> int: - count = 0 - for a, b in _segments(points): - for c, d in existing_segments: - if _segments_cross(a, b, c, d): - count += 1 - return count - - -def _obstacle_overlaps(points: list[list[float]], obstacles: dict[str, dict]) -> list[str]: - overlaps = [] - for obj_id, obj in obstacles.items(): - bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None - if not bbox: - continue - if any(_segment_bbox_overlap(a, b, bbox) for a, b in _segments(points)): - overlaps.append(obj_id) - return overlaps - - -def _score_candidate(points: list[list[float]], obstacles: dict[str, dict], existing_segments: list[tuple[list[float], list[float]]]) -> tuple[float, dict[str, Any]]: - overlaps = _obstacle_overlaps(points, obstacles) - crossings = _crossing_count(points, existing_segments) - bends = max(0, len(points) - 2) - length = _path_length(points) - near_edge_penalty = sum(1 for x, y in points if x < 0.012 or x > 0.988 or y < 0.012 or y > 0.988) - score = len(overlaps) * 1000 + crossings * 80 + bends * 8 + length * 16 + near_edge_penalty * 10 - return score, { - "candidate_score": round(score, 4), - "obstacle_overlap_count": len(overlaps), - "obstacle_overlap_ids": overlaps[:12], - "crossing_count": crossings, - "bend_count": bends, - "path_length": length, - } - - -def _best_fallback_path( - source: dict | None, - target: dict | None, - obstacles: dict[str, dict], - existing_segments: list[tuple[list[float], list[float]]], -) -> tuple[list[list[float]], dict[str, Any]]: - if not source or not target: - return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_source_or_target"} - direct = _edge_path(source, target) - if len(direct) < 2: - return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_anchor_points"} - - sx, sy = direct[0] - tx, ty = direct[-1] - x_lanes = _candidate_lanes(source, target, obstacles, "x") - y_lanes = _candidate_lanes(source, target, obstacles, "y") - candidates = [_clean_path(direct)] - candidates.extend(_clean_path([direct[0], [x, sy], [x, ty], direct[-1]]) for x in x_lanes) - candidates.extend(_clean_path([direct[0], [sx, y], [tx, y], direct[-1]]) for y in y_lanes) - # Two-lane orthogonal paths can move around large central obstacles without - # changing the source-target semantics from the reference-derived program. - for x in x_lanes[:12]: - for y in y_lanes[:12]: - candidates.append(_clean_path([direct[0], [x, sy], [x, y], [tx, y], direct[-1]])) - candidates.append(_clean_path([direct[0], [sx, y], [x, y], [x, ty], direct[-1]])) - - best_path: list[list[float]] = direct - best_meta: dict[str, Any] = {} - best_score = float("inf") - seen: set[tuple[tuple[float, float], ...]] = set() - for candidate in candidates: - if len(candidate) < 2: - continue - key = tuple((point[0], point[1]) for point in candidate) - if key in seen: - continue - seen.add(key) - score, meta = _score_candidate(candidate, obstacles, existing_segments) - if score < best_score: - best_score = score - best_path = candidate - best_meta = meta - best_meta.update({ - "routing_algorithm": ROUTER_VERSION, - "route_generation_status": "fallback_route_selected", - "candidate_count": len(seen), - }) - return best_path, best_meta - - -def _infer_role(arrow: dict, out_counts: dict[str, int], in_counts: dict[str, int], panel_ids: set[str]) -> str: - kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() - source = str(arrow.get("source_id") or arrow.get("source") or "") - target = str(arrow.get("target_id") or arrow.get("target") or "") - if "loop" in kind or "dashed" in kind: - return "feedback_loop" - # Branch/convergence semantics should win over panel membership. Some RFS - # programs use panel-like IDs for central modules, and treating those first - # collapses every connector into the same main-flow style. - if out_counts.get(source, 0) > 1: - return "branch" - if in_counts.get(target, 0) > 1: - return "convergence" - if source in panel_ids or target in panel_ids: - return "main_flow" - return "module_flow" - - -def _style_for_role(role: str, arrow: dict, mode: str = "reference") -> dict[str, Any]: - kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() - if str(mode).lower() == "aesthetic": - if role == "main_flow": - return {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "line_cap": "round", "halo_width_pt": 4.4, "halo_color": "#FFFFFF"} - if role == "branch": - return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} - if role == "convergence": - return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} - if role == "feedback_loop": - return {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} - if "elbow" in kind: - return {"route_style": "rounded_elbow", "stroke_width_pt": 1.6, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} - return {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} - if role == "main_flow": - return {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med", "line_cap": "round"} - if role == "branch": - return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} - if role == "convergence": - return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} - if role == "feedback_loop": - return {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash"} - if "elbow" in kind: - return {"route_style": "rounded_elbow", "stroke_width_pt": 1.65, "arrowhead_size": "sm", "line_cap": "round"} - return {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round"} - - -def _reference_locked(arrow: dict) -> bool: - path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - if len(path) < 2: - return False - source = str(arrow.get("binding_source") or arrow.get("detected_by") or "").lower() - if any(term in source for term in ("reference", "opencv", "vlm", "explicit", "candidate", "layout")): - return True - # In reference-primary mode, existing normalized paths are treated as a - # reference contract unless explicitly marked as fallback. - return str(arrow.get("route_policy", "")).lower() != "fallback_reroute_allowed" - - -def _route_metrics(arrows: list[dict], objects: dict[str, dict]) -> tuple[dict[str, dict], int]: - metrics: dict[str, dict] = {} - all_segments: list[tuple[str, list[float], list[float]]] = [] - for arrow in arrows: - points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - sid = str(arrow.get("source_id") or arrow.get("source") or "") - tid = str(arrow.get("target_id") or arrow.get("target") or "") - obstacles = [] - for obj_id, obj in objects.items(): - if obj_id in {sid, tid} or not isinstance(obj.get("bbox_percent"), dict): - continue - if any(_segment_bbox_overlap(a, b, obj["bbox_percent"]) for a, b in _segments(points)): - obstacles.append(obj_id) - metrics[str(arrow.get("id"))] = { - "path_length": _path_length(points), - "bend_count": max(0, len(points) - 2), - "obstacle_overlap_count": len(obstacles), - "obstacle_overlap_ids": obstacles[:12], - "point_count": len(points), - } - for a, b in _segments(points): - all_segments.append((str(arrow.get("id")), a, b)) - - crossing_count = 0 - crossings_by_arrow = {str(arrow.get("id")): 0 for arrow in arrows} - for idx, (aid, a1, a2) in enumerate(all_segments): - for bid, b1, b2 in all_segments[idx + 1:]: - if aid == bid: - continue - if _segments_cross(a1, a2, b1, b2): - crossing_count += 1 - crossings_by_arrow[aid] = crossings_by_arrow.get(aid, 0) + 1 - crossings_by_arrow[bid] = crossings_by_arrow.get(bid, 0) + 1 - for aid, count in crossings_by_arrow.items(): - if aid in metrics: - metrics[aid]["crossing_count"] = count - return metrics, crossing_count - - -def style_and_route_arrows( - program: dict, - out_dir: str | Path, - mode: str = "reference", -) -> dict: - """Add reference-preserving arrow aesthetics and QA artifacts. - - The reference image remains the hard constraint. Existing reference-derived - routes are not freely rerouted; this stage mostly adds style, bundling - metadata, and diagnostics. Only missing/fallback paths are synthesized. - """ - out = Path(out_dir) - if str(mode).lower() == "off": - write_json(out / "arrow_style_profile.json", {"summary": "Arrow styling skipped.", "mode": "off"}) - write_json(out / "selected_arrow_routes.json", {"summary": "Arrow routing skipped.", "mode": "off", "routes": []}) - write_json(out / "arrow_quality_report.json", {"summary": "Arrow quality skipped.", "mode": "off", "status": "skipped"}) - return program - - panels = program.get("panels", []) if isinstance(program.get("panels"), list) else [] - slots = program.get("slots", []) if isinstance(program.get("slots"), list) else [] - program_style = program.get("style") if isinstance(program.get("style"), dict) else {} - color_tokens = program_style.get("color_tokens", []) if isinstance(program_style.get("color_tokens"), list) else [] - default_style_token_id = next( - ( - str(item.get("token_id") or "") - for item in color_tokens - if isinstance(item, dict) and any(term in str(item.get("usage") or "").casefold() for term in ("arrow", "connector", "line")) - ), - "", - ) - if not default_style_token_id: - default_style_token_id = next((str(item.get("token_id") or "") for item in color_tokens if isinstance(item, dict) and item.get("token_id")), "") - objects = {str(item.get("id")): item for item in panels + slots if isinstance(item, dict) and item.get("id")} - slot_objects = {str(item.get("id")): item for item in slots if isinstance(item, dict) and item.get("id")} - panel_ids = {str(panel.get("id")) for panel in panels if isinstance(panel, dict)} - arrows = [dict(item) for item in program.get("arrows", []) if isinstance(item, dict)] - out_counts: dict[str, int] = {} - in_counts: dict[str, int] = {} - for arrow in arrows: - source = str(arrow.get("source_id") or arrow.get("source") or "") - target = str(arrow.get("target_id") or arrow.get("target") or "") - out_counts[source] = out_counts.get(source, 0) + 1 - in_counts[target] = in_counts.get(target, 0) + 1 - - reference_segments: list[tuple[list[float], list[float]]] = [] - for original in arrows: - original_points = original.get("path_percent") if isinstance(original.get("path_percent"), list) else [] - if _reference_locked(original) and len(original_points) >= 2: - reference_segments.extend(_segments(_clean_path(original_points))) - - grouped: dict[str, list[dict]] = {} - enriched = [] - committed_fallback_segments: list[tuple[list[float], list[float]]] = [] - for index, arrow in enumerate(arrows, start=1): - source = str(arrow.get("source_id") or arrow.get("source") or "") - target = str(arrow.get("target_id") or arrow.get("target") or "") - arrow["source_id"] = source - arrow["target_id"] = target - arrow["source"] = source - arrow["target"] = target - if not str(arrow.get("style_token_id") or "").strip(): - arrow["style_token_id"] = default_style_token_id - points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] - locked = _reference_locked(arrow) - route_meta: dict[str, Any] = {"routing_algorithm": "preserve_reference_path", "route_generation_status": "reference_locked"} - fallback_allowed = str(arrow.get("route_policy", "")).lower() == "fallback_reroute_allowed" - if (len(points) < 2 or (fallback_allowed and not locked)) and mode in {"reference", "aesthetic"}: - obstacles = {obj_id: obj for obj_id, obj in slot_objects.items() if obj_id not in {source, target}} - points, route_meta = _best_fallback_path( - objects.get(source), - objects.get(target), - obstacles, - reference_segments + committed_fallback_segments, - ) - locked = False - else: - points = _clean_path(points) - arrow["path_percent"] = [[round(float(p[0]), 4), round(float(p[1]), 4)] for p in points if isinstance(p, list) and len(p) >= 2] - role = _infer_role(arrow, out_counts, in_counts, panel_ids) - style = _style_for_role(role, arrow, mode=mode) - recompute_generated_style = str(arrow.get("aesthetic_policy", "")).startswith("reference_first") - for key, value in style.items(): - if recompute_generated_style or key not in arrow or not str(arrow.get(key, "")).strip(): - arrow[key] = value - if recompute_generated_style or not str(arrow.get("semantic_role", "")).strip(): - arrow["semantic_role"] = role - arrow.setdefault("aesthetic_policy", "reference_first_soft_editable_ppt_connector") - arrow.setdefault("reference_locked", locked) - arrow.setdefault("reference_path_preserved", locked) - arrow.setdefault("route_policy", "preserve_reference_path" if locked else "synthesize_missing_path_only") - arrow.setdefault("routing_algorithm", route_meta.get("routing_algorithm", ROUTER_VERSION)) - arrow.setdefault("route_generation_status", route_meta.get("route_generation_status", "fallback_route_selected" if not locked else "reference_locked")) - if "candidate_count" in route_meta: - arrow.setdefault("candidate_count", route_meta["candidate_count"]) - if "candidate_score" in route_meta: - arrow.setdefault("candidate_score", route_meta["candidate_score"]) - arrow.setdefault("corner_radius_percent", 0.018 if arrow.get("route_style") in {"rounded_elbow", "bundled_elbow"} else 0.0) - if role == "branch": - bundle_id = f"from_{source}" - elif role == "convergence": - bundle_id = f"to_{target}" - elif role == "feedback_loop": - bundle_id = f"loop_{source}_{target}" - else: - bundle_id = f"flow_{index:02d}" - arrow.setdefault("bundle_id", bundle_id) - grouped.setdefault(bundle_id, []).append(arrow) - enriched.append(arrow) - if not locked and len(arrow["path_percent"]) >= 2: - committed_fallback_segments.extend(_segments(arrow["path_percent"])) - - for bundle_id, items in grouped.items(): - items.sort(key=lambda item: (item.get("target_id", ""), item.get("id", ""))) - for lane_index, arrow in enumerate(items): - arrow["lane_index"] = lane_index - arrow["lane_count"] = len(items) - - if str(mode).lower() == "aesthetic": - tunnel_percent = DEFAULT_REFERENCE_TUNNEL_PERCENT - for arrow in enriched: - original_path = _clean_path(arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else []) - arrow["reference_original_path_percent"] = original_path - if arrow.get("reference_locked") and len(original_path) >= 2: - offset_allowed = bool(arrow.get("aesthetic_offset_allowed")) or str(arrow.get("route_policy", "")).lower() == "aesthetic_tunnel_allowed" - if offset_allowed: - adjusted_path, aesthetic_meta = _aesthetic_tunnel_path( - original_path, - int(arrow.get("lane_index") or 0), - int(arrow.get("lane_count") or 1), - str(arrow.get("semantic_role") or ""), - tunnel_percent=tunnel_percent, - ) - arrow["path_percent"] = adjusted_path - arrow["routing_algorithm"] = aesthetic_meta["routing_algorithm"] - arrow["route_generation_status"] = aesthetic_meta["route_generation_status"] - arrow["reference_tunnel_percent"] = aesthetic_meta["reference_tunnel_percent"] - arrow["reference_path_delta_max"] = aesthetic_meta["reference_path_delta_max"] - arrow["reference_tunnel_preserved"] = aesthetic_meta["reference_tunnel_preserved"] - arrow["lane_offset_percent"] = aesthetic_meta["lane_offset_percent"] - arrow["reference_path_preserved"] = bool(aesthetic_meta["reference_tunnel_preserved"]) - else: - arrow["routing_algorithm"] = AESTHETIC_ROUTER_VERSION - arrow["route_generation_status"] = "aesthetic_style_only" - arrow["reference_tunnel_percent"] = tunnel_percent - arrow["reference_path_delta_max"] = 0.0 - arrow["reference_tunnel_preserved"] = True - arrow["lane_offset_percent"] = 0.0 - arrow["reference_path_preserved"] = True +import sys as _sys - metrics, total_crossings = _route_metrics(enriched, slot_objects) - routes = [] - for arrow in enriched: - aid = str(arrow.get("id")) - m = metrics.get(aid, {}) - clutter_score = ( - int(m.get("crossing_count", 0)) * 12 - + int(m.get("bend_count", 0)) * 3 - + int(m.get("obstacle_overlap_count", 0)) * 18 - ) - arrow["aesthetic_score"] = max(0, 100 - clutter_score) - routes.append({ - "id": aid, - "source_id": arrow.get("source_id"), - "target_id": arrow.get("target_id"), - "semantic_role": arrow.get("semantic_role"), - "route_style": arrow.get("route_style"), - "bundle_id": arrow.get("bundle_id"), - "lane_index": arrow.get("lane_index"), - "lane_count": arrow.get("lane_count"), - "reference_locked": arrow.get("reference_locked"), - "reference_path_preserved": arrow.get("reference_path_preserved"), - "path_percent": arrow.get("path_percent"), - "style_token_id": arrow.get("style_token_id"), - "stroke_width_pt": arrow.get("stroke_width_pt"), - "arrowhead_size": arrow.get("arrowhead_size"), - "line_cap": arrow.get("line_cap"), - "line_pattern": arrow.get("line_pattern", "solid"), - "routing_algorithm": arrow.get("routing_algorithm"), - "route_generation_status": arrow.get("route_generation_status"), - "reference_original_path_percent": arrow.get("reference_original_path_percent"), - "reference_tunnel_percent": arrow.get("reference_tunnel_percent"), - "reference_path_delta_max": arrow.get("reference_path_delta_max"), - "reference_tunnel_preserved": arrow.get("reference_tunnel_preserved"), - "lane_offset_percent": arrow.get("lane_offset_percent"), - "halo_width_pt": arrow.get("halo_width_pt"), - "halo_color": arrow.get("halo_color"), - "candidate_count": arrow.get("candidate_count"), - "candidate_score": arrow.get("candidate_score"), - "metrics": m, - "aesthetic_score": arrow["aesthetic_score"], - }) +from .composition.rebuild import arrows as _implementation - arrow_style_profile = { - "summary": "Reference-first arrow styling profile for editable PPT connectors.", - "mode": mode, - "reference_priority": "reference_image_hard_constraint", - "routing_principle": "preserve reference-derived source-target logic and path geometry; only synthesize missing fallback paths", - "routing_algorithm": ROUTER_VERSION, - "aesthetic_routing_algorithm": AESTHETIC_ROUTER_VERSION, - "reference_tunnel_percent": DEFAULT_REFERENCE_TUNNEL_PERCENT, - "fallback_routing_policy": "only arrows with missing paths or route_policy=fallback_reroute_allowed may use obstacle-aware orthogonal routing", - "aesthetic_mode_policy": "mode=aesthetic may adjust reference-locked paths only inside the reference tunnel and must keep reference_tunnel_preserved=true", - "style_rules": { - "main_flow": {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med"}, - "branch": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, - "convergence": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, - "feedback_loop": {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm"}, - "module_flow": {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm"}, - }, - "aesthetic_style_rules": { - "main_flow": {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "halo_width_pt": 4.4}, - "branch": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, - "convergence": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, - "feedback_loop": {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "halo_width_pt": 3.0}, - "module_flow": {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "halo_width_pt": 3.0}, - }, - "ppt_editability": "all arrows render as PPT connector shapes, not raster assets", - "rounded_line_policy": "use round line caps and editable connector segments; reference-locked paths are not geometrically rewritten", - } - selected_routes = { - "summary": "Selected reference-preserving arrow routes and style assignments.", - "mode": mode, - "route_count": len(routes), - "routes": routes, - } - tunnel_violations = [route["id"] for route in routes if route.get("reference_tunnel_preserved") is False] - quality_report = { - "summary": "Arrow routing and styling quality report.", - "mode": mode, - "status": "pass" if all(route["aesthetic_score"] >= 35 for route in routes) and not tunnel_violations else "needs_review", - "arrow_count": len(routes), - "total_crossing_count": total_crossings, - "total_obstacle_overlap_count": sum(int(route["metrics"].get("obstacle_overlap_count", 0)) for route in routes), - "fallback_route_count": sum(1 for route in routes if route.get("route_generation_status") == "fallback_route_selected"), - "aesthetic_tunnel_adjusted_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_tunnel_adjusted"), - "aesthetic_style_only_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_style_only"), - "reference_tunnel_violations": tunnel_violations, - "average_aesthetic_score": round(sum(float(route["aesthetic_score"]) for route in routes) / max(len(routes), 1), 2), - "reference_path_overrides": [route["id"] for route in routes if route.get("reference_locked") and not route.get("reference_path_preserved")], - "routes": routes, - } - program["arrows"] = enriched - program["control_shapes"] = [dict(item) for item in enriched] - program.setdefault("style", {})["arrow_style_profile_path"] = "arrow_style_profile.json" - write_json(out / "arrow_style_profile.json", arrow_style_profile) - write_json(out / "selected_arrow_routes.json", selected_routes) - write_json(out / "arrow_quality_report.json", quality_report) - write_json(out / "figure_program.json", program) - return program +_sys.modules[__name__] = _implementation diff --git a/rfs/asset_generator.py b/rfs/asset_generator.py index bca97af..fe28292 100644 --- a/rfs/asset_generator.py +++ b/rfs/asset_generator.py @@ -1,974 +1,9 @@ -from __future__ import annotations +"""Compatibility alias for :mod:`rfs.generation.assets`.""" -import base64 -from concurrent.futures import ThreadPoolExecutor, as_completed -import json -import math -import os -import shutil -import time -from pathlib import Path -from statistics import median -from typing import Iterable +from __future__ import annotations -import requests -import numpy as np -from PIL import Image, ImageDraw, ImageFont +import sys as _sys -from .utils import ensure_dir, write_json, write_text +from .generation import assets as _implementation -IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff"} - -CANDIDATE_DIRECTIONS = [ - "dense mechanism-first scientific mini scene with foreground, secondary objects, and micro-detail", - "reference-crop reconstruction with layered local objects and edge-to-edge support detail", - "compact paper-figure mini panel with concrete objects, internal mechanism, and background cues", - "structured process card with visible before-and-after relation plus local reference texture", - "high-density method card focused on the named operation, not a standalone pictogram", -] - -NEGATIVE_PROMPT = ( - "Avoid: large empty margin, tiny centered object, sparse white background, isolated small icon on blank canvas, " - "cropped subject, cut-off edges, fake formulas, fake axes, fake metrics, readable paragraphs, paper title text, " - "watermarks, logos, screenshots, low-resolution blur, poster style, decorative clutter without scientific meaning, " - "extra white presentation tile, white matting, large white border, generic blue-green dashboard styling, " - "simple centered icon, single object on clean blank background, standalone pictogram." -) - - -def _parse_ratio(ratio: str) -> tuple[float, float]: - try: - left, right = ratio.split(":", 1) - return max(0.001, float(left)), max(0.001, float(right)) - except Exception: - return 1.0, 1.0 - - -def _ratio_value(ratio: str) -> float: - rw, rh = _parse_ratio(ratio) - return rw / max(rh, 1) - - -def _target_size(ratio: str, max_side: int = 768, min_side: int = 96) -> tuple[int, int]: - rw, rh = _parse_ratio(ratio) - if rw >= rh: - return max_side, max(min_side, int(max_side * rh / rw)) - return max(min_side, int(max_side * rw / rh)), max_side - - -def _as_list_text(value) -> str: - if isinstance(value, list): - items = [str(item).strip() for item in value if str(item).strip()] - return "; ".join(items) - return str(value or "").strip() - - -def _fallback_visual_metaphor(slot: dict) -> str: - concept = str(slot.get("paper_concept", "scientific method component")) - panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "") - low = f"{concept} {panel}".lower() - if "sparse" in low or "nonzero" in low: - return "draw a matrix grid with many pale empty cells and a few highlighted trainable nonzero cells" - if "empty" in low or "zero" in low: - return "draw a mostly empty matrix grid with muted zero cells and clear blank positions" - if "frozen" in low or "pre-trained" in low or "pretrained" in low: - return "draw a large locked frozen weight matrix slab, with icy muted cells and no trainable highlights" - if "lora" in low: - return "draw two slim low-rank matrix factors forming a compact adapter update beside a larger frozen weight matrix" - if "ift" in low or "fourier" in low or "frequency" in low: - return "draw a frequency-domain sparse grid transforming into a smooth spatial-domain matrix through wave arcs" - if "split" in low or "submat" in low: - return "draw one matrix split into multiple smaller aligned submatrices with visible separation cuts" - if "concat" in low: - return "draw multiple transformed matrix blocks stacked vertically into one taller concatenated matrix" - if "adaptor" in low or "adapter" in low: - return "draw a frozen shared projection block receiving matrix features and emitting a compact update" - if "multiplication" in low: - return "draw two matrix blocks combining through a multiplication junction into a compact output block" - if "budget" in low or "parameter" in low: - return "draw a tiny trainable budget highlighted against a large muted frozen parameter field" - if "benchmark" in low or "evaluation" in low or "glue" in low: - return "draw a benchmark suite board with multiple task tiles and abstract bars without readable numbers" - return f"draw a concrete paper-specific visual object for {concept}, showing its role inside {panel}" - - -def build_slot_prompt(slot: dict, style: dict, candidate_index: int = 1) -> str: - palette = ", ".join(style.get("palette", [])[:5]) - direction = CANDIDATE_DIRECTIONS[(candidate_index - 1) % len(CANDIDATE_DIRECTIONS)] - composition = slot.get("composition_type", "full_frame_icon") - visual_metaphor = str(slot.get("visual_metaphor") or "").strip() or _fallback_visual_metaphor(slot) - slot_function = str(slot.get("slot_function") or "").strip() - image_prompt_core = str(slot.get("image_prompt_core") or "").strip() - reference_content_priority = str(slot.get("reference_content_priority") or "reference_primary_visual_object").strip() - reference_visual_object = str(slot.get("reference_visual_object") or "").strip() - reference_visual_elements = _as_list_text(slot.get("reference_visual_elements")) - reference_color_palette = str(slot.get("reference_color_palette") or "").strip() - paper_label_mapping = str(slot.get("paper_label_mapping") or "").strip() - must_show = _as_list_text(slot.get("must_show")) or "the concrete scientific object, its internal structure, and its input-output relation" - avoid_showing = _as_list_text(slot.get("avoid_showing")) or "generic sci-fi dashboard; unrelated robot or brain; fake formulas; fake numeric chart text" - panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "paper method") - reference_slot_role = str(slot.get("reference_slot_role") or "slot in the user-provided reference layout").strip() - reference_shape_language = str(slot.get("reference_shape_language") or slot.get("target_canvas_ratio") or "").strip() - reference_local_style = str(slot.get("reference_local_style") or "match the reference figure's local card/icon discipline").strip() - reference_prompt_hint = str(slot.get("reference_prompt_hint") or "use the reference image as local layout/style guidance, not as scientific content").strip() - reference_crop_path = str(slot.get("reference_crop_path") or "").strip() - reference_style_profile_path = str(slot.get("reference_style_profile_path") or "reference_style_profile.json").strip() - local_color_token_ids = ", ".join(str(item) for item in slot.get("local_color_token_ids", []) if str(item).strip()) - complexity_kind = str(slot.get("complexity_kind") or "pipeline_module").strip() - required_visual_complexity = str(slot.get("required_visual_complexity") or "dense").strip() - foreground_subject = str(slot.get("foreground_subject") or reference_visual_object or visual_metaphor).strip() - secondary_objects = _as_list_text(slot.get("secondary_objects")) or "2-5 supporting local objects from the reference crop" - micro_details = _as_list_text(slot.get("micro_details")) or "small non-critical marks, separators, texture, and internal details" - background_fill_elements = _as_list_text(slot.get("background_fill_elements")) or "edge-to-edge local reference-colored support detail" - scientific_mechanism_detail = str(slot.get("scientific_mechanism_detail") or "show the paper concept as a visible mechanism, not a generic icon").strip() - forbidden_simplification = _as_list_text(slot.get("forbidden_simplification")) or "simple icon; centered icon; clean blank background; single object on white canvas" - is_simple_allowed = complexity_kind == "legend_icon" - - if composition == "scene_thumbnail": - type_clause = ( - "Make it a dense small mechanism thumbnail: one concrete foreground scientific object, multiple visible process cues, secondary context objects, and enough environment to show what changes. Use edge-to-edge supporting scene detail so the crop is visually full." - ) - elif composition == "full_bleed_card": - type_clause = ( - "Make it a compact dense mechanism card: the central object fills the card, with internal subparts, small supporting objects, and micro process cues that clarify this exact operation. The card surface should extend almost to the image edge, with no separate white mat." - ) - elif composition == "symbol_cutout": - type_clause = ( - "Make it a large complete symbolic operation object plus local reference-colored support detail; the operation symbol or object should be visually distinct from other slots. If it is a line icon, thicken the strokes, enlarge it to nearly touch the safe area, and add compact supporting shapes so it is not sparse." - ) - else: - type_clause = ( - "Make it a full-frame scientific mechanism object with a large distinct subject, mechanism-specific internal details, and supporting local objects. If the subject is a waveform, face, pose skeleton, document, database, badge, or legend icon, make it bold, oversized, and backed by local reference-colored detail rather than a tiny object on a blank white field." - ) - - exact_ratio = float(slot.get("aspect_ratio_decimal") or _ratio_value(slot.get("target_canvas_ratio", "1.000:1.000"))) - center = slot.get("center_percent") if isinstance(slot.get("center_percent"), dict) else {} - center_text = f"x={float(center.get('x', 0.5)):.3f}, y={float(center.get('y', 0.5)):.3f}" - width_percent = float(slot.get("width_percent", slot.get("bbox_percent", {}).get("w", 0))) - height_percent = float(slot.get("height_percent", slot.get("bbox_percent", {}).get("h", 0))) - - return ( - f"Create candidate {candidate_index} for one slot-level image block in a publication-quality AI/ML research framework figure. " - f"This block belongs to macro panel: {panel}. Slot id: {slot['id']}. Paper concept: {slot['paper_concept']}. " - f"Slot function: {slot_function or 'visualize this paper concept as one small editable figure asset'}. " - f"Model-planned image prompt core: {image_prompt_core or visual_metaphor}. " - f"Reference content priority: {reference_content_priority}. " - f"Reference visual object to recreate: {reference_visual_object or visual_metaphor}. " - f"Reference visual elements to preserve: {reference_visual_elements or must_show}. " - f"Reference local color palette to preserve: {reference_color_palette or 'inherit colors from the local reference slot, not one global palette'}. " - f"Reference style profile source: {reference_style_profile_path}. Local reference crop used by the prompt planner: {reference_crop_path or 'missing crop path - preserve described local crop anyway'}. " - f"Local reference color token ids to inherit: {local_color_token_ids or 'none recorded; use local reference colors described in prompt plan'}. " - f"Paper label mapping: {paper_label_mapping or 'paper terminology is added later as editable PPT text'}. " - f"Main visual metaphor: {visual_metaphor}. " - f"Must visibly show: {must_show}. " - f"Reference-local slot role: {reference_slot_role}. " - f"Reference shape language: {reference_shape_language}. " - f"Reference-local style: {reference_local_style}. " - f"Reference prompt hint: {reference_prompt_hint}. " - f"Slot visual complexity contract: complexity_profile={slot.get('complexity_profile', 'reference-dense')}; complexity_kind={complexity_kind}; required_visual_complexity={required_visual_complexity}. " - f"Foreground subject must be large: {foreground_subject}. " - f"Secondary objects to include: {secondary_objects}. " - f"Micro details to include: {micro_details}. " - f"Background fill elements: {background_fill_elements}. " - f"Scientific mechanism detail: {scientific_mechanism_detail}. " - f"Forbidden simplifications: {forbidden_simplification}. " - "Use the local reference slot object first; do not invent a different abstract paper-internal object when the reference shows a concrete object. " - "The goal is to redraw/recreate the reference slot's object as a clean high-resolution asset, then adapt only minor details to the paper concept. " - f"Reference geometry is mandatory: canvas aspect ratio = exact {exact_ratio:.3f}:1, target_canvas_ratio = {slot['target_canvas_ratio']}, " - f"slot center in full figure = {center_text}, slot width_percent = {width_percent:.3f}, slot height_percent = {height_percent:.3f}. " - "Generate this asset for the exact reference slot shape, not a generic preset ratio such as 1:1, 4:3, 3:4, 16:9, or 9:16. " - "No semantic cropping: the whole subject must be visible, no object extending outside the frame, no cut-off hands/edges/bottom. " - "Safe area means important subject remains complete inside the frame, but supporting background details should extend close to edges. " - "Useful visual content should fill 90-97% of the canvas; minimum 85%; maximum empty margin is strictly below 10% on every edge. " - "Use a full-frame composition, large subject, maximal detail density, rich scientific visual texture, minimal blank canvas, no tiny centered object. " - + ("" if is_simple_allowed else "This must be a dense mini scientific scene/card with 2-5 layered objects, edge-to-edge supporting detail, and not a standalone pictogram. ") - + "The subject plus its local reference-colored support surface should occupy almost the entire frame; avoid isolated cutout icons floating on white or transparent-looking empty space. " - "For thin symbols such as pose skeletons, waveforms, clocks, microphones, documents, badges, and legend icons, use thick readable strokes, enlarged scale, and compact supporting shapes to raise visual fill while keeping the subject complete. " - f"Candidate visual direction: {direction}. {type_clause} " - f"Style constraints: polished academic paper illustration, crisp edges, consistent line quality, clean high-resolution vector-illustration / 2.5D hybrid only when useful. " - f"Global palette may include {palette}, but local reference colors override the global palette. Preserve color variety across slots. " - "Style consistency matters, but visual identity must vary by the reference crop: monitors look like monitors, video strips look like video strips, microphones look like microphones, waveforms look like waveforms, face cards look like face cards, pose skeletons look like pose skeletons, clocks look like clocks, clipboards look like clipboards, databases look like databases. " - "Allow only very small decorative non-critical UI marks or blurred pseudo-labels. " - "Critical scientific labels, formulas, variables, panel IDs, arrow labels, and metric values will be added later in PowerPoint and must not be generated in the image. " - f"Do not show: {avoid_showing}. " - "Do not add a white presentation tile, white mat, extra card background, or artificial border unless that exact border exists in the local reference crop. " - f"{NEGATIVE_PROMPT}" - ) - - -def _draw_placeholder(slot: dict, path: Path, style: dict, candidate_index: int = 1) -> None: - width, height = _target_size(slot["target_canvas_ratio"], max_side=512) - palette = style.get("palette", ["#EAF5FF", "#DFF7EF", "#8EC9E8", "#66C6A4", "#163B4D"]) - bg = palette[(candidate_index - 1) % 2] - line = palette[4] if len(palette) > 4 else "#163B4D" - accent = palette[2 + ((candidate_index + len(slot["id"])) % max(1, min(2, len(palette) - 2)))] if len(palette) >= 4 else "#8EC9E8" - img = Image.new("RGB", (width, height), bg) - draw = ImageDraw.Draw(img) - - margin = max(8, int(min(width, height) * 0.035)) - draw.rounded_rectangle([margin, margin, width - margin, height - margin], radius=max(14, margin * 2), fill=bg, outline=line, width=max(3, margin // 4)) - - # Dense edge-to-edge detail so placeholder QA exercises high-fill layouts. - for i in range(8 + candidate_index): - x = int(width * (0.035 + i * 0.12)) % width - draw.line([x, margin, min(width - margin, x + int(width * 0.22)), height - margin], fill=accent, width=max(1, margin // 6)) - for i in range(6): - y = int(height * (0.10 + i * 0.15)) - draw.arc([margin + i * 5, y, width - margin - i * 4, y + int(height * 0.26)], 180, 360, fill=accent, width=max(1, margin // 7)) - - cx, cy = width // 2, height // 2 - main_w = int(width * (0.72 + 0.03 * ((candidate_index - 1) % 3))) - main_h = int(height * (0.64 + 0.03 * ((candidate_index - 1) % 2))) - kind = slot.get("composition_type", "full_frame_icon") - if kind in {"scene_thumbnail", "full_bleed_card"}: - draw.rounded_rectangle([cx - main_w // 2, cy - main_h // 2, cx + main_w // 2, cy + main_h // 2], radius=max(12, margin), fill="#FFFFFF", outline=line, width=max(3, margin // 4)) - panel_gap = max(4, margin // 3) - for row in range(2): - for col in range(2): - px0 = cx - main_w // 2 + margin + col * ((main_w - margin * 2) // 2) - py0 = cy - main_h // 2 + margin + row * ((main_h - margin * 2) // 2) - px1 = px0 + (main_w - margin * 2) // 2 - panel_gap - py1 = py0 + (main_h - margin * 2) // 2 - panel_gap - draw.rounded_rectangle([px0, py0, px1, py1], radius=max(6, margin // 2), fill=accent if (row + col + candidate_index) % 2 == 0 else "#DFF7EF", outline=line, width=max(1, margin // 8)) - draw.polygon([(px0 + panel_gap, py1 - panel_gap), (px0 + (px1 - px0) // 2, py0 + panel_gap), (px1 - panel_gap, py1 - panel_gap)], fill="#9AD7C4") - else: - radius = min(main_w, main_h) // 2 - draw.ellipse([cx - radius, cy - radius, cx + radius, cy + radius], fill="#FFFFFF", outline=line, width=max(4, margin // 3)) - for angle_idx in range(12): - ang = angle_idx * math.pi / 6 + candidate_index * 0.08 - x1 = cx + int(math.cos(ang) * radius * 0.18) - y1 = cy + int(math.sin(ang) * radius * 0.18) - x2 = cx + int(math.cos(ang) * radius * 0.82) - y2 = cy + int(math.sin(ang) * radius * 0.82) - draw.line([x1, y1, x2, y2], fill=line, width=max(3, margin // 4)) - draw.ellipse([x2 - margin // 2, y2 - margin // 2, x2 + margin // 2, y2 + margin // 2], fill=accent, outline=line) - inner = int(radius * 0.36) - draw.rounded_rectangle([cx - inner, cy - inner, cx + inner, cy + inner], radius=max(8, margin // 2), fill=accent, outline=line, width=max(2, margin // 6)) - - try: - font = ImageFont.truetype("arial.ttf", max(9, int(min(width, height) * 0.045))) - except Exception: - font = ImageFont.load_default() - draw.text((margin * 1.2, height - margin * 2.2), "UI", fill=line, font=font) - path.parent.mkdir(parents=True, exist_ok=True) - img.save(path) - - -def _supported_gemini_ratio(ratio: str) -> str: - value = _ratio_value(ratio) - supported = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "16:9": 16 / 9, "9:16": 9 / 16} - return min(supported, key=lambda key: abs(supported[key] - value)) - - -def _image2_model_name() -> str: - model = os.getenv("RFS_IMAGE_MODEL") or os.getenv("IMAGE_MODEL") or "image-2" - # Yunwu currently exposes GPT Image 2 under this model id. Keep "image-2" - # as the user-facing logical name while sending the listed model id. - if model == "image-2": - return "gpt-image-2" - return model - - -def _image2_size(ratio: str) -> str: - value = _ratio_value(ratio) - if value >= 1.25: - return "1536x1024" - if value <= 0.80: - return "1024x1536" - return "1024x1024" - - -def _write_image_response(data: dict, output_path: Path) -> bool: - items = data.get("data") if isinstance(data.get("data"), list) else [] - if not items: - return False - item = items[0] - output_path.parent.mkdir(parents=True, exist_ok=True) - if item.get("b64_json"): - output_path.write_bytes(base64.b64decode(item["b64_json"])) - return True - if item.get("url"): - response = requests.get(item["url"], timeout=180) - response.raise_for_status() - output_path.write_bytes(response.content) - return True - return False - - -def _call_image2(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: - api_base = os.getenv("API_BASE", "https://yunwu.ai/v1").rstrip("/") - api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") - model = _image2_model_name() - if not api_key: - raise RuntimeError("API_KEY/GEMINI_API_KEY is required for --asset-mode image2") - payload = { - "model": model, - "prompt": prompt, - "n": 1, - "size": _image2_size(ratio), - } - max_attempts = max(1, int(retries) + 1) - last_error: Exception | None = None - for attempt in range(1, max_attempts + 1): - try: - response = requests.post( - f"{api_base}/images/generations", - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=240, - ) - if response.status_code == 400 and payload.get("size") != "1024x1024": - # Some image gateways accept only square output for a model. The - # prompt and final ratio-normalization still preserve slot ratio. - payload = dict(payload) - payload["size"] = "1024x1024" - response = requests.post( - f"{api_base}/images/generations", - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=240, - ) - if response.status_code == 429 or 500 <= response.status_code < 600: - raise RuntimeError(f"image2 request returned HTTP {response.status_code}") - response.raise_for_status() - if _write_image_response(response.json(), output_path): - return True - raise RuntimeError("image2 response did not contain b64_json or url image data") - except Exception as exc: - last_error = exc - if attempt >= max_attempts: - break - time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) - if last_error: - raise last_error - return False - - -def _call_gemini(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: - api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") - url = os.getenv("GEMINI_GEN_IMG_URL") - if not api_key or not url: - raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode gemini") - payload = { - "contents": [{"role": "user", "parts": [{"text": prompt}]}], - "generationConfig": { - "responseModalities": ["IMAGE"], - "imageConfig": {"imageSize": "2K", "aspectRatio": _supported_gemini_ratio(ratio)}, - }, - } - max_attempts = max(1, int(retries) + 1) - last_error: Exception | None = None - for attempt in range(1, max_attempts + 1): - try: - response = requests.post( - url, - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - data=json.dumps(payload), - timeout=180, - ) - if response.status_code == 429 or 500 <= response.status_code < 600: - raise RuntimeError(f"Gemini image request returned HTTP {response.status_code}") - response.raise_for_status() - data = response.json() - for candidate in data.get("candidates", []): - for part in candidate.get("content", {}).get("parts", []): - inline = part.get("inlineData") or part.get("inline_data") - if inline and inline.get("data"): - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_bytes(base64.b64decode(inline["data"])) - return True - raise RuntimeError("Gemini image response did not contain inline image data") - except Exception as exc: - last_error = exc - if attempt >= max_attempts: - break - # Back off on rate limits, transient server errors, and network timeouts. - time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) - if last_error: - raise last_error - return False - - -def _enforce_target_canvas_ratio(path: Path, ratio: str) -> None: - """Resize to the exact slot canvas ratio without semantic cropping.""" - target_w, target_h = _target_size(ratio, max_side=1024, min_side=96) - with Image.open(path) as image: - converted = image.convert("RGBA") - if converted.size == (target_w, target_h): - return - resized = converted.resize((target_w, target_h), Image.LANCZOS) - path.parent.mkdir(parents=True, exist_ok=True) - resized.save(path) - - -def _content_bbox(rgba: Image.Image) -> tuple[list[int], float, float]: - width, height = rgba.size - pixels = rgba.load() - background = _median_corner_color(pixels, width, height) - rgba_array = np.asarray(rgba, dtype=np.int16) - rgb = rgba_array[:, :, :3] - alpha = rgba_array[:, :, 3] - background_array = np.asarray(background, dtype=np.float32) - color_distance_sq = np.sum((rgb.astype(np.float32) - background_array) ** 2, axis=2) - mask = (alpha > 12) & ((alpha < 245) | (color_distance_sq > 30.0 ** 2)) - ys, xs = np.nonzero(mask) - if xs.size == 0 or ys.size == 0: - return [0, 0, width, height], 0.0, 100.0 - left, right = int(xs.min()), int(xs.max()) - top, bottom = int(ys.min()), int(ys.max()) - bbox_w = max(1, right - left + 1) - bbox_h = max(1, bottom - top + 1) - content_fill = bbox_w * bbox_h / float(width * height) * 100.0 - margins = ( - left / width * 100.0, - (width - right - 1) / width * 100.0, - top / height * 100.0, - (height - bottom - 1) / height * 100.0, - ) - return [left, top, right + 1, bottom + 1], content_fill, max(margins) - - -def _hex_rgb(value: str, fallback: tuple[int, int, int] = (229, 242, 248)) -> tuple[int, int, int]: - text = str(value or "").strip().lstrip("#") - if len(text) >= 6: - try: - return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) - except Exception: - return fallback - return fallback - - -def _fit_subject_to_canvas(path: Path, min_fill: float = 88.0, max_margin: float = 9.0, support_color: str | None = None) -> None: - """Remove only plain background slack and enlarge the complete subject.""" - with Image.open(path) as image: - rgba = image.convert("RGBA") - width, height = rgba.size - bbox, content_fill, empty_margin = _content_bbox(rgba) - if content_fill >= min_fill and empty_margin <= max_margin: - return - left, top, right, bottom = bbox - pad_x = max(4, int((right - left) * 0.06)) - pad_y = max(4, int((bottom - top) * 0.06)) - left = max(0, left - pad_x) - top = max(0, top - pad_y) - right = min(width, right + pad_x) - bottom = min(height, bottom + pad_y) - if right <= left or bottom <= top: - return - subject = rgba.crop((left, top, right, bottom)) - scale = min(width * 0.94 / max(subject.width, 1), height * 0.94 / max(subject.height, 1)) - if scale <= 1.02: - canvas = rgba - else: - new_w = max(1, min(width, int(subject.width * scale))) - new_h = max(1, min(height, int(subject.height * scale))) - subject = subject.resize((new_w, new_h), Image.LANCZOS) - pixels = rgba.load() - bg = tuple(int(v) for v in _median_corner_color(pixels, width, height)) + (255,) - canvas = Image.new("RGBA", (width, height), bg) - canvas.alpha_composite(subject, ((width - new_w) // 2, (height - new_h) // 2)) - _bbox, final_fill, final_margin = _content_bbox(canvas) - if (final_fill < min_fill or final_margin > max_margin) and support_color: - support = Image.new("RGBA", (width, height), (0, 0, 0, 0)) - draw = ImageDraw.Draw(support) - r, g, b = _hex_rgb(support_color) - inset_x = max(2, int(width * 0.025)) - inset_y = max(2, int(height * 0.025)) - draw.rounded_rectangle( - (inset_x, inset_y, width - inset_x, height - inset_y), - radius=max(8, min(width, height) // 12), - fill=(r, g, b, 82), - outline=(r, g, b, 150), - width=max(2, min(width, height) // 80), - ) - for i in range(5): - x = int(width * (0.10 + i * 0.18)) - draw.line((x, inset_y + 4, x + int(width * 0.10), height - inset_y - 4), fill=(r, g, b, 58), width=2) - canvas.alpha_composite(support) - canvas.save(path) - - -def _median_corner_color(pixels, width: int, height: int) -> tuple[float, float, float]: - sample = [] - band_x = max(1, min(width // 12, 28)) - band_y = max(1, min(height // 12, 28)) - regions = ( - (0, 0, band_x, band_y), - (width - band_x, 0, width, band_y), - (0, height - band_y, band_x, height), - (width - band_x, height - band_y, width, height), - ) - for x0, y0, x1, y1 in regions: - for y in range(y0, y1): - for x in range(x0, x1): - r, g, b, _a = pixels[x, y] - sample.append((r, g, b)) - if not sample: - return (255.0, 255.0, 255.0) - return ( - float(median([rgb[0] for rgb in sample])), - float(median([rgb[1] for rgb in sample])), - float(median([rgb[2] for rgb in sample])), - ) - - -def _color_distance(a: tuple[int, int, int], b: tuple[float, float, float]) -> float: - return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2) ** 0.5 - - -def _estimate_quality(path: Path, slot: dict, candidate_index: int, source: str) -> dict: - with Image.open(path) as image: - rgba = image.convert("RGBA") - width, height = rgba.size - bbox, content_fill, empty_margin = _content_bbox(rgba) - - image_ratio = width / max(height, 1) - target_ratio = _ratio_value(slot["target_canvas_ratio"]) - ratio_mismatch_percent = abs(image_ratio - target_ratio) / max(target_ratio, 0.001) * 100.0 - - issue_tags = [] - if content_fill < float(slot.get("min_content_fill_percent", 80)): - issue_tags.append("too_much_whitespace") - if empty_margin > float(slot.get("max_empty_margin_percent", 12)): - issue_tags.append("large_blank_canvas") - if content_fill < 65 and empty_margin > 18: - issue_tags.append("tiny_centered_subject") - if ratio_mismatch_percent > 10: - issue_tags.append("ratio_mismatch") - - fill_score = min(content_fill, 95.0) - max(0.0, content_fill - 95.0) * 1.5 - margin_penalty = max(0.0, empty_margin - 12.0) * 4.0 - ratio_penalty = max(0.0, ratio_mismatch_percent - 10.0) * 2.0 - score = fill_score - margin_penalty - ratio_penalty - - return { - "candidate_index": candidate_index, - "path": str(path), - "source": source, - "width": width, - "height": height, - "content_bbox_px": bbox, - "content_fill_percent": round(content_fill, 2), - "min_content_fill_percent": slot["min_content_fill_percent"], - "empty_margin_percent": round(empty_margin, 2), - "max_empty_margin_percent": slot["max_empty_margin_percent"], - "ratio_mismatch_percent": round(ratio_mismatch_percent, 2), - "edge_cutoff_status": "ok", - "ratio_status": "ok" if ratio_mismatch_percent <= 10 else "needs_ratio_regeneration", - "issue_tags": issue_tags, - "selection_score": round(score, 2), - "action": "ok_no_crop" if not issue_tags else "candidate_rejected", - "estimation_method": "corner-background-content-bbox", - } - - -def _simple_allowed_slot(slot: dict) -> bool: - text = " ".join( - str(slot.get(key, "")) - for key in ["id", "paper_concept", "macro_panel", "parent_panel", "complexity_kind", "composition_type"] - ).lower() - return str(slot.get("complexity_kind", "")).lower() == "legend_icon" or any( - term in text - for term in ["legend", "badge", "label", "ocean", "openness", "conscientiousness", "extraversion", "agreeableness", "neuroticism"] - ) - - -def _estimate_component_count(mask: list[list[bool]]) -> int: - if not mask or not mask[0]: - return 0 - height = len(mask) - width = len(mask[0]) - seen = [[False] * width for _ in range(height)] - components = 0 - min_area = max(6, int(width * height * 0.002)) - for y in range(height): - for x in range(width): - if seen[y][x] or not mask[y][x]: - continue - stack = [(x, y)] - seen[y][x] = True - area = 0 - while stack: - px, py = stack.pop() - area += 1 - for nx, ny in ((px + 1, py), (px - 1, py), (px, py + 1), (px, py - 1)): - if 0 <= nx < width and 0 <= ny < height and not seen[ny][nx] and mask[ny][nx]: - seen[ny][nx] = True - stack.append((nx, ny)) - if area >= min_area: - components += 1 - return components - - -def _estimate_complexity(path: Path, slot: dict, quality: dict) -> dict: - with Image.open(path) as image: - rgba = image.convert("RGBA") - width, height = rgba.size - small_w = 96 - small_h = max(16, int(96 * height / max(width, 1))) - if small_h > 128: - small_h = 128 - small_w = max(16, int(128 * width / max(height, 1))) - small = rgba.resize((small_w, small_h), Image.LANCZOS) - pixels = small.load() - background = _median_corner_color(pixels, small_w, small_h) - mask: list[list[bool]] = [] - quantized_colors: set[tuple[int, int, int]] = set() - edge_hits = 0 - comparisons = 0 - for y in range(small_h): - row: list[bool] = [] - for x in range(small_w): - r, g, b, a = pixels[x, y] - active = a > 12 and (a < 245 or _color_distance((r, g, b), background) > 30.0) - row.append(active) - if active: - quantized_colors.add((r // 24, g // 24, b // 24)) - if x > 0: - pr, pg, pb, pa = pixels[x - 1, y] - comparisons += 1 - if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: - edge_hits += 1 - if y > 0: - pr, pg, pb, pa = pixels[x, y - 1] - comparisons += 1 - if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: - edge_hits += 1 - mask.append(row) - component_count = _estimate_component_count(mask) - color_count = len(quantized_colors) - edge_density = edge_hits / max(comparisons, 1) * 100.0 - content_fill = float(quality.get("content_fill_percent", 0)) - spec_objects = len(slot.get("secondary_objects") if isinstance(slot.get("secondary_objects"), list) else []) - spec_micro = len(slot.get("micro_details") if isinstance(slot.get("micro_details"), list) else []) - object_count_estimate = max(component_count, 1 + min(4, spec_objects // 2)) - detail_score = min( - 100.0, - content_fill * 0.30 - + min(color_count, 24) * 1.25 - + min(edge_density, 28.0) * 1.15 - + min(component_count, 8) * 4.0 - + min(spec_objects + spec_micro, 8) * 2.5, - ) - target = float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)) - object_target = int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)) - simple_icon_risk = False - issue_tags: list[str] = [] - if not _simple_allowed_slot(slot): - if detail_score < target: - simple_icon_risk = True - issue_tags.append("too_simple") - if object_count_estimate < object_target: - simple_icon_risk = True - issue_tags.append("generic_icon") - if content_fill >= 85 and color_count <= 4 and edge_density < 3: - simple_icon_risk = True - issue_tags.append("single_object_on_blank_background") - if not str(slot.get("reference_crop_path", "")).strip(): - issue_tags.append("reference_crop_ignored") - if not str(slot.get("reference_style_profile_path", "")).strip(): - issue_tags.append("style_drift") - return { - "detail_score": round(detail_score, 2), - "detail_score_target": target, - "object_count_estimate": object_count_estimate, - "object_count_target": object_target, - "simple_icon_risk": simple_icon_risk, - "reference_crop_match": "planned_reference_crop_grounded" if str(slot.get("reference_crop_path", "")).strip() else "missing_reference_crop", - "style_match": "planned_reference_style_grounded" if str(slot.get("reference_style_profile_path", "")).strip() else "missing_reference_style", - "color_bin_count": color_count, - "edge_density_percent": round(edge_density, 2), - "component_count_estimate": component_count, - "complexity_kind": slot.get("complexity_kind") or "pipeline_module", - "required_visual_complexity": slot.get("required_visual_complexity") or "dense", - "complexity_issue_tags": sorted(set(issue_tags)), - } - - -def _select_candidate(candidate_reports: list[dict]) -> dict: - passing = [item for item in candidate_reports if not item.get("issue_tags")] - pool = passing or candidate_reports - return max(pool, key=lambda item: item["selection_score"]) - - -def _make_contact_sheet(asset_paths: Iterable[Path], output_path: Path, title: str = "selected assets") -> None: - paths = list(asset_paths) - thumbs = [] - cell_w, cell_h = 260, 228 - for path in paths: - img = Image.open(path).convert("RGB") - img.thumbnail((224, 162), Image.LANCZOS) - cell = Image.new("RGB", (cell_w, cell_h), "white") - draw = ImageDraw.Draw(cell) - cell.paste(img, ((cell_w - img.width) // 2, 14)) - draw.text((10, 184), path.stem[:34], fill=(20, 20, 20)) - thumbs.append(cell) - cols = 5 - rows = max(1, (len(thumbs) + cols - 1) // cols) - header_h = 34 - sheet = Image.new("RGB", (cols * cell_w, rows * cell_h + header_h), (245, 247, 248)) - draw = ImageDraw.Draw(sheet) - draw.text((12, 10), title, fill=(20, 20, 20)) - for index, thumb in enumerate(thumbs): - sheet.paste(thumb, ((index % cols) * cell_w, header_h + (index // cols) * cell_h)) - output_path.parent.mkdir(parents=True, exist_ok=True) - sheet.save(output_path) - - -def _generate_one_candidate(slot: dict, style: dict, path: Path, asset_mode: str, candidate_index: int, prompt: str, retries: int = 2) -> str: - support_color = "" - if isinstance(slot.get("reference_dominant_colors"), list) and slot.get("reference_dominant_colors"): - support_color = str(slot["reference_dominant_colors"][0]) - elif isinstance(style.get("reference_palette"), list) and style.get("reference_palette"): - support_color = str(style["reference_palette"][0]) - if asset_mode == "placeholder": - _draw_placeholder(slot, path, style, candidate_index=candidate_index) - _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) - return "explicit_placeholder_candidate" - if asset_mode == "gemini": - ok = _call_gemini(prompt, slot["target_canvas_ratio"], path, retries=retries) - if not ok: - raise RuntimeError(f"Gemini returned no image for slot {slot['id']} candidate {candidate_index}") - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) - _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) - return "gemini_candidate_generation" - if asset_mode == "image2": - ok = _call_image2(prompt, slot["target_canvas_ratio"], path, retries=retries) - if not ok: - raise RuntimeError(f"image2 returned no image for slot {slot['id']} candidate {candidate_index}") - _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) - _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) - return f"image2_candidate_generation:{_image2_model_name()}" - raise ValueError(f"Unsupported asset_mode: {asset_mode}") - - -def _generation_failure_report(slot: dict, candidate_index: int, path: Path, exc: Exception) -> dict: - return { - "candidate_index": candidate_index, - "path": str(path), - "source": "generation_failed", - "width": 0, - "height": 0, - "content_bbox_px": [0, 0, 0, 0], - "content_fill_percent": 0.0, - "min_content_fill_percent": slot["min_content_fill_percent"], - "empty_margin_percent": 100.0, - "max_empty_margin_percent": slot["max_empty_margin_percent"], - "ratio_mismatch_percent": 100.0, - "edge_cutoff_status": "unknown_generation_failed", - "ratio_status": "unknown_generation_failed", - "issue_tags": ["generation_failed"], - "selection_score": -9999.0, - "detail_score": 0.0, - "detail_score_target": float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)), - "object_count_estimate": 0, - "object_count_target": int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)), - "simple_icon_risk": True, - "reference_crop_match": "unknown_generation_failed", - "style_match": "unknown_generation_failed", - "complexity_issue_tags": ["generation_failed"], - "action": "candidate_rejected", - "error": str(exc), - } - - -def _generate_and_score_candidate(task: dict) -> dict: - slot = task["slot"] - candidate_path = task["path"] - try: - source = _generate_one_candidate( - slot, - task["style"], - candidate_path, - task["asset_mode"], - task["candidate_index"], - task["prompt"], - retries=task["asset_retries"], - ) - quality = _estimate_quality(candidate_path, slot, task["candidate_index"], source) - complexity = _estimate_complexity(candidate_path, slot, quality) - issue_tags = sorted(set(list(quality.get("issue_tags", [])) + list(complexity.get("complexity_issue_tags", [])))) - quality.update(complexity) - quality["issue_tags"] = issue_tags - complexity_penalty = 0.0 - if complexity.get("simple_icon_risk"): - complexity_penalty += 35.0 - complexity_penalty += max(0.0, float(complexity.get("detail_score_target", 0)) - float(complexity.get("detail_score", 0))) * 0.75 - quality["selection_score"] = round(float(quality.get("selection_score", 0)) + float(complexity.get("detail_score", 0)) * 0.35 - complexity_penalty, 2) - quality["action"] = "ok_no_crop" if not issue_tags else "candidate_rejected" - return quality - except Exception as exc: - return _generation_failure_report(slot, task["candidate_index"], candidate_path, exc) - - -def generate_assets( - program: dict, - style: dict, - out_dir: str | Path, - asset_mode: str = "gemini", - candidates_per_slot: int = 3, - asset_workers: int = 1, - asset_retries: int = 2, -) -> dict: - out = Path(out_dir) - assets_dir = ensure_dir(out / "assets") - candidates_root = ensure_dir(out / "asset_candidates") - candidates_per_slot = max(1, min(5, int(candidates_per_slot))) - asset_workers = max(1, min(12, int(asset_workers))) - asset_retries = max(0, min(5, int(asset_retries))) - - prompt_lines = [ - "# Summary", - "Slot-level image prompts generated after paper brief, reference slot analysis, style sheet, layout_plan.json, figure_program.json, and slot_prompt_plan.json.", - "Each prompt inherits the model-planned image_prompt_core for its slot before adding canvas, fill, text-control, and no-crop constraints.", - f"Each slot generates {candidates_per_slot} candidate image block(s) with up to {asset_workers} worker(s); the selected asset is copied into `assets/` without semantic cropping.", - "", - ] - report_items = [] - complexity_items = [] - selected_paths = [] - all_candidate_paths = [] - tasks = [] - ordered_slot_candidates: dict[str, list[dict]] = {} - - for slot in program["slots"]: - slot_candidate_dir = ensure_dir(candidates_root / slot["id"]) - ordered_slot_candidates[slot["id"]] = [] - prompt_lines.append(f"## {slot['id']}") - prompt_lines.append(f"Paper concept: {slot['paper_concept']}") - prompt_lines.append(f"Prompt plan id: {slot.get('prompt_plan_id', 'missing')}") - prompt_lines.append(f"Visual spec id: {slot.get('visual_spec_id', 'missing')}") - prompt_lines.append(f"Complexity kind: {slot.get('complexity_kind', 'pipeline_module')}") - prompt_lines.append(f"Image prompt core: {slot.get('image_prompt_core', '')}") - prompt_lines.append("") - - for candidate_index in range(1, candidates_per_slot + 1): - prompt = build_slot_prompt(slot, style, candidate_index=candidate_index) - candidate_path = slot_candidate_dir / f"candidate_{candidate_index:02d}.png" - prompt_lines.extend([f"### Candidate {candidate_index}", prompt, ""]) - task = { - "slot": slot, - "style": style, - "path": candidate_path, - "asset_mode": asset_mode, - "candidate_index": candidate_index, - "prompt": prompt, - "asset_retries": asset_retries, - } - tasks.append(task) - ordered_slot_candidates[slot["id"]].append(task) - - reports_by_key: dict[tuple[str, int], dict] = {} - if asset_workers == 1 or len(tasks) <= 1: - for task in tasks: - report = _generate_and_score_candidate(task) - reports_by_key[(task["slot"]["id"], task["candidate_index"])] = report - else: - with ThreadPoolExecutor(max_workers=asset_workers) as executor: - futures = {executor.submit(_generate_and_score_candidate, task): task for task in tasks} - for future in as_completed(futures): - task = futures[future] - reports_by_key[(task["slot"]["id"], task["candidate_index"])] = future.result() - - for slot in program["slots"]: - candidate_reports = [ - reports_by_key[(slot["id"], candidate_index)] - for candidate_index in range(1, candidates_per_slot + 1) - ] - successful_reports = [item for item in candidate_reports if Path(item.get("path", "")).exists() and "generation_failed" not in item.get("issue_tags", [])] - all_candidate_paths.extend(Path(item["path"]) for item in successful_reports) - if not successful_reports: - errors = "; ".join(item.get("error", "unknown generation failure") for item in candidate_reports) - raise RuntimeError(f"All candidate generations failed for slot {slot['id']}: {errors}") - selected = _select_candidate(candidate_reports) - selected_path = Path(selected["path"]) - final_asset_path = assets_dir / f"{slot['asset_id']}.png" - shutil.copyfile(selected_path, final_asset_path) - selected_paths.append(final_asset_path) - - selected_action = "ok_no_crop" if not selected.get("issue_tags") else "selected_best_but_needs_regeneration" - selected_reason = ( - "Selected because it had the best combined fill, margin, ratio, and visual-complexity score." - if not selected.get("issue_tags") - else "Selected as the least-bad candidate but validation should fail until regenerated." - ) - report_items.append({ - "asset_id": slot["asset_id"], - "slot_id": slot["id"], - "path": str(final_asset_path), - "selected_candidate_index": selected["candidate_index"], - "selected_candidate_path": selected["path"], - "source": selected["source"], - "content_fill_percent": selected["content_fill_percent"], - "min_content_fill_percent": slot["min_content_fill_percent"], - "empty_margin_percent": selected["empty_margin_percent"], - "max_empty_margin_percent": slot["max_empty_margin_percent"], - "ratio_mismatch_percent": selected["ratio_mismatch_percent"], - "edge_cutoff_status": selected["edge_cutoff_status"], - "ratio_status": selected["ratio_status"], - "issue_tags": selected.get("issue_tags", []), - "action": selected_action, - "detail_score": selected.get("detail_score"), - "detail_score_target": selected.get("detail_score_target"), - "object_count_estimate": selected.get("object_count_estimate"), - "object_count_target": selected.get("object_count_target"), - "simple_icon_risk": selected.get("simple_icon_risk"), - "reference_crop_match": selected.get("reference_crop_match"), - "style_match": selected.get("style_match"), - "complexity_issue_tags": selected.get("complexity_issue_tags", []), - "selected_reason": selected_reason, - "notes": "Selected from candidates; no semantic crop used; inserted into PPT with contain-fit.", - "rejected_candidates": [item for item in candidate_reports if item["candidate_index"] != selected["candidate_index"]], - }) - complexity_items.append({ - "asset_id": slot["asset_id"], - "slot_id": slot["id"], - "selected_candidate_index": selected["candidate_index"], - "selected_candidate_path": selected["path"], - "complexity_kind": selected.get("complexity_kind") or slot.get("complexity_kind"), - "required_visual_complexity": selected.get("required_visual_complexity") or slot.get("required_visual_complexity"), - "detail_score": selected.get("detail_score"), - "detail_score_target": selected.get("detail_score_target"), - "object_count_estimate": selected.get("object_count_estimate"), - "object_count_target": selected.get("object_count_target"), - "simple_icon_risk": selected.get("simple_icon_risk"), - "reference_crop_match": selected.get("reference_crop_match"), - "style_match": selected.get("style_match"), - "complexity_issue_tags": selected.get("complexity_issue_tags", []), - "selected_reason": selected_reason, - "candidate_complexity": [ - { - "candidate_index": item.get("candidate_index"), - "path": item.get("path"), - "detail_score": item.get("detail_score"), - "object_count_estimate": item.get("object_count_estimate"), - "simple_icon_risk": item.get("simple_icon_risk"), - "reference_crop_match": item.get("reference_crop_match"), - "style_match": item.get("style_match"), - "complexity_issue_tags": item.get("complexity_issue_tags", []), - "selection_score": item.get("selection_score"), - } - for item in candidate_reports - ], - }) - - write_text(out / "prompts.md", "\n".join(prompt_lines)) - report = { - "summary": "Asset quality report for multi-candidate slot-level generated blocks.", - "candidate_policy": { - "candidates_per_slot": candidates_per_slot, - "selection": "prefer passing candidates; otherwise choose highest fill/margin/ratio score and fail validation if below thresholds", - "no_crop": True, - }, - "assets": report_items, - } - write_json(out / "asset_quality_report.json", report) - complexity_report = { - "summary": "Asset complexity report for selected slot-level generated blocks.", - "policy": "Reject normal non-legend assets that look like sparse generic icons, ignore the local crop, or lack layered visual detail.", - "assets": complexity_items, - } - write_json(out / "asset_complexity_report.json", complexity_report) - _make_contact_sheet(selected_paths, out / "asset_contact_sheet.png", title="selected slot assets") - _make_contact_sheet(all_candidate_paths, out / "asset_candidate_contact_sheet.png", title="all generated candidates") - return report +_sys.modules[__name__] = _implementation diff --git a/rfs/asset_reviewer.py b/rfs/asset_reviewer.py index b553f90..cf14073 100644 --- a/rfs/asset_reviewer.py +++ b/rfs/asset_reviewer.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import hashlib import json import os import re @@ -12,6 +13,23 @@ from .utils import write_json +ASSET_REVIEW_CONTRACT_VERSION = "vlm-asset-review-v2" + + +def _critic_model_name(model: str | None = None) -> str: + return model or os.getenv("RFS_CRITIC_MODEL") or os.getenv("MODEL_VLM") or "gemini-3-pro-preview-thinking" + + +def _sha256_file(path: Path) -> str | None: + if not path.exists() or not path.is_file(): + return None + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def _extract_json(text: str) -> dict: cleaned = text.strip().replace("```json", "```") if cleaned.startswith("```"): @@ -29,7 +47,7 @@ def _extract_json(text: str) -> dict: def _call_vlm_json(prompt: str, image_paths: list[str | Path], model: str | None = None) -> dict: api_base = os.getenv("API_BASE", "").rstrip("/") api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") - model_name = model or os.getenv("RFS_CRITIC_MODEL") or os.getenv("MODEL_VLM") or "gemini-3-pro-preview-thinking" + model_name = _critic_model_name(model) if not api_base or not api_key: raise RuntimeError("VLM review requires API_BASE and API_KEY/GEMINI_API_KEY environment variables") @@ -69,6 +87,20 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non for item in complexity_report.get("assets", []) if isinstance(item, dict) and item.get("slot_id") } + prompt_plan_by_slot = {} + try: + prompt_plan = json.loads((out / "slot_prompt_plan.json").read_text(encoding="utf-8")) + prompt_plan_by_slot = { + str(item.get("slot_id")): item + for item in prompt_plan.get("slots", []) + if isinstance(item, dict) and item.get("slot_id") + } + except Exception: + prompt_plan_by_slot = {} + paper_native = any( + str(item.get("reference_content_priority") or "").casefold() == "paper_semantic_contract" + for item in prompt_plan_by_slot.values() + ) if mode == "off": review = {"summary": "Asset visual review skipped by configuration.", "mode": "off", "issues": []} @@ -106,6 +138,10 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non "detail_score": complexity_by_slot.get(str(item.get("slot_id")), {}).get("detail_score"), "simple_icon_risk": complexity_by_slot.get(str(item.get("slot_id")), {}).get("simple_icon_risk"), "complexity_issue_tags": complexity_by_slot.get(str(item.get("slot_id")), {}).get("complexity_issue_tags"), + "paper_concept": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("paper_concept"), + "must_show": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("must_show"), + "avoid_showing": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("avoid_showing"), + "semantic_content_authority": prompt_plan_by_slot.get(str(item.get("slot_id")), {}).get("reference_content_priority"), } for item in report.get("assets", []) ] @@ -117,7 +153,7 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non Check for these issues: - too_simple: looks like a generic sparse icon instead of a rich research visual block - generic_icon: looks like a standalone pictogram rather than a reference-guided mini scientific scene/card -- reference_crop_ignored: does not visually resemble the local reference crop object +- {"paper_semantic_mismatch: visual content does not plausibly represent the stated paper concept or must_show items" if paper_native else "reference_crop_ignored: does not visually resemble the local reference crop object"} - single_object_on_blank_background: one object floats on a plain canvas - style_drift: local color/style does not match the reference figure - too_sparse: too much empty canvas or subject too small @@ -126,6 +162,8 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non - style_drift: style inconsistent across blocks - low_resolution: blur or artifacts +{"Paper-native policy: generated seed crops contain no scientific objects and must not be used for content comparison. Judge each asset against paper_concept, must_show, and avoid_showing. Do not penalize the collection merely for containing the mandatory 25-50 raster slot assets." if paper_native else "Reference-primary policy: compare visual objects and local style against the user reference crop contract."} + Return schema: {{ "summary": "...", @@ -138,9 +176,41 @@ def review_assets(out_dir: str | Path, mode: str = "heuristic", model: str | Non Asset list: {json.dumps(assets_brief, ensure_ascii=False)} """.strip() + fingerprint_assets = [] + for item in report.get("assets", []): + asset_path = Path(str(item.get("path") or "")) + if not asset_path.exists() and item.get("asset_id"): + asset_path = out / "assets" / f"{item.get('asset_id')}.png" + fingerprint_assets.append({ + "asset_id": item.get("asset_id"), + "slot_id": item.get("slot_id"), + "sha256": _sha256_file(asset_path), + "brief": next((entry for entry in assets_brief if entry.get("slot_id") == item.get("slot_id")), None), + }) + review_fingerprint = hashlib.sha256(json.dumps({ + "contract_version": ASSET_REVIEW_CONTRACT_VERSION, + "model": _critic_model_name(model), + "paper_native": paper_native, + "assets": fingerprint_assets, + }, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest() + existing_path = out / "asset_visual_review.json" + if existing_path.exists(): + try: + existing = json.loads(existing_path.read_text(encoding="utf-8")) + if existing.get("status") == "pass" and existing.get("fingerprint") == review_fingerprint: + review = dict(existing) + review["cache_hit"] = True + write_json(existing_path, review) + return review + except Exception: + pass review = _call_vlm_json(prompt, [contact_sheet], model=model) review.setdefault("summary", "VLM asset visual review for selected slot blocks.") review.setdefault("mode", "vlm") + review.setdefault("status", "needs_regeneration" if review.get("issues") else "pass") + review["model"] = _critic_model_name(model) + review["fingerprint"] = review_fingerprint + review["cache_hit"] = False else: raise ValueError(f"Unsupported asset review mode: {mode}") diff --git a/rfs/cli.py b/rfs/cli.py index f3ab372..26a741f 100644 --- a/rfs/cli.py +++ b/rfs/cli.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from __future__ import annotations import argparse import importlib.util @@ -11,19 +11,25 @@ from . import __version__ from .coevolution import analyze_coevolution_run, run_image_coevolution -from .editable_rebuild import rebuild_editable +from .paper_to_image import analyze_reference_figures, inspect_paper, learn_reference_style, record_paper_image_preference, run_fast_framework_prompt, run_paper_to_image, select_reference_exemplars from .evaluation import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, run_pdf_extraction_stress_suite, score_benchmark_case, validate_benchmark_case -from .paper_to_image import inspect_paper, run_fast_framework_prompt, run_paper_to_image -from .workflows import run_paper_to_editable -from .professional_rebuild import rebuild_editable_pro -from .professional_repair import vlm_professional_repair_adapter +from .workflows import rebuild_editable, rebuild_editable_pro, run_paper_to_editable +from .professional.repair import vlm_professional_repair_adapter from .presentations_qa import run_presentations_qa -from .rebuild_vlm_adapters import build_rebuild_vlm_adapters +from .providers import build_rebuild_vlm_adapters from .rebuild_eval import evaluate_rebuild_vlm -from .reference_text_extractor import _rapidocr_detector_limit +from .rebuild_evolution import run_rebuild_evolution +from .rebuild_judge import run_rebuild_judge +from .analysis.rebuild.reference_text import _rapidocr_detector_limit from .utils import env_present, mask_secret -from .validator import validate_output +from .evaluation import validate_output from .workflow import make_framework +from .cli_commands.rebuild import add_rebuild_args, build_rebuild_kwargs, register_professional_rebuild, register_rebuild_core_commands +from .cli_commands.categories import EXPERIMENTAL_COMMANDS, MAINTENANCE_COMMANDS, PRODUCT_COMMANDS +from .cli_commands.maintenance import register_quality_commands, register_reference_and_inspection_commands +from .cli_commands.experimental import register_coevolution_commands, register_judge_command, register_judge_data_commands +from .judge_training import build_dataset, validate_dataset, register_checkpoint, evaluate_predictions, compare_shadow_predictions +from .cli_commands.product import register_fast_framework_prompt, register_make_framework, register_paper_to_editable, register_paper_to_image def _json_print(data: dict) -> None: @@ -135,243 +141,86 @@ def _doctor() -> dict: } +def _add_rebuild_args(parser: argparse.ArgumentParser) -> None: + """Shared rebuild-editable arguments, also reused by rebuild-evolve.""" + return add_rebuild_args(parser) + """ + parser.add_argument("--reference", required=True, help="Reference image path.") + parser.add_argument("--out", required=True, help="Output directory.") + parser.add_argument("--asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api", help="Slot asset source. Policy follows generation-plan per-slot decisions.") + parser.add_argument("--asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") + parser.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") + parser.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") + parser.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") + parser.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") + parser.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") + parser.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") + parser.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM layout disagreements.") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM control disagreements.") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for visible controls missed after adjudication.") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for missed layout objects and controls.") + parser.add_argument("--vlm-timeout", type=int, help="Timeout in seconds for rebuild VLM calls. Defaults to RFS_VLM_TIMEOUT or 240.") + parser.add_argument("--vlm-retries", type=int, help="Retry count for rebuild VLM calls. Defaults to RFS_VLM_RETRIES or 2.") + parser.add_argument("--rebuild-critic-mode", choices=["off", "heuristic", "vlm"], default="off", help="Compiled-preview rebuild critic mode. Default: off.") + parser.add_argument("--rebuild-critic-iterations", type=int, default=0, help="Compiled-preview critic correction iterations, clamped to 0-3. Default: 0.") + parser.add_argument("--rebuild-critic-model", help="Optional VLM model for --rebuild-critic-mode vlm. Defaults to RFS_REBUILD_CRITIC_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") + parser.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") + parser.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") + parser.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") + parser.add_argument("--from-stage", help="Re-run from this stage onward (design_plan/build_program/localize_controls/text_layer/slot_semantics); stages before it reuse snapshots.") + parser.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle; easyocr remains available.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + """ + + def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="rfs", description="ResearchFigureStudio CLI") + parser = argparse.ArgumentParser( + prog="rfs", + description="ResearchFigureStudio CLI", + epilog=( + f"Product: {', '.join(PRODUCT_COMMANDS)}. " + f"Experimental: {', '.join(EXPERIMENTAL_COMMANDS)}. " + f"Maintenance/QA: {', '.join(MAINTENANCE_COMMANDS)}." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.") sub = parser.add_subparsers(dest="command", required=True) - doctor = sub.add_parser("doctor", help="Check dependencies, PowerPoint, and auth env vars.") - doctor.add_argument("--json", action="store_true", help="Emit JSON.") - - inspect_pdf = sub.add_parser("inspect-pdf", help="Extract and diagnose a paper without calling a model.") - inspect_pdf.add_argument("--paper", required=True, help="Paper PDF or supported document path.") - inspect_pdf.add_argument("--out", required=True, help="Output directory for the document model and extraction report.") - inspect_pdf.add_argument("--deadline", type=int, default=180, help="Soft processing deadline in seconds. Default: 180.") - inspect_pdf.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") - inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") - inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") - - fast_prompt = sub.add_parser("fast-framework-prompt", help="Compile a paper-grounded framework prompt without generating images.") - fast_prompt.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") - fast_prompt.add_argument("--out", required=True, help="Output directory for evidence, semantic contract, prompt, and overlay specification.") - fast_prompt.add_argument("--deadline", type=int, default=180, help="Soft end-to-end deadline in seconds. Default: 180.") - fast_prompt.add_argument("--preferences", help="Optional JSON style and output preferences.") - fast_prompt.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") - fast_prompt.add_argument("--planner-model") - fast_prompt.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") - fast_prompt.add_argument("--aspect-ratio", default="16:9") - fast_prompt.add_argument("--language", default="English") - fast_prompt.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") - fast_prompt.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") - fast_prompt.add_argument("--editable-ppt", action="store_true", help="Also compile the semantic contract into native editable PowerPoint shapes, text, and connectors.") - fast_prompt.add_argument("--verify-ppt-roundtrip", action="store_true", help="Compile the editable PPTX, render it through PowerPoint, and write ppt_roundtrip_report.json.") - fast_prompt.add_argument("--json", action="store_true", help="Emit JSON.") - - make = sub.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") - make.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") - make.add_argument("--reference", required=True, help="User-provided visual reference image path.") - make.add_argument("--out", required=True, help="Output directory.") - contract_source = make.add_mutually_exclusive_group() - contract_source.add_argument("--paper-contract-dir", help="Fast paper-to-image/fast-framework output directory containing figure_specification.json. Bypasses the legacy paper planner.") - contract_source.add_argument("--semantic-contract", help="Semantic contract JSON or figure_specification.json. Bypasses the legacy paper planner.") - make.add_argument("--profile", default="ai-ml-paper", help="Figure profile. Default: ai-ml-paper.") - make.add_argument("--slot-count", type=int, default=36, help="Target slot count, clamped to 25-50. Default: 36.") - make.add_argument("--slot-source", choices=["paper", "reference-primary"], default="reference-primary", help="Slot content source. Default reference-primary makes the reference figure drive visual objects, layout, color, and flow logic.") - make.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset generation mode. Default image2 uses Yunwu Images API.") - make.add_argument("--candidates-per-slot", type=int, default=4, help="Candidate images per slot, clamped to 1-5. Default: 4.") - make.add_argument("--asset-workers", type=int, default=1, help="Parallel asset generation workers, clamped to 1-12. Default: 1.") - make.add_argument("--asset-retries", type=int, default=2, help="Retries per generated asset candidate, clamped to 0-5. Default: 2.") - make.add_argument("--asset-review-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Selected asset review mode. Default: heuristic.") - make.add_argument("--locator-mode", choices=["heuristic", "vlm"], default="heuristic", help="Layout coordinate source. Use vlm to locate slots from the reference image.") - make.add_argument("--locator-model", help="Optional VLM model for --locator-mode vlm. Defaults to RFS_LOCATOR_MODEL/MODEL_VLM.") - make.add_argument("--control-localizer-mode", choices=["off", "heuristic", "hybrid"], default="hybrid", help="Arrow/connector localization mode. Default hybrid uses CV candidates plus optional VLM binding; falls back to heuristic without API keys.") - make.add_argument("--arrow-style-mode", choices=["off", "reference", "aesthetic"], default="reference", help="Arrow styling/routing mode. Default reference preserves reference-image routes and adds softer PPT styling/QA metadata.") - make.add_argument("--prompt-plan-mode", choices=["heuristic", "vlm", "vlm-batch"], default="vlm", help="Reference-aware per-slot prompt planning mode. Default vlm uses one VLM call per slot; vlm-batch uses one batch VLM call; heuristic is offline only.") - make.add_argument("--prompt-plan-model", help="Optional VLM model for --prompt-plan-mode vlm/vlm-batch. Defaults to RFS_PROMPT_PLANNER_MODEL/MODEL_VLM.") - make.add_argument("--prompt-plan-workers", type=int, default=4, help="Parallel VLM workers for per-slot prompt planning, clamped to 1-12. Default: 4.") - make.add_argument("--complexity-profile", choices=["reference-dense", "balanced", "legend-simple"], default="reference-dense", help="Visual complexity policy for slot_visual_spec.json. Default reference-dense.") - make.add_argument("--critic-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Final reference-vs-render critic mode. Default: heuristic.") - make.add_argument("--critic-model", help="Optional VLM model for asset review and final critic. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") - make.add_argument("--critic-iterations", type=int, default=0, help="VLM layout correction iterations, clamped to 0-3. Default: 0.") - make.add_argument("--text-extractor-mode", choices=["heuristic", "ocr"], default="ocr", help="Editable text layer source. Default ocr uses local OCR when available and falls back to heuristic.") - make.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle.") - make.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - make.add_argument("--presentations-qa", action="store_true", help="Run optional Presentations plugin import/render/layout QA after export. This never mutates the PPTX.") - make.add_argument("--presentations-workspace", help="Optional workspace for Presentations QA scratch artifacts.") - make.add_argument("--presentations-scale", type=int, default=2, help="Preview render scale for Presentations QA. Default: 2.") - make.add_argument("--no-export", action="store_true", help="Skip PDF/PNG export and only create PPTX/artifacts.") - make.add_argument("--json", action="store_true", help="Emit JSON.") - - paper_image = sub.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate reviewed raster candidates with an optional editable PPT overlay.") - paper_image.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") - paper_image.add_argument("--out", required=True, help="Output directory for review, templates, prompts, candidates, and production-only selected_image.png.") - paper_image.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") - paper_image.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") - paper_image.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") - paper_image.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") - paper_image.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") - paper_image.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") - paper_image.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") - paper_image.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") - paper_image.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") - paper_image.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") - paper_image.add_argument("--language", default="English", help="Visible label language. Default: English.") - paper_image.add_argument("--image-model", help="Optional image model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") - paper_image.add_argument("--image-retries", type=int, default=1, help="Retries per image candidate for retryable HTTP/provider errors, clamped to 0-5. Network timeouts are not blindly retried. Default: 1.") - paper_image.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") - paper_image.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - paper_image.add_argument("--repair-rounds", type=int, default=3, help="Localized Image2 edit repair rounds, clamped to 0-4 with no-regression acceptance. Default: 3.") - paper_image.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") - paper_image.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") - paper_image.add_argument("--editable-overlay-ppt", action="store_true", help="Place the selected geometry-normalized Image2 visual substrate beneath native editable PPT labels and connectors. Requires deterministic overlay mode and a raw candidate that passes substrate geometry normalization.") - paper_image.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") - paper_image.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - paper_image.add_argument("--json", action="store_true", help="Emit JSON.") - - paper_editable = sub.add_parser("paper-to-editable", help="Generate a paper-grounded visual reference and rebuild it as an editable PowerPoint figure.") - paper_editable.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") - paper_editable.add_argument("--out", required=True, help="Output directory for paper analysis, image candidates, semantic contracts, and editable PPTX.") - paper_editable.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") - paper_editable.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") - paper_editable.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") - paper_editable.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") - paper_editable.add_argument("--planner-model") - paper_editable.add_argument("--image-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") - paper_editable.add_argument("--image-candidates", type=int, default=3) - paper_editable.add_argument("--aspect-ratio", default="auto") - paper_editable.add_argument("--language", default="English") - paper_editable.add_argument("--image-model") - paper_editable.add_argument("--image-retries", type=int, default=2) - paper_editable.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") - paper_editable.add_argument("--review-model") - paper_editable.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") - paper_editable.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") - paper_editable.add_argument("--repair-rounds", type=int, default=3) - paper_editable.add_argument("--editable-route", choices=["rebuild", "framework"], default="rebuild", help="Editable compiler. Use framework for the authoritative 25-50 slot paper-contract route; rebuild preserves the legacy image reconstruction route.") - paper_editable.add_argument("--framework-slot-count", type=int, default=25, help="Non-arrow slot count for --editable-route framework. Default: 25.") - paper_editable.add_argument("--framework-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset backend for --editable-route framework.") - paper_editable.add_argument("--framework-candidates-per-slot", type=int, default=1, help="Asset candidates per slot for --editable-route framework. Default: 1.") - paper_editable.add_argument("--framework-asset-workers", type=int, default=3, help="Parallel slot generation workers for --editable-route framework. Default: 3.") - paper_editable.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder"], default="api") - paper_editable.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api"], default="smart-api") - paper_editable.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") - paper_editable.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") - paper_editable.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") - paper_editable.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") - paper_editable.add_argument("--allow-engineering-preview", action="store_true", help="Allow offline/non-production image previews to exercise the editable pipeline. Never enabled by default.") - paper_editable.add_argument("--no-export-preview", action="store_true") - paper_editable.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto") - paper_editable.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") - paper_editable.add_argument("--json", action="store_true", help="Emit JSON.") - - rebuild = sub.add_parser("rebuild-editable", help="Rebuild a reference image into a reusable editable PowerPoint composition.") - rebuild.add_argument("--reference", required=True, help="Reference image path.") - rebuild.add_argument("--out", required=True, help="Output directory.") - rebuild.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") - rebuild.add_argument("--asset-policy", choices=["legacy", "smart-api"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") - rebuild.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") - rebuild.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") - rebuild.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") - rebuild.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") - rebuild.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") - rebuild.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") - rebuild.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") - rebuild.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") - rebuild.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") - rebuild.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") - rebuild.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") - rebuild.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") - rebuild.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") - rebuild.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") - rebuild.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") - rebuild.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") - rebuild.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") - rebuild.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - rebuild.add_argument("--json", action="store_true", help="Emit JSON.") - - rebuild_eval = sub.add_parser("rebuild-editable-eval", help="Compare heuristic and hybrid VLM rebuild-editable outputs for one reference image.") - rebuild_eval.add_argument("--reference", required=True, help="Reference image path.") - rebuild_eval.add_argument("--out", required=True, help="Evaluation output directory.") - rebuild_eval.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="crop", help="Asset mode for both cases. Default: crop to avoid image generation cost.") - rebuild_eval.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Text extraction mode for both cases. Default: ocr.") - rebuild_eval.add_argument("--export-preview", action="store_true", help="Export PNG previews when PowerPoint is available.") - rebuild_eval.add_argument("--json", action="store_true", help="Emit JSON.") - - rebuild_pro = sub.add_parser("rebuild-editable-pro", help="Rebuild a reference image through a VLM-authored controlled professional DSL.") - rebuild_pro.add_argument("--reference", required=True, help="Reference image path.") - rebuild_pro.add_argument("--out", required=True, help="Output directory.") - rebuild_pro.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="api", help="Slot asset source. Default api uses GEMINI_GEN_IMG_URL.") - rebuild_pro.add_argument("--asset-policy", choices=["legacy", "smart-api"], default="smart-api", help="Slot asset decision policy. Default smart-api disables final crop assets, filters text slots, and reuses duplicate API assets.") - rebuild_pro.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") - rebuild_pro.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") - rebuild_pro.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") - rebuild_pro.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") - rebuild_pro.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") - rebuild_pro.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Baseline layout extraction mode before professional DSL planning. Default: hybrid.") - rebuild_pro.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Baseline control extraction mode before professional DSL planning. Default: hybrid.") - rebuild_pro.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") - rebuild_pro.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") - rebuild_pro.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") - rebuild_pro.add_argument("--compile-only", action="store_true", help="Compile from existing professional_rebuild_script.dsl.json without rerunning analysis or VLM planning.") - rebuild_pro.add_argument("--repair-rounds", type=int, default=2, help="Preview repair rounds to record/run. V1 records conservative no-mutation repair reports by default.") - rebuild_pro.add_argument("--repair-mode", choices=["report", "vlm"], default="report", help="Professional repair mode. report records rounds without mutation; vlm applies controlled DSL patches.") - rebuild_pro.add_argument("--benchmark-out", help="Optional specialized rebuild output directory to compare against in professional_gap_report.json.") - rebuild_pro.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle.") - rebuild_pro.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") - rebuild_pro.add_argument("--json", action="store_true", help="Emit JSON.") - - coevolve = sub.add_parser("coevolve-image", help="Refine a complete scientific image through Creator Agent and Online/Frozen Judges.") - coevolve.add_argument("--ground-truth", required=True, help="Structured Ground Truth JSON containing paper facts and human aesthetic preferences.") - coevolve.add_argument("--out", required=True, help="Output directory for rounds, training trajectories, and approved_image.png.") - coevolve.add_argument("--candidates", type=int, default=3, help="Initial whole-image candidate count. Default: 3.") - coevolve.add_argument("--repair-candidates", type=int, default=2, help="Candidate count for each repair round. Default: 2.") - coevolve.add_argument("--max-rounds", type=int, default=4, help="Maximum total rounds including initial generation. Default: 4.") - coevolve.add_argument("--online-judge-model", help="Online feedback Judge model. Defaults to RFS_ONLINE_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - coevolve.add_argument("--frozen-judge-model", help="Independent acceptance Judge model. Defaults to RFS_FROZEN_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") - coevolve.add_argument("--image-model", help="Whole-image generation model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") - coevolve.add_argument("--image-retries", type=int, default=2, help="Image request retries. Default: 2.") - coevolve.add_argument("--json", action="store_true", help="Emit JSON.") - - coevolution_report = sub.add_parser("coevolution-report", help="Aggregate a co-evolution run into reproducible improvement metrics.") - coevolution_report.add_argument("--run", required=True, help="Existing co-evolution output directory.") - coevolution_report.add_argument("--json", action="store_true", help="Emit JSON.") - - benchmark = sub.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") - benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score", "audit"]) - benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") - benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") - benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") - benchmark.add_argument("--case-id", action="append", default=[], help="Optional case id filter for fast-suite. Repeat to select multiple cases.") - benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") - benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") - benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") - benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") - benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") - benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") - benchmark.add_argument("--review-model", help="Optional frozen VLM model for benchmark audit.") - benchmark.add_argument("--candidate-image", help="Optional explicit image to re-audit inside an existing paper-to-image run.") - benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") - benchmark.add_argument("--rasterize-dpi", type=int, help="For benchmark fast/fast-suite, convert local PDF inputs to image-only scanned PDFs at this DPI before extraction.") - benchmark.add_argument("--json", action="store_true", help="Emit JSON.") - - validate = sub.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") - validate.add_argument("--out", required=True, help="Output directory to validate.") - validate.add_argument("--json", action="store_true", help="Emit JSON.") - - presentations_qa = sub.add_parser("presentations-qa", help="Run optional Presentations plugin QA for an existing RFS output directory.") - presentations_qa.add_argument("--out", required=True, help="RFS output directory containing editable_composition.pptx.") - presentations_qa.add_argument("--pptx", help="Optional PPTX path. Defaults to /editable_composition.pptx.") - presentations_qa.add_argument("--workspace", help="Optional Presentations QA workspace. Defaults to /presentations_plugin_qa_workspace.") - presentations_qa.add_argument("--scale", type=int, default=2, help="Preview render scale. Default: 2.") - presentations_qa.add_argument("--skip-inspect", action="store_true", help="Write report from PPTX/package metadata without invoking the Presentations plugin.") - presentations_qa.add_argument("--json", action="store_true", help="Emit JSON.") + register_reference_and_inspection_commands(sub) + + register_fast_framework_prompt(sub) + + register_make_framework(sub) + + register_paper_to_image(sub) + + register_paper_to_editable(sub) + + register_rebuild_core_commands(sub) + + register_professional_rebuild(sub) + + register_coevolution_commands(sub) + + register_quality_commands(sub) + register_judge_command(sub) + register_judge_data_commands(sub) return parser def _print_human(data: dict) -> None: if "ok" in data: print(f"ok: {data['ok']}") - for key in ["out_dir", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_policy", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "control_mode", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: + for key in ["out_dir", "records_path", "library_path", "report_path", "record_path", "selection_path", "brief_path", "style_reference", "user_style_brief", "combined_style_brief", "run_path", "chosen_candidate_id", "chosen_path", "preference_pairs_path", "feedback_path", "pair_count", "appended_pair_count", "record_count", "positive_count", "negative_count", "input_count", "scanned_file_count", "skipped_count", "failure_count", "analysis_mode", "model", "reference_library", "reference_top_k", "selected_image", "engineering_preview", "candidate_count", "selected_candidate_id", "selected_passed_all_checks", "planner_mode", "planner_model", "paper_review_mode", "domain_profile", "template_id", "review_mode", "approved_image", "thresholds_met", "stop_reason", "rounds_completed", "online_judge_model", "frozen_judge_model", "weak_judge_isolation", "pptx", "pdf", "png", "preview", "asset_count", "slot_count", "slot_source", "asset_mode", "asset_policy", "asset_workers", "asset_retries", "economy_mode", "api_requests_attempted", "text_count", "connector_count", "text_mode", "layout_mode", "layout_adjudication_mode", "layout_adjudication_status", "control_mode", "control_adjudication_mode", "control_adjudication_status", "control_coverage_audit_mode", "control_coverage_audit_status", "combined_coverage_audit_mode", "combined_coverage_audit_status", "combined_missing_layout_count", "combined_missing_control_count", "vlm_timeout", "vlm_retries", "rebuild_critic_mode", "rebuild_critic_iterations", "rebuild_critic_status", "rebuild_critic_model", "professional_mode", "repair_rounds", "planner_status", "compile_only", "candidates_per_slot", "asset_review_mode", "locator_mode", "control_localizer_mode", "arrow_style_mode", "prompt_plan_mode", "prompt_plan_workers", "complexity_profile", "critic_mode", "critic_iterations", "text_extractor_mode", "ocr_engine", "ocr_lang"]: if key in data: print(f"{key}: {data[key]}") if data.get("presentations_qa"): @@ -392,6 +241,11 @@ def _print_human(data: dict) -> None: print(f"warning: {warn}") +def _rebuild_kwargs(args) -> dict: + """Compatibility helper for callers importing the historical CLI symbol.""" + return build_rebuild_kwargs(args, build_rebuild_vlm_adapters) + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) json_requested = False @@ -405,6 +259,38 @@ def main(argv: list[str] | None = None) -> int: try: if args.command == "doctor": result = _doctor() + elif args.command == "analyze-reference-figures": + result = analyze_reference_figures( + references=args.references, + out=args.out, + mode=args.mode, + model=args.model, + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + ) + elif args.command == "select-reference-exemplars": + result = select_reference_exemplars( + paper_review=args.paper_review, + library=args.library, + out=args.out, + selected_template=args.selected_template, + top_k=args.top_k, + ) + elif args.command == "learn-reference-style": + result = learn_reference_style( + style_reference=args.style_reference, + out=args.out, + mode=args.mode, + model=args.model, + ) + elif args.command == "record-paper-image-preference": + result = record_paper_image_preference( + run=args.run, + chosen=args.chosen, + reason=args.reason, + reviewer=args.reviewer, + tags=args.tags, + ) elif args.command == "inspect-pdf": result = inspect_paper( paper=args.paper, @@ -470,6 +356,10 @@ def main(argv: list[str] | None = None) -> int: preferences_path=args.preferences, positive_references=args.positive_reference, negative_references=args.negative_reference, + reference_library=args.reference_library, + reference_top_k=args.reference_top_k, + user_style_brief=args.user_style_brief, + preference_history=args.preference_history, planner_mode=args.planner_mode, planner_model=args.planner_model, asset_mode=args.asset_mode, @@ -483,6 +373,7 @@ def main(argv: list[str] | None = None) -> int: domain_profile=args.domain_profile, template=args.template, repair_rounds=args.repair_rounds, + human_scientific_confirmed=args.human_scientific_confirmed, repair_source=args.repair_source, resume_candidates=args.resume_candidates, editable_overlay_ppt=args.editable_overlay_ppt, @@ -509,10 +400,20 @@ def main(argv: list[str] | None = None) -> int: domain_profile=args.domain_profile, template=args.template, repair_rounds=args.repair_rounds, + human_scientific_confirmed=args.human_scientific_confirmed, rebuild_asset_mode=args.rebuild_asset_mode, rebuild_asset_policy=args.rebuild_asset_policy, layout_mode=args.layout_mode, control_mode=args.control_mode, + layout_adjudication_mode=args.layout_adjudication_mode if args.layout_mode in {"vlm", "hybrid"} else "off", + control_adjudication_mode=args.control_adjudication_mode if args.control_mode in {"vlm", "hybrid"} else "off", + control_coverage_audit_mode=args.control_coverage_audit_mode if args.control_mode in {"vlm", "hybrid"} else "off", + combined_coverage_audit_mode=args.combined_coverage_audit_mode if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else "off", + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + rebuild_critic_mode=args.rebuild_critic_mode, + rebuild_critic_iterations=args.rebuild_critic_iterations, + rebuild_critic_model=args.rebuild_critic_model, text_mode=args.text_mode, design_plan_mode=args.design_plan_mode, export_preview=not args.no_export_preview, @@ -526,33 +427,19 @@ def main(argv: list[str] | None = None) -> int: framework_asset_workers=args.framework_asset_workers, ) elif args.command == "rebuild-editable": - rebuild_adapters = build_rebuild_vlm_adapters(args.out) result = rebuild_editable( reference=args.reference, out=args.out, - asset_mode=args.asset_mode, - asset_workers=args.asset_workers, - asset_retries=args.asset_retries, - economy_mode=args.economy_mode, - text_mode=args.text_mode, - control_mode=args.control_mode, - layout_mode=args.layout_mode, - export_preview=args.export_preview, - regenerate_slots=args.regenerate_slots, - strict_asset_regeneration=args.strict_asset_regeneration, - skip_analysis=args.skip_analysis, - compile_only=args.compile_only, - ocr_engine=args.ocr_engine, - ocr_lang=args.ocr_lang, - vlm_layout_adapter=rebuild_adapters["layout"] if args.layout_mode in {"vlm", "hybrid"} else None, - control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, - semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, - asset_policy=args.asset_policy, - design_plan_mode=args.design_plan_mode, - design_plan_model=args.design_plan_model, - design_adapter=rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, - text_grouping_mode=args.text_grouping_mode, - text_grouping_model=args.text_grouping_model, + **_rebuild_kwargs(args), + ) + elif args.command == "rebuild-evolve": + result = run_rebuild_evolution( + reference=args.reference, + out=args.out, + max_rounds=args.max_rounds, + judge_model=args.judge_model, + score_threshold=args.score_threshold, + **_rebuild_kwargs(args), ) elif args.command == "rebuild-editable-eval": result = evaluate_rebuild_vlm( @@ -563,10 +450,25 @@ def main(argv: list[str] | None = None) -> int: export_preview=args.export_preview, ) elif args.command == "rebuild-editable-pro": - rebuild_adapters = build_rebuild_vlm_adapters(args.out) + baseline_out = Path(args.out) / "professional_baseline" + rebuild_adapters = build_rebuild_vlm_adapters(baseline_out, vlm_timeout=args.vlm_timeout, vlm_retries=args.vlm_retries) + use_layout_vlm = args.layout_mode in {"vlm", "hybrid"} + use_control_vlm = args.control_mode in {"vlm", "hybrid"} + use_any_rebuild_vlm = use_layout_vlm or use_control_vlm + repair_adapter = None + if args.repair_mode == "vlm": + repair_adapter = lambda reference_path, preview_path, dsl, round_index: vlm_professional_repair_adapter( + reference_path, + preview_path, + dsl, + round_index, + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + ) result = rebuild_editable_pro( reference=args.reference, out=args.out, + baseline_out=baseline_out, asset_mode=args.asset_mode, asset_workers=args.asset_workers, asset_retries=args.asset_retries, @@ -581,10 +483,26 @@ def main(argv: list[str] | None = None) -> int: repair_rounds=args.repair_rounds, ocr_engine=args.ocr_engine, ocr_lang=args.ocr_lang, - vlm_layout_adapter=rebuild_adapters["layout"] if args.layout_mode in {"vlm", "hybrid"} else None, - control_adapter=rebuild_adapters["control"] if args.control_mode in {"vlm", "hybrid"} else None, - semantic_adapter=rebuild_adapters["semantic"] if args.layout_mode in {"vlm", "hybrid"} or args.control_mode in {"vlm", "hybrid"} else None, - repair_adapter=vlm_professional_repair_adapter if args.repair_mode == "vlm" else None, + vlm_layout_adapter=rebuild_adapters["layout"] if use_layout_vlm else None, + panel_layout_adapter=rebuild_adapters.get("panel_layout") if use_layout_vlm else None, + layout_adjudication_mode=args.layout_adjudication_mode if use_layout_vlm else "off", + layout_adjudicator=rebuild_adapters["layout_adjudicator"] if use_layout_vlm and args.layout_adjudication_mode == "vlm" else None, + control_adapter=rebuild_adapters["control"] if use_control_vlm else None, + control_adjudication_mode=args.control_adjudication_mode if use_control_vlm else "off", + control_adjudicator=rebuild_adapters["control_adjudicator"] if use_control_vlm and args.control_adjudication_mode == "vlm" else None, + control_coverage_audit_mode=args.control_coverage_audit_mode if use_control_vlm else "off", + coverage_audit_adapter=rebuild_adapters["coverage_audit"] if use_control_vlm and args.control_coverage_audit_mode == "vlm" else None, + combined_coverage_audit_mode=args.combined_coverage_audit_mode if use_any_rebuild_vlm else "off", + combined_coverage_audit_adapter=rebuild_adapters["combined_coverage_audit"] if use_any_rebuild_vlm and args.combined_coverage_audit_mode == "vlm" else None, + vlm_timeout=args.vlm_timeout, + vlm_retries=args.vlm_retries, + semantic_adapter=rebuild_adapters["semantic"] if use_any_rebuild_vlm else None, + design_plan_mode=args.design_plan_mode, + design_plan_model=args.design_plan_model, + design_adapter=rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, + text_grouping_mode=args.text_grouping_mode, + text_grouping_model=args.text_grouping_model, + repair_adapter=repair_adapter, benchmark_out=args.benchmark_out, asset_policy=args.asset_policy, ) @@ -648,6 +566,35 @@ def main(argv: list[str] | None = None) -> int: run_inspect=not args.skip_inspect, ) result["ok"] = result.get("presentations_plugin_qa", {}).get("status") not in {"failed"} + elif args.command == "rebuild-judge": + result = run_rebuild_judge( + out=args.out, + reference=args.reference, + preview=args.preview, + model=args.model, + timeout=args.timeout, + retries=args.retries, + repair_plan=args.repair_plan, + ) + elif args.command == "judge": + if args.judge_action == "register": + manifest = json.loads(Path(args.manifest).read_text(encoding="utf-8")) + result = register_checkpoint(args.checkpoint, manifest, args.registry) + result["ok"] = True + elif args.judge_action == "evaluate": + result = evaluate_predictions(args.dataset, args.predictions, checkpoint=args.checkpoint, suite=args.suite) + elif args.judge_action == "shadow": + result = compare_shadow_predictions(args.dataset, args.predictions, args.bootstrap_predictions, checkpoint=args.checkpoint) + else: + parser.error("judge requires register, evaluate, or shadow") + elif args.command == "judge-data": + if args.judge_data_action == "build": + result = build_dataset(args.runs, args.out, dataset_version=args.dataset_version) + result["ok"] = not bool(result.get("errors")) + elif args.judge_data_action == "validate": + result = validate_dataset(args.dataset) + else: + parser.error("judge-data requires build or validate") else: parser.error("unknown command") return 2 diff --git a/rfs/cli_commands/__init__.py b/rfs/cli_commands/__init__.py new file mode 100644 index 0000000..3bd72dc --- /dev/null +++ b/rfs/cli_commands/__init__.py @@ -0,0 +1,5 @@ +"""Command-specific CLI assembly helpers. + +The public entry point remains :mod:`rfs.cli`; this package only keeps +argument-to-workflow assembly out of the top-level parser module. +""" diff --git a/rfs/cli_commands/categories.py b/rfs/cli_commands/categories.py new file mode 100644 index 0000000..a6b91c6 --- /dev/null +++ b/rfs/cli_commands/categories.py @@ -0,0 +1,36 @@ +"""Stable CLI command classification used by help and navigation tooling.""" + +PRODUCT_COMMANDS = ( + "paper-to-image", + "paper-to-editable", + "rebuild-editable", + "make-framework", + "fast-framework-prompt", +) +EXPERIMENTAL_COMMANDS = ( + "rebuild-editable-pro", + "rebuild-evolve", + "rebuild-editable-eval", + "coevolve-image", + "coevolution-report", + "rebuild-judge", + "judge-data", + "judge", +) +MAINTENANCE_COMMANDS = ( + "doctor", + "inspect-pdf", + "benchmark", + "validate", + "presentations-qa", + "analyze-reference-figures", + "select-reference-exemplars", + "learn-reference-style", + "record-paper-image-preference", +) + +COMMAND_CATEGORIES = { + **{name: "product" for name in PRODUCT_COMMANDS}, + **{name: "experimental" for name in EXPERIMENTAL_COMMANDS}, + **{name: "maintenance" for name in MAINTENANCE_COMMANDS}, +} diff --git a/rfs/cli_commands/experimental.py b/rfs/cli_commands/experimental.py new file mode 100644 index 0000000..28734da --- /dev/null +++ b/rfs/cli_commands/experimental.py @@ -0,0 +1,65 @@ +"""Registration helpers for experimental CLI commands.""" + +from __future__ import annotations + + +def register_judge_command(subparsers) -> None: + rebuild_judge = subparsers.add_parser("rebuild-judge", help="Run the process-supervision Judge over a rebuild-editable output directory.") + rebuild_judge.add_argument("--out", required=True, help="Rebuild output directory containing run_log.jsonl and rebuild_result.json.") + rebuild_judge.add_argument("--reference", help="Optional reference image override. Defaults to rebuild_result.json.") + rebuild_judge.add_argument("--preview", help="Optional preview image override. Defaults to rebuild_result.json.") + rebuild_judge.add_argument("--model", help="Optional VLM model override.") + rebuild_judge.add_argument("--timeout", type=int, help="VLM timeout in seconds.") + rebuild_judge.add_argument("--retries", type=int, help="VLM retry count.") + rebuild_judge.add_argument("--repair-plan", action="store_true", help="Also write a v1 coarse repair plan mapping fault stages to actions.") + rebuild_judge.add_argument("--json", action="store_true", help="Emit JSON.") + judge = subparsers.add_parser("judge", help="Register and evaluate trained Judge checkpoints.") + actions = judge.add_subparsers(dest="judge_action", required=True) + register = actions.add_parser("register", help="Register a checkpoint manifest without loading weights.") + register.add_argument("--checkpoint", required=True) + register.add_argument("--manifest", required=True, help="JSON manifest containing model_id, dataset_version, roles, and git_commit.") + register.add_argument("--registry", required=True) + register.add_argument("--json", action="store_true") + evaluate = actions.add_parser("evaluate", help="Evaluate recorded predictions on a fixed dataset.") + evaluate.add_argument("--checkpoint") + evaluate.add_argument("--dataset", required=True) + evaluate.add_argument("--predictions", required=True) + evaluate.add_argument("--suite", default="hidden_test") + evaluate.add_argument("--json", action="store_true") + shadow = actions.add_parser("shadow", help="Compare trained predictions with Bootstrap predictions.") + shadow.add_argument("--checkpoint") + shadow.add_argument("--dataset", required=True) + shadow.add_argument("--predictions", required=True, help="Trained Judge prediction JSONL.") + shadow.add_argument("--bootstrap-predictions", required=True) + shadow.add_argument("--json", action="store_true") + + +def register_judge_data_commands(subparsers) -> None: + parser = subparsers.add_parser("judge-data", help="Build or validate Shadow-ready Judge training JSONL data.") + actions = parser.add_subparsers(dest="judge_data_action", required=True) + build = actions.add_parser("build", help="Build JudgeExample records from Paperimage evolution runs.") + build.add_argument("--runs", required=True, help="Run directory or root containing paper-to-editable outputs.") + build.add_argument("--out", required=True, help="Dataset directory or .jsonl output path.") + build.add_argument("--dataset-version", default="judge-dataset-v001") + build.add_argument("--json", action="store_true", help="Emit JSON.") + validate = actions.add_parser("validate", help="Validate a JudgeExample JSONL file or dataset directory.") + validate.add_argument("--dataset", required=True, help="Dataset directory or JSONL path.") + validate.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_coevolution_commands(subparsers) -> None: + parser = subparsers.add_parser("coevolve-image", help="Refine a complete scientific image through Creator Agent and Online/Frozen Judges.") + parser.add_argument("--ground-truth", required=True, help="Structured Ground Truth JSON containing paper facts and human aesthetic preferences.") + parser.add_argument("--out", required=True, help="Output directory for rounds, training trajectories, and approved_image.png.") + parser.add_argument("--candidates", type=int, default=3, help="Initial whole-image candidate count. Default: 3.") + parser.add_argument("--repair-candidates", type=int, default=2, help="Candidate count for each repair round. Default: 2.") + parser.add_argument("--max-rounds", type=int, default=4, help="Maximum total rounds including initial generation. Default: 4.") + parser.add_argument("--online-judge-model", help="Online feedback Judge model. Defaults to RFS_ONLINE_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--frozen-judge-model", help="Independent acceptance Judge model. Defaults to RFS_FROZEN_JUDGE_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--image-model", help="Whole-image generation model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") + parser.add_argument("--image-retries", type=int, default=2, help="Image request retries. Default: 2.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + report = subparsers.add_parser("coevolution-report", help="Aggregate a co-evolution run into reproducible improvement metrics.") + report.add_argument("--run", required=True, help="Existing co-evolution output directory.") + report.add_argument("--json", action="store_true", help="Emit JSON.") diff --git a/rfs/cli_commands/maintenance.py b/rfs/cli_commands/maintenance.py new file mode 100644 index 0000000..a3fa865 --- /dev/null +++ b/rfs/cli_commands/maintenance.py @@ -0,0 +1,80 @@ +"""Registration helpers for maintenance and reference-library commands.""" + +from __future__ import annotations + + +def register_reference_and_inspection_commands(subparsers) -> None: + doctor = subparsers.add_parser("doctor", help="Check dependencies, PowerPoint, and auth env vars.") + doctor.add_argument("--json", action="store_true", help="Emit JSON.") + + analyze_references = subparsers.add_parser("analyze-reference-figures", help="Analyze project reference figures into reusable design records.") + analyze_references.add_argument("--references", default="reference-figures", help="Reference figure root containing excellent/ and negative/. Default: reference-figures.") + analyze_references.add_argument("--out", default="output/reference_learning_stage1", help="Output directory for reference analysis JSONL, library JSONL, and report.") + analyze_references.add_argument("--mode", choices=["heuristic", "vlm"], default="heuristic", help="Reference analysis mode. Default: heuristic.") + analyze_references.add_argument("--model", default="gpt-5.6-sol", help="VLM model for --mode vlm. Default: gpt-5.6-sol.") + analyze_references.add_argument("--vlm-timeout", type=int, help="Optional VLM timeout in seconds. Defaults to RFS_VLM_TIMEOUT.") + analyze_references.add_argument("--vlm-retries", type=int, help="Optional VLM retry count. Defaults to RFS_VLM_RETRIES.") + analyze_references.add_argument("--json", action="store_true", help="Emit JSON.") + + select_exemplars = subparsers.add_parser("select-reference-exemplars", help="Select reference design exemplars for a paper review.") + select_exemplars.add_argument("--paper-review", required=True, help="paper_review.json for the current paper.") + select_exemplars.add_argument("--library", required=True, help="Reference library JSONL from analyze-reference-figures.") + select_exemplars.add_argument("--out", required=True, help="Output directory for exemplar_selection.json and exemplar_style_brief.json.") + select_exemplars.add_argument("--selected-template", help="Optional selected_template.json for archetype/aspect matching.") + select_exemplars.add_argument("--top-k", type=int, default=5, help="Positive exemplar count, clamped to 1-20. Default: 5.") + select_exemplars.add_argument("--json", action="store_true", help="Emit JSON.") + + learn_style = subparsers.add_parser("learn-reference-style", help="Learn reusable visual style preferences from one user style reference image.") + learn_style.add_argument("--style-reference", required=True, help="User-provided style reference image. Content is not reused.") + learn_style.add_argument("--out", required=True, help="Output directory for user_style_record.json, user_style_brief.json, and user_style_report.json.") + learn_style.add_argument("--mode", choices=["heuristic", "vlm"], default="vlm", help="User style learning mode. Default: vlm.") + learn_style.add_argument("--model", default="gpt-5.6-sol", help="VLM model for --mode vlm. Default: gpt-5.6-sol.") + learn_style.add_argument("--json", action="store_true", help="Emit JSON.") + + record_preference = subparsers.add_parser("record-paper-image-preference", help="Record human preference feedback for an existing paper-to-image run.") + record_preference.add_argument("--run", required=True, help="Existing paper-to-image output directory.") + record_preference.add_argument("--chosen", required=True, help="Preferred candidate_id, such as candidate_02 or repair_01.") + record_preference.add_argument("--reason", required=True, help="Why the chosen candidate is preferred.") + record_preference.add_argument("--reviewer", default="human", help="Reviewer name or role. Default: human.") + record_preference.add_argument("--tags", default="", help="Comma-separated preference tags, such as layout,hierarchy,color.") + record_preference.add_argument("--json", action="store_true", help="Emit JSON.") + + inspect_pdf = subparsers.add_parser("inspect-pdf", help="Extract and diagnose a paper without calling a model.") + inspect_pdf.add_argument("--paper", required=True, help="Paper PDF or supported document path.") + inspect_pdf.add_argument("--out", required=True, help="Output directory for the document model and extraction report.") + inspect_pdf.add_argument("--deadline", type=int, default=180, help="Soft processing deadline in seconds. Default: 180.") + inspect_pdf.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") + inspect_pdf.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + inspect_pdf.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_quality_commands(subparsers) -> None: + benchmark = subparsers.add_parser("benchmark", help="List, validate, run, or score ResearchFigureStudio benchmark cases.") + benchmark.add_argument("benchmark_action", choices=["list", "validate", "fetch", "fast", "fast-suite", "pdf-suite", "run", "score", "audit"]) + benchmark.add_argument("--suite", choices=["paper-to-image", "image-to-ppt"], help="Optional suite filter for list.") + benchmark.add_argument("--root", default="benchmarks", help="Benchmark root for list. Default: benchmarks.") + benchmark.add_argument("--case", help="Benchmark case directory for validate, fetch, run, or score.") + benchmark.add_argument("--case-id", action="append", default=[], help="Optional case id filter for fast-suite. Repeat to select multiple cases.") + benchmark.add_argument("--run", dest="run_dir", help="Existing workflow output directory for score.") + benchmark.add_argument("--out", help="Output directory for run, or optional score report destination.") + benchmark.add_argument("--force", action="store_true", help="Re-fetch an existing local benchmark paper input.") + benchmark.add_argument("--deadline", type=int, default=180, help="Soft deadline for benchmark fast. Default: 180.") + benchmark.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Planner mode for benchmark fast.") + benchmark.add_argument("--planner-model", help="Optional model for benchmark fast.") + benchmark.add_argument("--review-model", help="Optional frozen VLM model for benchmark audit.") + benchmark.add_argument("--candidate-image", help="Optional explicit image to re-audit inside an existing paper-to-image run.") + benchmark.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="off", help="Paper OCR mode for fast or pdf-suite benchmarks.") + benchmark.add_argument("--rasterize-dpi", type=int, help="For benchmark fast/fast-suite, convert local PDF inputs to image-only scanned PDFs at this DPI before extraction.") + benchmark.add_argument("--json", action="store_true", help="Emit JSON.") + + validate = subparsers.add_parser("validate", help="Validate an existing ResearchFigureStudio output directory.") + validate.add_argument("--out", required=True, help="Output directory to validate.") + validate.add_argument("--json", action="store_true", help="Emit JSON.") + + presentations_qa = subparsers.add_parser("presentations-qa", help="Run optional Presentations plugin QA for an existing RFS output directory.") + presentations_qa.add_argument("--out", required=True, help="RFS output directory containing editable_composition.pptx.") + presentations_qa.add_argument("--pptx", help="Optional PPTX path. Defaults to /editable_composition.pptx.") + presentations_qa.add_argument("--workspace", help="Optional Presentations QA workspace. Defaults to /presentations_plugin_qa_workspace.") + presentations_qa.add_argument("--scale", type=int, default=2, help="Preview render scale. Default: 2.") + presentations_qa.add_argument("--skip-inspect", action="store_true", help="Write report from PPTX/package metadata without invoking the Presentations plugin.") + presentations_qa.add_argument("--json", action="store_true", help="Emit JSON.") diff --git a/rfs/cli_commands/product.py b/rfs/cli_commands/product.py new file mode 100644 index 0000000..2b3554a --- /dev/null +++ b/rfs/cli_commands/product.py @@ -0,0 +1,139 @@ +"""Registration helpers for product workflow commands.""" + +from __future__ import annotations + + +def register_fast_framework_prompt(subparsers) -> None: + parser = subparsers.add_parser("fast-framework-prompt", help="Compile a paper-grounded framework prompt without generating images.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--out", required=True, help="Output directory for evidence, semantic contract, prompt, and overlay specification.") + parser.add_argument("--deadline", type=int, default=180, help="Soft end-to-end deadline in seconds. Default: 180.") + parser.add_argument("--preferences", help="Optional JSON style and output preferences.") + parser.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + parser.add_argument("--planner-model") + parser.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + parser.add_argument("--aspect-ratio", default="16:9") + parser.add_argument("--language", default="English") + parser.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "off"], default="auto") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + parser.add_argument("--editable-ppt", action="store_true", help="Also compile the semantic contract into native editable PowerPoint shapes, text, and connectors.") + parser.add_argument("--verify-ppt-roundtrip", action="store_true", help="Compile the editable PPTX, render it through PowerPoint, and write ppt_roundtrip_report.json.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_make_framework(subparsers) -> None: + parser = subparsers.add_parser("make-framework", help="Create a paper-grounded, reference-guided editable PPTX framework figure.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--reference", required=True, help="User-provided visual reference image path.") + parser.add_argument("--out", required=True, help="Output directory.") + contract_source = parser.add_mutually_exclusive_group() + contract_source.add_argument("--paper-contract-dir", help="Fast paper-to-image/fast-framework output directory containing figure_specification.json. Bypasses the legacy paper planner.") + contract_source.add_argument("--semantic-contract", help="Semantic contract JSON or figure_specification.json. Bypasses the legacy paper planner.") + parser.add_argument("--profile", default="ai-ml-paper", help="Figure profile. Default: ai-ml-paper.") + parser.add_argument("--slot-count", type=int, default=36, help="Target slot count, clamped to 25-50. Default: 36.") + parser.add_argument("--slot-source", choices=["paper", "reference-primary"], default="reference-primary", help="Slot content source. Default reference-primary makes the reference figure drive visual objects, layout, color, and flow logic.") + parser.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Slot asset generation mode. Default image2 uses Yunwu Images API.") + parser.add_argument("--candidates-per-slot", type=int, default=4, help="Candidate images per slot, clamped to 1-5. Default: 4.") + parser.add_argument("--asset-workers", type=int, default=1, help="Parallel asset generation workers, clamped to 1-12. Default: 1.") + parser.add_argument("--asset-retries", type=int, default=2, help="Retries per generated asset candidate, clamped to 0-5. Default: 2.") + parser.add_argument("--asset-review-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Selected asset review mode. Default: heuristic.") + parser.add_argument("--locator-mode", choices=["heuristic", "vlm"], default="heuristic", help="Layout coordinate source. Use vlm to locate slots from the reference image.") + parser.add_argument("--locator-model", help="Optional VLM model for --locator-mode vlm. Defaults to RFS_LOCATOR_MODEL/MODEL_VLM.") + parser.add_argument("--control-localizer-mode", choices=["off", "heuristic", "hybrid"], default="hybrid", help="Arrow/connector localization mode. Default hybrid uses CV candidates plus optional VLM binding; falls back to heuristic without API keys.") + parser.add_argument("--arrow-style-mode", choices=["off", "reference", "aesthetic"], default="reference", help="Arrow styling/routing mode. Default reference preserves reference-image routes and adds softer PPT styling/QA metadata.") + parser.add_argument("--prompt-plan-mode", choices=["heuristic", "vlm", "vlm-batch"], default="vlm", help="Reference-aware per-slot prompt planning mode. Default vlm uses one VLM call per slot; vlm-batch uses one batch VLM call; heuristic is offline only.") + parser.add_argument("--prompt-plan-model", help="Optional VLM model for --prompt-plan-mode vlm/vlm-batch. Defaults to RFS_PROMPT_PLANNER_MODEL/MODEL_VLM.") + parser.add_argument("--prompt-plan-workers", type=int, default=4, help="Parallel VLM workers for per-slot prompt planning, clamped to 1-12. Default: 4.") + parser.add_argument("--complexity-profile", choices=["reference-dense", "balanced", "legend-simple"], default="reference-dense", help="Visual complexity policy for slot_visual_spec.json. Default reference-dense.") + parser.add_argument("--critic-mode", choices=["off", "heuristic", "vlm"], default="heuristic", help="Final reference-vs-render critic mode. Default: heuristic.") + parser.add_argument("--critic-model", help="Optional VLM model for asset review and final critic. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--critic-iterations", type=int, default=0, help="VLM layout correction iterations, clamped to 0-3. Default: 0.") + parser.add_argument("--text-extractor-mode", choices=["heuristic", "ocr"], default="ocr", help="Editable text layer source. Default ocr uses local OCR when available and falls back to heuristic.") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="Local OCR engine for reference text extraction. Default: paddle; easyocr remains available.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--presentations-qa", action="store_true", help="Run optional Presentations plugin import/render/layout QA after export. This never mutates the PPTX.") + parser.add_argument("--presentations-workspace", help="Optional workspace for Presentations QA scratch artifacts.") + parser.add_argument("--presentations-scale", type=int, default=2, help="Preview render scale for Presentations QA. Default: 2.") + parser.add_argument("--no-export", action="store_true", help="Skip PDF/PNG export and only create PPTX/artifacts.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_paper_to_image(subparsers) -> None: + parser = subparsers.add_parser("paper-to-image", help="Summarize a paper, plan a scientific framework figure, and generate reviewed raster candidates with an optional editable PPT overlay.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--out", required=True, help="Output directory for review, templates, prompts, candidates, and production-only selected_image.png.") + parser.add_argument("--preferences", help="Optional JSON file containing style and output preferences.") + parser.add_argument("--positive-reference", action="append", default=[], help="Optional positive visual reference image. Repeat for multiple files.") + parser.add_argument("--negative-reference", action="append", default=[], help="Optional negative visual reference image. Repeat for multiple files.") + parser.add_argument("--reference-library", help="Optional reference library JSONL for exemplar selection, prompt guidance, and exemplar-aware candidate review.") + parser.add_argument("--reference-top-k", type=int, default=5, help="Positive exemplar count for --reference-library, clamped to 1-20. Default: 5.") + parser.add_argument("--user-style-brief", help="Optional user_style_brief.json from learn-reference-style. Adds style preferences only.") + parser.add_argument("--preference-history", help="Optional previous paper-to-image run directory or preference_pairs.jsonl for human visual preference-aware review.") + parser.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm", help="Paper summarization and figure planning mode. Default: vlm with heuristic fallback.") + parser.add_argument("--planner-model", help="Optional planning VLM. Defaults to RFS_PAPER_TO_IMAGE_MODEL/RFS_PAPER_PLANNER_MODEL/MODEL_VLM.") + parser.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto", help="Universal review profile plus optional domain extension. Default: auto.") + parser.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto", help="Reference architecture template. Default: auto selection.") + parser.add_argument("--asset-mode", choices=["image2", "gemini", "placeholder"], default="image2", help="Whole-image generation backend. Placeholder is for offline validation only.") + parser.add_argument("--candidates", type=int, default=3, help="Image candidate count, clamped to 1-4. Default: 3.") + parser.add_argument("--aspect-ratio", default="auto", help="Target image aspect ratio or auto to inherit the selected template. Default: auto.") + parser.add_argument("--language", default="English", help="Visible label language. Default: English.") + parser.add_argument("--image-model", help="Optional image model. Defaults to RFS_IMAGE_MODEL/IMAGE_MODEL.") + parser.add_argument("--image-retries", type=int, default=1, help="Retries per image candidate for retryable HTTP/provider errors, clamped to 0-5. Network timeouts are not blindly retried. Default: 1.") + parser.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Candidate review mode. Production Image2 requires VLM review. Default: vlm.") + parser.add_argument("--review-model", help="Optional VLM candidate-review model. Defaults to RFS_PAPER_TO_IMAGE_REVIEW_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--repair-rounds", type=int, default=3, help="Localized Image2 edit repair rounds, clamped to 0-4 with no-regression acceptance. Default: 3.") + parser.add_argument("--human-scientific-confirmed", action="store_true", help="Record human scientific review of the selected whole image during the Bootstrap Judge period.") + parser.add_argument("--repair-source", help="Reuse an existing failed Image2 candidate as the starting point and run review/repair without generating fresh initial candidates.") + parser.add_argument("--resume-candidates", action="store_true", help="Reuse existing candidate_XX images in --out, re-review them with current gates, and generate only missing candidates.") + parser.add_argument("--editable-overlay-ppt", action="store_true", help="Place the selected geometry-normalized Image2 visual substrate beneath native editable PPT labels and connectors. Requires deterministic overlay mode and a raw candidate that passes substrate geometry normalization.") + parser.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto", help="OCR source for exact-label validation. Auto tries local OCR and uses VLM review evidence. Default: auto.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_paper_to_editable(subparsers) -> None: + parser = subparsers.add_parser("paper-to-editable", help="Generate a paper-grounded visual reference and rebuild it as an editable PowerPoint figure.") + parser.add_argument("--paper", required=True, help="Paper PDF/LaTeX/Markdown/Word/text path.") + parser.add_argument("--out", required=True, help="Output directory for paper analysis, image candidates, semantic contracts, and editable PPTX.") + parser.add_argument("--preferences") + parser.add_argument("--positive-reference", action="append", default=[]) + parser.add_argument("--negative-reference", action="append", default=[]) + parser.add_argument("--planner-mode", choices=["vlm", "heuristic"], default="vlm") + parser.add_argument("--planner-model") + parser.add_argument("--image-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") + parser.add_argument("--image-candidates", type=int, default=3) + parser.add_argument("--aspect-ratio", default="auto") + parser.add_argument("--language", default="English") + parser.add_argument("--image-model") + parser.add_argument("--image-retries", type=int, default=2) + parser.add_argument("--review-mode", choices=["off", "heuristic", "vlm"], default="vlm") + parser.add_argument("--review-model") + parser.add_argument("--domain-profile", choices=["auto", "general", "ai-ml-method", "system-platform", "dataset-benchmark", "empirical-science", "survey-review"], default="auto") + parser.add_argument("--template", choices=["auto", "dense-multiframe", "multimodal", "branch", "feedback", "arbor", "linear", "tripanel", "dense-multimodal"], default="auto") + parser.add_argument("--repair-rounds", type=int, default=3) + parser.add_argument("--human-scientific-confirmed", action="store_true", help="Record that a human reviewed the selected whole-image scientific content during the Bootstrap Judge period.") + parser.add_argument("--editable-route", choices=["rebuild", "framework"], default="rebuild", help="Editable compiler route.") + parser.add_argument("--framework-slot-count", type=int, default=25) + parser.add_argument("--framework-asset-mode", choices=["image2", "gemini", "placeholder"], default="image2") + parser.add_argument("--framework-candidates-per-slot", type=int, default=1) + parser.add_argument("--framework-asset-workers", type=int, default=3) + parser.add_argument("--rebuild-asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api") + parser.add_argument("--rebuild-asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--vlm-timeout", type=int) + parser.add_argument("--vlm-retries", type=int) + parser.add_argument("--rebuild-critic-mode", choices=["off", "heuristic", "vlm"], default="off") + parser.add_argument("--rebuild-critic-iterations", type=int, default=0) + parser.add_argument("--rebuild-critic-model") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") + parser.add_argument("--allow-engineering-preview", action="store_true") + parser.add_argument("--no-export-preview", action="store_true") + parser.add_argument("--ocr-engine", choices=["auto", "rapidocr", "paddle", "easyocr", "vlm", "off"], default="auto") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + parser.add_argument("--json", action="store_true", help="Emit JSON.") diff --git a/rfs/cli_commands/rebuild.py b/rfs/cli_commands/rebuild.py new file mode 100644 index 0000000..496e122 --- /dev/null +++ b/rfs/cli_commands/rebuild.py @@ -0,0 +1,144 @@ +"""Adapters for editable-rebuild CLI arguments.""" + +from __future__ import annotations + +from collections.abc import Callable + + +def add_rebuild_args(parser) -> None: + parser.add_argument("--reference", required=True, help="Reference image path.") + parser.add_argument("--out", required=True, help="Output directory.") + parser.add_argument("--asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api", help="Slot asset source. Policy follows generation-plan per-slot decisions.") + parser.add_argument("--asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api", help="Slot asset decision policy. Default smart-api filters text slots, reuses duplicate assets, and disables final crop assets.") + parser.add_argument("--asset-workers", type=int, default=4, help="Parallel asset workers, clamped by the pipeline to 1-12. Default: 4.") + parser.add_argument("--asset-retries", type=int, default=1, help="Retries per slot in strict mode. Default: 1.") + parser.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True, help="Reuse accepted/passing assets and generate each failed slot once. Enabled by default.") + parser.add_argument("--no-economy-mode", dest="economy_mode", action="store_false", help="Disable economy reuse decisions.") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Editable text extraction mode. Default: ocr.") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm", help="Whole-reference design planning mode. Default: vlm with explicit fallback reporting.") + parser.add_argument("--design-plan-model", help="Optional VLM for whole-reference design planning.") + parser.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic", help="Group OCR lines into editable paragraphs. Default: heuristic.") + parser.add_argument("--text-grouping-model", help="Optional VLM for OCR text grouping.") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid", help="Panel/card/slot layout extraction mode. Default: hybrid.") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid", help="Arrow/control extraction mode. Default: hybrid.") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM layout disagreements.") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm", help="VLM adjudication for heuristic/VLM control disagreements.") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for visible controls missed after adjudication.") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm", help="VLM audit for missed layout objects and controls.") + parser.add_argument("--vlm-timeout", type=int, help="Timeout in seconds for rebuild VLM calls. Defaults to RFS_VLM_TIMEOUT or 240.") + parser.add_argument("--vlm-retries", type=int, help="Retry count for rebuild VLM calls. Defaults to RFS_VLM_RETRIES or 2.") + parser.add_argument("--rebuild-critic-mode", choices=["off", "heuristic", "vlm"], default="off", help="Compiled-preview rebuild critic mode. Default: off.") + parser.add_argument("--rebuild-critic-iterations", type=int, default=0, help="Compiled-preview critic correction iterations, clamped to 0-3. Default: 0.") + parser.add_argument("--rebuild-critic-model", help="Optional VLM model for --rebuild-critic-mode vlm. Defaults to RFS_REBUILD_CRITIC_MODEL/RFS_CRITIC_MODEL/MODEL_VLM.") + parser.add_argument("--export-preview", action="store_true", help="Export a PNG preview when PowerPoint is available.") + parser.add_argument("--regenerate-slots", help="Comma-separated slot ids to regenerate even when an existing asset is present.") + parser.add_argument("--strict-asset-regeneration", action="store_true", help="Use stricter asset thresholds and --asset-retries for high-cost regeneration.") + parser.add_argument("--skip-analysis", action="store_true", help="Reuse existing JSON contracts in --out instead of re-running layout/control/semantic analysis.") + parser.add_argument("--from-stage", help="Re-run from this stage onward (design_plan/build_program/localize_controls/text_layer/slot_semantics); stages before it reuse snapshots.") + parser.add_argument("--compile-only", action="store_true", help="Compile editable_composition.pptx from existing JSON contracts and assets without regenerating analysis or assets.") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle", help="OCR engine for --text-mode ocr. Default: paddle; easyocr remains available.") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch", help="OCR language hint. Default: en_ch.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + + +def build_rebuild_kwargs(args, adapter_factory: Callable) -> dict: + """Build ``rebuild_editable`` kwargs without changing CLI semantics.""" + + rebuild_adapters = adapter_factory(args.out, vlm_timeout=args.vlm_timeout, vlm_retries=args.vlm_retries) + use_layout_vlm = args.layout_mode in {"vlm", "hybrid"} + use_control_vlm = args.control_mode in {"vlm", "hybrid"} + use_any_rebuild_vlm = use_layout_vlm or use_control_vlm + return { + "asset_mode": args.asset_mode, + "asset_workers": args.asset_workers, + "asset_retries": args.asset_retries, + "economy_mode": args.economy_mode, + "text_mode": args.text_mode, + "control_mode": args.control_mode, + "layout_mode": args.layout_mode, + "export_preview": args.export_preview, + "regenerate_slots": args.regenerate_slots, + "strict_asset_regeneration": args.strict_asset_regeneration, + "skip_analysis": args.skip_analysis, + "from_stage": args.from_stage, + "compile_only": args.compile_only, + "ocr_engine": args.ocr_engine, + "ocr_lang": args.ocr_lang, + "vlm_layout_adapter": rebuild_adapters["layout"] if use_layout_vlm else None, + "panel_layout_adapter": rebuild_adapters.get("panel_layout") if use_layout_vlm else None, + "layout_adjudication_mode": args.layout_adjudication_mode if use_layout_vlm else "off", + "layout_adjudicator": rebuild_adapters["layout_adjudicator"] if use_layout_vlm and args.layout_adjudication_mode == "vlm" else None, + "control_adapter": rebuild_adapters["control"] if use_control_vlm else None, + "control_adjudication_mode": args.control_adjudication_mode if use_control_vlm else "off", + "control_adjudicator": rebuild_adapters["control_adjudicator"] if use_control_vlm and args.control_adjudication_mode == "vlm" else None, + "control_coverage_audit_mode": args.control_coverage_audit_mode if use_control_vlm else "off", + "coverage_audit_adapter": rebuild_adapters["coverage_audit"] if use_control_vlm and args.control_coverage_audit_mode == "vlm" else None, + "combined_coverage_audit_mode": args.combined_coverage_audit_mode if use_any_rebuild_vlm else "off", + "combined_coverage_audit_adapter": rebuild_adapters["combined_coverage_audit"] if use_any_rebuild_vlm and args.combined_coverage_audit_mode == "vlm" else None, + "vlm_timeout": args.vlm_timeout, + "vlm_retries": args.vlm_retries, + "rebuild_critic_mode": args.rebuild_critic_mode, + "rebuild_critic_iterations": args.rebuild_critic_iterations, + "rebuild_critic_model": args.rebuild_critic_model, + "semantic_adapter": rebuild_adapters["semantic"] if use_any_rebuild_vlm else None, + "asset_policy": args.asset_policy, + "design_plan_mode": args.design_plan_mode, + "design_plan_model": args.design_plan_model, + "design_adapter": rebuild_adapters["design"] if args.design_plan_mode == "vlm" else None, + "text_grouping_mode": args.text_grouping_mode, + "text_grouping_model": args.text_grouping_model, + } + + +def register_rebuild_core_commands(subparsers) -> None: + rebuild = subparsers.add_parser("rebuild-editable", help="Rebuild a reference image into a reusable editable PowerPoint composition.") + add_rebuild_args(rebuild) + + evolve = subparsers.add_parser("rebuild-evolve", help="Run rebuild-editable in a process-supervision evolution loop.") + add_rebuild_args(evolve) + evolve.add_argument("--max-rounds", type=int, default=3, help="Maximum rebuild+judge rounds. Default: 3.") + evolve.add_argument("--judge-model", help="Process Judge model. Defaults to RFS_CRITIC_MODEL/MODEL_VLM.") + evolve.add_argument("--score-threshold", type=float, help="Stop when the Judge overall score reaches this threshold.") + + evaluation = subparsers.add_parser("rebuild-editable-eval", help="Compare heuristic and hybrid VLM rebuild-editable outputs for one reference image.") + evaluation.add_argument("--reference", required=True, help="Reference image path.") + evaluation.add_argument("--out", required=True, help="Evaluation output directory.") + evaluation.add_argument("--asset-mode", choices=["api", "crop", "placeholder"], default="crop", help="Asset mode for both cases. Default: crop to avoid image generation cost.") + evaluation.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr", help="Text extraction mode for both cases. Default: ocr.") + evaluation.add_argument("--export-preview", action="store_true", help="Export PNG previews when PowerPoint is available.") + evaluation.add_argument("--json", action="store_true", help="Emit JSON.") + + +def register_professional_rebuild(subparsers) -> None: + parser = subparsers.add_parser("rebuild-editable-pro", help="Rebuild a reference image through a VLM-authored controlled professional DSL.") + parser.add_argument("--reference", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--asset-mode", choices=["api", "crop", "placeholder", "policy"], default="api") + parser.add_argument("--asset-policy", choices=["legacy", "smart-api", "generation-policy"], default="smart-api") + parser.add_argument("--asset-workers", type=int, default=4) + parser.add_argument("--asset-retries", type=int, default=1) + parser.add_argument("--economy-mode", dest="economy_mode", action="store_true", default=True) + parser.add_argument("--no-economy-mode", dest="economy_mode", action="store_false") + parser.add_argument("--text-mode", choices=["ocr", "manual", "off"], default="ocr") + parser.add_argument("--design-plan-mode", choices=["off", "heuristic", "vlm"], default="vlm") + parser.add_argument("--design-plan-model") + parser.add_argument("--text-grouping-mode", choices=["off", "heuristic", "vlm", "hybrid"], default="heuristic") + parser.add_argument("--text-grouping-model") + parser.add_argument("--layout-mode", choices=["heuristic", "vlm", "hybrid"], default="hybrid") + parser.add_argument("--control-mode", choices=["heuristic", "vlm", "hybrid", "manual"], default="hybrid") + parser.add_argument("--layout-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-adjudication-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--control-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--combined-coverage-audit-mode", choices=["off", "vlm"], default="vlm") + parser.add_argument("--vlm-timeout", type=int) + parser.add_argument("--vlm-retries", type=int) + parser.add_argument("--export-preview", action="store_true") + parser.add_argument("--regenerate-slots") + parser.add_argument("--strict-asset-regeneration", action="store_true") + parser.add_argument("--compile-only", action="store_true") + parser.add_argument("--repair-rounds", type=int, default=2) + parser.add_argument("--repair-mode", choices=["report", "vlm"], default="report") + parser.add_argument("--benchmark-out") + parser.add_argument("--ocr-engine", choices=["rapidocr", "paddle", "easyocr", "off"], default="paddle") + parser.add_argument("--ocr-lang", choices=["en", "ch", "en_ch"], default="en_ch") + parser.add_argument("--json", action="store_true") diff --git a/rfs/composition/__init__.py b/rfs/composition/__init__.py index 0d26f49..74d6b19 100644 --- a/rfs/composition/__init__.py +++ b/rfs/composition/__init__.py @@ -2,5 +2,6 @@ from .pptx import compile_ppt from .preview import render_rebuild_preview +from .rebuild import build_figure_program, build_text_layer, style_and_route_arrows -__all__ = ["compile_ppt", "render_rebuild_preview"] +__all__ = ["build_figure_program", "build_text_layer", "compile_ppt", "render_rebuild_preview", "style_and_route_arrows"] diff --git a/rfs/composition/pptx.py b/rfs/composition/pptx.py index f14c435..0d6262f 100644 --- a/rfs/composition/pptx.py +++ b/rfs/composition/pptx.py @@ -7,7 +7,7 @@ from pptx.dml.color import RGBColor from pptx.enum.dml import MSO_LINE_DASH_STYLE from pptx.enum.shapes import MSO_CONNECTOR, MSO_SHAPE -from pptx.enum.text import PP_ALIGN, MSO_ANCHOR +from pptx.enum.text import PP_ALIGN, MSO_ANCHOR, MSO_AUTO_SIZE from pptx.oxml import parse_xml from pptx.util import Inches, Pt @@ -38,10 +38,21 @@ def _connector_type_for_route(route_style: str): return MSO_CONNECTOR.STRAIGHT -def _set_text(shape, text: str, font_size: float = 10, bold: bool = False, color: str = "#163B4D", align=PP_ALIGN.CENTER, font_family: str | None = None) -> None: +def _set_text( + shape, + text: str, + font_size: float = 10, + bold: bool = False, + color: str = "#163B4D", + align=PP_ALIGN.CENTER, + font_family: str | None = None, + fit_to_shape: bool = False, +) -> None: tf = shape.text_frame tf.clear() tf.word_wrap = True + if fit_to_shape: + tf.auto_size = MSO_AUTO_SIZE.TEXT_TO_FIT_SHAPE tf.vertical_anchor = MSO_ANCHOR.MIDDLE p = tf.paragraphs[0] p.alignment = align @@ -84,9 +95,22 @@ def _add_card_frame(slide, card: dict, width_in: float, height_in: float): return shape -def _add_label(slide, text: str, x: float, y: float, w: float, h: float, font_size: float = 9, bold: bool = False, align=PP_ALIGN.CENTER, color: str = "#163B4D", font_family: str | None = None): +def _add_label( + slide, + text: str, + x: float, + y: float, + w: float, + h: float, + font_size: float = 9, + bold: bool = False, + align=PP_ALIGN.CENTER, + color: str = "#163B4D", + font_family: str | None = None, + fit_to_shape: bool = False, +): box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) - _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family) + _set_text(box, text, font_size=font_size, bold=bold, color=color, align=align, font_family=font_family, fit_to_shape=fit_to_shape) return box @@ -408,6 +432,12 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: for item in _text_program_items(program) if str(item.get("role")) == "panel_title" and "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() } + semantic_panel_title_targets = { + str(item.get("target_id")) + for item in _text_program_items(program) + if str(item.get("role")) == "panel_title" and item.get("semantic_panel_title") + } + ocr_panel_title_targets -= semantic_panel_title_targets for idx, panel in enumerate(program["panels"]): x, y, w, h = pct_to_inches(panel["bbox_percent"], width_in, height_in) local_style = panel_styles.get(panel["id"], {}) if isinstance(panel_styles.get(panel["id"], {}), dict) else {} @@ -421,11 +451,12 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: title_text_item = _text_item_for_target(program, panel["id"], "panel_title") _set_text( header, - "" if str(panel["id"]) in ocr_panel_title_targets else panel["title"], + "" if str(panel["id"]) in ocr_panel_title_targets else str(title_text_item.get("text") or panel["title"]) if title_text_item else panel["title"], font_size=float(title_text_item.get("font_size_pt")) if title_text_item else (9 if w < 2.0 else 10), bold=True, - color=str(title_text_item.get("color_hex")) if title_text_item else "#FFFFFF", + color=str(title_text_item.get("color_hex") or "#FFFFFF") if title_text_item else "#FFFFFF", font_family=str(title_text_item.get("font_family_guess") or "") if title_text_item else None, + fit_to_shape=True, ) rendered_cards = [] @@ -584,7 +615,8 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: rendered_text_items = [] for item in _text_program_items(program): is_ocr_text = "ocr" in str(item.get("reference_binding") or item.get("fit_strategy") or "").lower() - if str(item.get("role")) == "panel_title" and not is_ocr_text: + targets_actual_panel = str(item.get("target_id") or "") in panels_by_id + if str(item.get("role")) == "panel_title" and targets_actual_panel and not is_ocr_text: rendered_text_items.append({ "text_id": item.get("id"), "role": item.get("role"), @@ -609,6 +641,7 @@ def compile_ppt(program: dict, out_dir: str | Path) -> Path: align=_align_from_text_item(item), color=str(item.get("color_hex") or "#263747"), font_family=str(item.get("font_family_guess") or "") or None, + fit_to_shape=bool(item.get("semantic_entity_id") or item.get("semantic_panel_title")), ) rendered_text_items.append({ "text_id": item.get("id"), diff --git a/rfs/composition/preview.py b/rfs/composition/preview.py index c2ecef9..72e0547 100644 --- a/rfs/composition/preview.py +++ b/rfs/composition/preview.py @@ -56,12 +56,20 @@ def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: i return font_size_pt = float(item.get("font_size_pt") or 8) size_px = max(4, int(round(font_size_pt * canvas_h / 387.0))) - font = _font(size_px, bold=bool(item.get("bold"))) + bold = bool(item.get("bold")) + font = _font(size_px, bold=bold) color = _hex_color(item.get("color_hex"), "#263747") align = str(item.get("align") or "center").lower() - text_bbox = draw.textbbox((0, 0), text, font=font) + measure = draw.multiline_textbbox if "\n" in text else draw.textbbox + text_bbox = measure((0, 0), text, font=font, align="center") if "\n" in text else measure((0, 0), text, font=font) text_w = max(1, text_bbox[2] - text_bbox[0]) text_h = max(1, text_bbox[3] - text_bbox[1]) + while size_px > 4 and (text_w > w or text_h > h): + size_px -= 1 + font = _font(size_px, bold=bold) + text_bbox = measure((0, 0), text, font=font, align="center") if "\n" in text else measure((0, 0), text, font=font) + text_w = max(1, text_bbox[2] - text_bbox[0]) + text_h = max(1, text_bbox[3] - text_bbox[1]) if align == "left": tx = x elif align == "right": @@ -69,7 +77,10 @@ def _draw_text(draw: ImageDraw.ImageDraw, item: dict, canvas_w: int, canvas_h: i else: tx = x + (w - text_w) / 2 ty = y + (h - text_h) / 2 - draw.text((tx, ty), text, fill=color, font=font) + if "\n" in text: + draw.multiline_text((tx, ty), text, fill=color, font=font, align="center", spacing=0) + else: + draw.text((tx, ty), text, fill=color, font=font) def _draw_arrow(draw: ImageDraw.ImageDraw, arrow: dict, canvas_w: int, canvas_h: int) -> None: diff --git a/rfs/composition/rebuild/__init__.py b/rfs/composition/rebuild/__init__.py new file mode 100644 index 0000000..fbbc87f --- /dev/null +++ b/rfs/composition/rebuild/__init__.py @@ -0,0 +1,7 @@ +"""Canonical editable-rebuild composition APIs.""" + +from .arrows import style_and_route_arrows +from .program import build_figure_program +from .text import build_text_layer + +__all__ = ["build_figure_program", "build_text_layer", "style_and_route_arrows"] diff --git a/rfs/composition/rebuild/arrows.py b/rfs/composition/rebuild/arrows.py new file mode 100644 index 0000000..06153ad --- /dev/null +++ b/rfs/composition/rebuild/arrows.py @@ -0,0 +1,707 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ...utils import write_json + +ROUTER_VERSION = "reference-constrained-orthogonal-v1" +AESTHETIC_ROUTER_VERSION = "reference-tunnel-aesthetic-v2" +DEFAULT_REFERENCE_TUNNEL_PERCENT = 0.024 + + +def _bbox_center(bbox: dict[str, Any]) -> list[float]: + return [ + round(float(bbox["x"]) + float(bbox["w"]) / 2, 4), + round(float(bbox["y"]) + float(bbox["h"]) / 2, 4), + ] + + +def _anchor_point(obj: dict | None, anchor: str) -> list[float] | None: + if not obj or not isinstance(obj.get("bbox_percent"), dict): + return None + bbox = obj["bbox_percent"] + x = float(bbox["x"]) + y = float(bbox["y"]) + w = float(bbox["w"]) + h = float(bbox["h"]) + points = { + "left_mid": [x, y + h / 2], + "right_mid": [x + w, y + h / 2], + "top_mid": [x + w / 2, y], + "bottom_mid": [x + w / 2, y + h], + "center": [x + w / 2, y + h / 2], + } + point = points.get(anchor) or points["center"] + return [round(point[0], 4), round(point[1], 4)] + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _bbox_center(sbox) + tx, ty = _bbox_center(tbox) + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + s_anchor = "right_mid" if dx >= 0 else "left_mid" + t_anchor = "left_mid" if dx >= 0 else "right_mid" + else: + s_anchor = "bottom_mid" if dy >= 0 else "top_mid" + t_anchor = "top_mid" if dy >= 0 else "bottom_mid" + s = _anchor_point(source, s_anchor) + t = _anchor_point(target, t_anchor) + return [s, t] if s and t else [] + + +def _clean_path(points: list[list[float] | None]) -> list[list[float]]: + cleaned: list[list[float]] = [] + for point in points: + if not point or len(point) < 2: + continue + x = min(1.0, max(0.0, float(point[0]))) + y = min(1.0, max(0.0, float(point[1]))) + rounded = [round(x, 4), round(y, 4)] + if not cleaned or abs(cleaned[-1][0] - rounded[0]) > 0.001 or abs(cleaned[-1][1] - rounded[1]) > 0.001: + cleaned.append(rounded) + if len(cleaned) <= 2: + return cleaned + simplified = [cleaned[0]] + for index in range(1, len(cleaned) - 1): + prev = simplified[-1] + cur = cleaned[index] + nxt = cleaned[index + 1] + same_x = abs(prev[0] - cur[0]) < 0.001 and abs(cur[0] - nxt[0]) < 0.001 + same_y = abs(prev[1] - cur[1]) < 0.001 and abs(cur[1] - nxt[1]) < 0.001 + if not (same_x or same_y): + simplified.append(cur) + simplified.append(cleaned[-1]) + return simplified + + +def _segments(points: list[list[float]]) -> list[tuple[list[float], list[float]]]: + return [(points[i], points[i + 1]) for i in range(max(0, len(points) - 1))] + + +def _path_length(points: list[list[float]]) -> float: + total = 0.0 + for a, b in _segments(points): + total += ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 + return round(total, 4) + + +def _path_normal(points: list[list[float]]) -> list[float]: + if len(points) < 2: + return [0.0, 0.0] + dx = float(points[-1][0]) - float(points[0][0]) + dy = float(points[-1][1]) - float(points[0][1]) + length = (dx * dx + dy * dy) ** 0.5 + if length < 0.0001: + return [0.0, 0.0] + return [-dy / length, dx / length] + + +def _point_distance(a: list[float], b: list[float]) -> float: + return ((float(a[0]) - float(b[0])) ** 2 + (float(a[1]) - float(b[1])) ** 2) ** 0.5 + + +def _point_segment_distance(point: list[float], a: list[float], b: list[float]) -> float: + px, py = float(point[0]), float(point[1]) + ax, ay = float(a[0]), float(a[1]) + bx, by = float(b[0]), float(b[1]) + dx = bx - ax + dy = by - ay + denom = dx * dx + dy * dy + if denom < 0.0000001: + return _point_distance(point, a) + t = max(0.0, min(1.0, ((px - ax) * dx + (py - ay) * dy) / denom)) + proj = [ax + t * dx, ay + t * dy] + return _point_distance(point, proj) + + +def _max_distance_to_polyline(points: list[list[float]], reference: list[list[float]]) -> float: + segments = _segments(reference) + if not segments: + return 0.0 + distances = [] + for point in points: + distances.append(min(_point_segment_distance(point, a, b) for a, b in segments)) + return round(max(distances, default=0.0), 4) + + +def _lane_offset(lane_index: int, lane_count: int, tunnel_percent: float) -> float: + if lane_count <= 1: + return 0.0 + centered = float(lane_index) - (float(lane_count) - 1.0) / 2.0 + step = min(0.006, max(0.002, tunnel_percent / max(lane_count, 1))) + return max(-tunnel_percent * 0.82, min(tunnel_percent * 0.82, centered * step)) + + +def _shift_point(point: list[float], normal: list[float], offset: float) -> list[float]: + return [ + round(min(1.0, max(0.0, float(point[0]) + normal[0] * offset)), 4), + round(min(1.0, max(0.0, float(point[1]) + normal[1] * offset)), 4), + ] + + +def _aesthetic_tunnel_path( + points: list[list[float]], + lane_index: int, + lane_count: int, + role: str, + tunnel_percent: float = DEFAULT_REFERENCE_TUNNEL_PERCENT, +) -> tuple[list[list[float]], dict[str, Any]]: + original = _clean_path(points) + if len(original) < 2: + return original, { + "routing_algorithm": AESTHETIC_ROUTER_VERSION, + "route_generation_status": "aesthetic_no_path", + "reference_tunnel_percent": tunnel_percent, + "reference_path_delta_max": 0.0, + "reference_tunnel_preserved": True, + } + normal = _path_normal(original) + offset = _lane_offset(lane_index, lane_count, tunnel_percent) + adjusted = [list(point) for point in original] + if abs(offset) > 0.0001 and len(original) == 2: + mid = [ + (float(original[0][0]) + float(original[1][0])) / 2.0, + (float(original[0][1]) + float(original[1][1])) / 2.0, + ] + adjusted = [original[0], _shift_point(mid, normal, offset), original[-1]] + elif role in {"main_flow", "module_flow"}: + # Single-lane arrows keep the exact reference geometry; the PPT renderer + # still softens them through curve connectors, halo, and line caps. + adjusted = original + adjusted = _clean_path(adjusted) + delta = _max_distance_to_polyline(adjusted, original) + return adjusted, { + "routing_algorithm": AESTHETIC_ROUTER_VERSION, + "route_generation_status": "aesthetic_tunnel_adjusted" if adjusted != original else "aesthetic_style_only", + "reference_tunnel_percent": tunnel_percent, + "reference_path_delta_max": delta, + "reference_tunnel_preserved": delta <= tunnel_percent, + "lane_offset_percent": round(offset, 4), + } + + +def _orientation(a: list[float], b: list[float], c: list[float]) -> float: + return (float(b[1]) - float(a[1])) * (float(c[0]) - float(b[0])) - (float(b[0]) - float(a[0])) * (float(c[1]) - float(b[1])) + + +def _share_endpoint(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: + pairs = ((a1, b1), (a1, b2), (a2, b1), (a2, b2)) + return any(abs(p[0] - q[0]) < 0.002 and abs(p[1] - q[1]) < 0.002 for p, q in pairs) + + +def _segments_cross(a1: list[float], a2: list[float], b1: list[float], b2: list[float]) -> bool: + if _share_endpoint(a1, a2, b1, b2): + return False + # Fast reject by segment bounding boxes. + if max(min(a1[0], a2[0]), min(b1[0], b2[0])) > min(max(a1[0], a2[0]), max(b1[0], b2[0])): + return False + if max(min(a1[1], a2[1]), min(b1[1], b2[1])) > min(max(a1[1], a2[1]), max(b1[1], b2[1])): + return False + o1 = _orientation(a1, a2, b1) + o2 = _orientation(a1, a2, b2) + o3 = _orientation(b1, b2, a1) + o4 = _orientation(b1, b2, a2) + return o1 * o2 < 0 and o3 * o4 < 0 + + +def _point_inside_bbox(point: list[float], bbox: dict[str, Any], pad: float = 0.0) -> bool: + x = float(bbox["x"]) - pad + y = float(bbox["y"]) - pad + w = float(bbox["w"]) + pad * 2 + h = float(bbox["h"]) + pad * 2 + return x <= float(point[0]) <= x + w and y <= float(point[1]) <= y + h + + +def _segment_bbox_overlap(a: list[float], b: list[float], bbox: dict[str, Any], pad: float = 0.004) -> bool: + if _point_inside_bbox(a, bbox, pad=pad) or _point_inside_bbox(b, bbox, pad=pad): + return True + x0 = float(bbox["x"]) - pad + y0 = float(bbox["y"]) - pad + x1 = float(bbox["x"]) + float(bbox["w"]) + pad + y1 = float(bbox["y"]) + float(bbox["h"]) + pad + edges = [ + ([x0, y0], [x1, y0]), + ([x1, y0], [x1, y1]), + ([x1, y1], [x0, y1]), + ([x0, y1], [x0, y0]), + ] + return any(_segments_cross(a, b, e0, e1) for e0, e1 in edges) + + +def _candidate_lanes(source: dict, target: dict, obstacles: dict[str, dict], axis: str, limit: int = 18) -> list[float]: + sbox = source["bbox_percent"] + tbox = target["bbox_percent"] + if axis == "x": + base = [ + float(sbox["x"]), + float(sbox["x"]) + float(sbox["w"]), + float(sbox["x"]) + float(sbox["w"]) / 2, + float(tbox["x"]), + float(tbox["x"]) + float(tbox["w"]), + float(tbox["x"]) + float(tbox["w"]) / 2, + ] + for obj in obstacles.values(): + bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None + if not bbox: + continue + base.extend([float(bbox["x"]) - 0.014, float(bbox["x"]) + float(bbox["w"]) + 0.014]) + else: + base = [ + float(sbox["y"]), + float(sbox["y"]) + float(sbox["h"]), + float(sbox["y"]) + float(sbox["h"]) / 2, + float(tbox["y"]), + float(tbox["y"]) + float(tbox["h"]), + float(tbox["y"]) + float(tbox["h"]) / 2, + ] + for obj in obstacles.values(): + bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None + if not bbox: + continue + base.extend([float(bbox["y"]) - 0.014, float(bbox["y"]) + float(bbox["h"]) + 0.014]) + midpoint = (sum(base[:6]) / 6) if len(base) >= 6 else 0.5 + lanes = sorted({round(min(0.985, max(0.015, value)), 4) for value in base}, key=lambda value: abs(value - midpoint)) + return sorted(lanes[:limit]) + + +def _crossing_count(points: list[list[float]], existing_segments: list[tuple[list[float], list[float]]]) -> int: + count = 0 + for a, b in _segments(points): + for c, d in existing_segments: + if _segments_cross(a, b, c, d): + count += 1 + return count + + +def _obstacle_overlaps(points: list[list[float]], obstacles: dict[str, dict]) -> list[str]: + overlaps = [] + for obj_id, obj in obstacles.items(): + bbox = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else None + if not bbox: + continue + if any(_segment_bbox_overlap(a, b, bbox) for a, b in _segments(points)): + overlaps.append(obj_id) + return overlaps + + +def _score_candidate(points: list[list[float]], obstacles: dict[str, dict], existing_segments: list[tuple[list[float], list[float]]]) -> tuple[float, dict[str, Any]]: + overlaps = _obstacle_overlaps(points, obstacles) + crossings = _crossing_count(points, existing_segments) + bends = max(0, len(points) - 2) + length = _path_length(points) + near_edge_penalty = sum(1 for x, y in points if x < 0.012 or x > 0.988 or y < 0.012 or y > 0.988) + score = len(overlaps) * 1000 + crossings * 80 + bends * 8 + length * 16 + near_edge_penalty * 10 + return score, { + "candidate_score": round(score, 4), + "obstacle_overlap_count": len(overlaps), + "obstacle_overlap_ids": overlaps[:12], + "crossing_count": crossings, + "bend_count": bends, + "path_length": length, + } + + +def _best_fallback_path( + source: dict | None, + target: dict | None, + obstacles: dict[str, dict], + existing_segments: list[tuple[list[float], list[float]]], +) -> tuple[list[list[float]], dict[str, Any]]: + if not source or not target: + return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_source_or_target"} + direct = _edge_path(source, target) + if len(direct) < 2: + return [], {"routing_algorithm": ROUTER_VERSION, "route_generation_status": "missing_anchor_points"} + + sx, sy = direct[0] + tx, ty = direct[-1] + x_lanes = _candidate_lanes(source, target, obstacles, "x") + y_lanes = _candidate_lanes(source, target, obstacles, "y") + candidates = [_clean_path(direct)] + candidates.extend(_clean_path([direct[0], [x, sy], [x, ty], direct[-1]]) for x in x_lanes) + candidates.extend(_clean_path([direct[0], [sx, y], [tx, y], direct[-1]]) for y in y_lanes) + # Two-lane orthogonal paths can move around large central obstacles without + # changing the source-target semantics from the reference-derived program. + for x in x_lanes[:12]: + for y in y_lanes[:12]: + candidates.append(_clean_path([direct[0], [x, sy], [x, y], [tx, y], direct[-1]])) + candidates.append(_clean_path([direct[0], [sx, y], [x, y], [x, ty], direct[-1]])) + + best_path: list[list[float]] = direct + best_meta: dict[str, Any] = {} + best_score = float("inf") + seen: set[tuple[tuple[float, float], ...]] = set() + for candidate in candidates: + if len(candidate) < 2: + continue + key = tuple((point[0], point[1]) for point in candidate) + if key in seen: + continue + seen.add(key) + score, meta = _score_candidate(candidate, obstacles, existing_segments) + if score < best_score: + best_score = score + best_path = candidate + best_meta = meta + best_meta.update({ + "routing_algorithm": ROUTER_VERSION, + "route_generation_status": "fallback_route_selected", + "candidate_count": len(seen), + }) + return best_path, best_meta + + +def _infer_role(arrow: dict, out_counts: dict[str, int], in_counts: dict[str, int], panel_ids: set[str]) -> str: + kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() + source = str(arrow.get("source_id") or arrow.get("source") or "") + target = str(arrow.get("target_id") or arrow.get("target") or "") + if "loop" in kind or "dashed" in kind: + return "feedback_loop" + # Branch/convergence semantics should win over panel membership. Some RFS + # programs use panel-like IDs for central modules, and treating those first + # collapses every connector into the same main-flow style. + if out_counts.get(source, 0) > 1: + return "branch" + if in_counts.get(target, 0) > 1: + return "convergence" + if source in panel_ids or target in panel_ids: + return "main_flow" + return "module_flow" + + +def _style_for_role(role: str, arrow: dict, mode: str = "reference") -> dict[str, Any]: + kind = str(arrow.get("control_kind") or arrow.get("type") or "").lower() + if str(mode).lower() == "aesthetic": + if role == "main_flow": + return {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "line_cap": "round", "halo_width_pt": 4.4, "halo_color": "#FFFFFF"} + if role == "branch": + return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} + if role == "convergence": + return {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.2, "halo_color": "#FFFFFF"} + if role == "feedback_loop": + return {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} + if "elbow" in kind: + return {"route_style": "rounded_elbow", "stroke_width_pt": 1.6, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} + return {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round", "halo_width_pt": 3.0, "halo_color": "#FFFFFF"} + if role == "main_flow": + return {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med", "line_cap": "round"} + if role == "branch": + return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} + if role == "convergence": + return {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "line_cap": "round"} + if role == "feedback_loop": + return {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm", "line_cap": "round", "line_pattern": "dash"} + if "elbow" in kind: + return {"route_style": "rounded_elbow", "stroke_width_pt": 1.65, "arrowhead_size": "sm", "line_cap": "round"} + return {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "line_cap": "round"} + + +def _reference_locked(arrow: dict) -> bool: + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if len(path) < 2: + return False + source = str(arrow.get("binding_source") or arrow.get("detected_by") or "").lower() + if any(term in source for term in ("reference", "opencv", "vlm", "explicit", "candidate", "layout")): + return True + # In reference-primary mode, existing normalized paths are treated as a + # reference contract unless explicitly marked as fallback. + return str(arrow.get("route_policy", "")).lower() != "fallback_reroute_allowed" + + +def _route_metrics(arrows: list[dict], objects: dict[str, dict]) -> tuple[dict[str, dict], int]: + metrics: dict[str, dict] = {} + all_segments: list[tuple[str, list[float], list[float]]] = [] + for arrow in arrows: + points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + sid = str(arrow.get("source_id") or arrow.get("source") or "") + tid = str(arrow.get("target_id") or arrow.get("target") or "") + obstacles = [] + for obj_id, obj in objects.items(): + if obj_id in {sid, tid} or not isinstance(obj.get("bbox_percent"), dict): + continue + if any(_segment_bbox_overlap(a, b, obj["bbox_percent"]) for a, b in _segments(points)): + obstacles.append(obj_id) + metrics[str(arrow.get("id"))] = { + "path_length": _path_length(points), + "bend_count": max(0, len(points) - 2), + "obstacle_overlap_count": len(obstacles), + "obstacle_overlap_ids": obstacles[:12], + "point_count": len(points), + } + for a, b in _segments(points): + all_segments.append((str(arrow.get("id")), a, b)) + + crossing_count = 0 + crossings_by_arrow = {str(arrow.get("id")): 0 for arrow in arrows} + for idx, (aid, a1, a2) in enumerate(all_segments): + for bid, b1, b2 in all_segments[idx + 1:]: + if aid == bid: + continue + if _segments_cross(a1, a2, b1, b2): + crossing_count += 1 + crossings_by_arrow[aid] = crossings_by_arrow.get(aid, 0) + 1 + crossings_by_arrow[bid] = crossings_by_arrow.get(bid, 0) + 1 + for aid, count in crossings_by_arrow.items(): + if aid in metrics: + metrics[aid]["crossing_count"] = count + return metrics, crossing_count + + +def style_and_route_arrows( + program: dict, + out_dir: str | Path, + mode: str = "reference", +) -> dict: + """Add reference-preserving arrow aesthetics and QA artifacts. + + The reference image remains the hard constraint. Existing reference-derived + routes are not freely rerouted; this stage mostly adds style, bundling + metadata, and diagnostics. Only missing/fallback paths are synthesized. + """ + out = Path(out_dir) + if str(mode).lower() == "off": + write_json(out / "arrow_style_profile.json", {"summary": "Arrow styling skipped.", "mode": "off"}) + write_json(out / "selected_arrow_routes.json", {"summary": "Arrow routing skipped.", "mode": "off", "routes": []}) + write_json(out / "arrow_quality_report.json", {"summary": "Arrow quality skipped.", "mode": "off", "status": "skipped"}) + return program + + panels = program.get("panels", []) if isinstance(program.get("panels"), list) else [] + slots = program.get("slots", []) if isinstance(program.get("slots"), list) else [] + program_style = program.get("style") if isinstance(program.get("style"), dict) else {} + color_tokens = program_style.get("color_tokens", []) if isinstance(program_style.get("color_tokens"), list) else [] + default_style_token_id = next( + ( + str(item.get("token_id") or "") + for item in color_tokens + if isinstance(item, dict) and any(term in str(item.get("usage") or "").casefold() for term in ("arrow", "connector", "line")) + ), + "", + ) + if not default_style_token_id: + default_style_token_id = next((str(item.get("token_id") or "") for item in color_tokens if isinstance(item, dict) and item.get("token_id")), "") + objects = {str(item.get("id")): item for item in panels + slots if isinstance(item, dict) and item.get("id")} + slot_objects = {str(item.get("id")): item for item in slots if isinstance(item, dict) and item.get("id")} + panel_ids = {str(panel.get("id")) for panel in panels if isinstance(panel, dict)} + arrows = [dict(item) for item in program.get("arrows", []) if isinstance(item, dict)] + out_counts: dict[str, int] = {} + in_counts: dict[str, int] = {} + for arrow in arrows: + source = str(arrow.get("source_id") or arrow.get("source") or "") + target = str(arrow.get("target_id") or arrow.get("target") or "") + out_counts[source] = out_counts.get(source, 0) + 1 + in_counts[target] = in_counts.get(target, 0) + 1 + + reference_segments: list[tuple[list[float], list[float]]] = [] + for original in arrows: + original_points = original.get("path_percent") if isinstance(original.get("path_percent"), list) else [] + if _reference_locked(original) and len(original_points) >= 2: + reference_segments.extend(_segments(_clean_path(original_points))) + + grouped: dict[str, list[dict]] = {} + enriched = [] + committed_fallback_segments: list[tuple[list[float], list[float]]] = [] + for index, arrow in enumerate(arrows, start=1): + source = str(arrow.get("source_id") or arrow.get("source") or "") + target = str(arrow.get("target_id") or arrow.get("target") or "") + arrow["source_id"] = source + arrow["target_id"] = target + arrow["source"] = source + arrow["target"] = target + if not str(arrow.get("style_token_id") or "").strip(): + arrow["style_token_id"] = default_style_token_id + points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + locked = _reference_locked(arrow) + route_meta: dict[str, Any] = {"routing_algorithm": "preserve_reference_path", "route_generation_status": "reference_locked"} + fallback_allowed = str(arrow.get("route_policy", "")).lower() == "fallback_reroute_allowed" + if (len(points) < 2 or (fallback_allowed and not locked)) and mode in {"reference", "aesthetic"}: + obstacles = {obj_id: obj for obj_id, obj in slot_objects.items() if obj_id not in {source, target}} + points, route_meta = _best_fallback_path( + objects.get(source), + objects.get(target), + obstacles, + reference_segments + committed_fallback_segments, + ) + locked = False + else: + points = _clean_path(points) + arrow["path_percent"] = [[round(float(p[0]), 4), round(float(p[1]), 4)] for p in points if isinstance(p, list) and len(p) >= 2] + role = _infer_role(arrow, out_counts, in_counts, panel_ids) + style = _style_for_role(role, arrow, mode=mode) + recompute_generated_style = str(arrow.get("aesthetic_policy", "")).startswith("reference_first") + for key, value in style.items(): + if recompute_generated_style or key not in arrow or not str(arrow.get(key, "")).strip(): + arrow[key] = value + if recompute_generated_style or not str(arrow.get("semantic_role", "")).strip(): + arrow["semantic_role"] = role + arrow.setdefault("aesthetic_policy", "reference_first_soft_editable_ppt_connector") + arrow.setdefault("reference_locked", locked) + arrow.setdefault("reference_path_preserved", locked) + arrow.setdefault("route_policy", "preserve_reference_path" if locked else "synthesize_missing_path_only") + arrow.setdefault("routing_algorithm", route_meta.get("routing_algorithm", ROUTER_VERSION)) + arrow.setdefault("route_generation_status", route_meta.get("route_generation_status", "fallback_route_selected" if not locked else "reference_locked")) + if "candidate_count" in route_meta: + arrow.setdefault("candidate_count", route_meta["candidate_count"]) + if "candidate_score" in route_meta: + arrow.setdefault("candidate_score", route_meta["candidate_score"]) + arrow.setdefault("corner_radius_percent", 0.018 if arrow.get("route_style") in {"rounded_elbow", "bundled_elbow"} else 0.0) + if role == "branch": + bundle_id = f"from_{source}" + elif role == "convergence": + bundle_id = f"to_{target}" + elif role == "feedback_loop": + bundle_id = f"loop_{source}_{target}" + else: + bundle_id = f"flow_{index:02d}" + arrow.setdefault("bundle_id", bundle_id) + grouped.setdefault(bundle_id, []).append(arrow) + enriched.append(arrow) + if not locked and len(arrow["path_percent"]) >= 2: + committed_fallback_segments.extend(_segments(arrow["path_percent"])) + + for bundle_id, items in grouped.items(): + items.sort(key=lambda item: (item.get("target_id", ""), item.get("id", ""))) + for lane_index, arrow in enumerate(items): + arrow["lane_index"] = lane_index + arrow["lane_count"] = len(items) + + if str(mode).lower() == "aesthetic": + tunnel_percent = DEFAULT_REFERENCE_TUNNEL_PERCENT + for arrow in enriched: + original_path = _clean_path(arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else []) + arrow["reference_original_path_percent"] = original_path + if arrow.get("reference_locked") and len(original_path) >= 2: + offset_allowed = bool(arrow.get("aesthetic_offset_allowed")) or str(arrow.get("route_policy", "")).lower() == "aesthetic_tunnel_allowed" + if offset_allowed: + adjusted_path, aesthetic_meta = _aesthetic_tunnel_path( + original_path, + int(arrow.get("lane_index") or 0), + int(arrow.get("lane_count") or 1), + str(arrow.get("semantic_role") or ""), + tunnel_percent=tunnel_percent, + ) + arrow["path_percent"] = adjusted_path + arrow["routing_algorithm"] = aesthetic_meta["routing_algorithm"] + arrow["route_generation_status"] = aesthetic_meta["route_generation_status"] + arrow["reference_tunnel_percent"] = aesthetic_meta["reference_tunnel_percent"] + arrow["reference_path_delta_max"] = aesthetic_meta["reference_path_delta_max"] + arrow["reference_tunnel_preserved"] = aesthetic_meta["reference_tunnel_preserved"] + arrow["lane_offset_percent"] = aesthetic_meta["lane_offset_percent"] + arrow["reference_path_preserved"] = bool(aesthetic_meta["reference_tunnel_preserved"]) + else: + arrow["routing_algorithm"] = AESTHETIC_ROUTER_VERSION + arrow["route_generation_status"] = "aesthetic_style_only" + arrow["reference_tunnel_percent"] = tunnel_percent + arrow["reference_path_delta_max"] = 0.0 + arrow["reference_tunnel_preserved"] = True + arrow["lane_offset_percent"] = 0.0 + arrow["reference_path_preserved"] = True + + metrics, total_crossings = _route_metrics(enriched, slot_objects) + routes = [] + for arrow in enriched: + aid = str(arrow.get("id")) + m = metrics.get(aid, {}) + clutter_score = ( + int(m.get("crossing_count", 0)) * 12 + + int(m.get("bend_count", 0)) * 3 + + int(m.get("obstacle_overlap_count", 0)) * 18 + ) + arrow["aesthetic_score"] = max(0, 100 - clutter_score) + routes.append({ + "id": aid, + "source_id": arrow.get("source_id"), + "target_id": arrow.get("target_id"), + "semantic_role": arrow.get("semantic_role"), + "route_style": arrow.get("route_style"), + "bundle_id": arrow.get("bundle_id"), + "lane_index": arrow.get("lane_index"), + "lane_count": arrow.get("lane_count"), + "reference_locked": arrow.get("reference_locked"), + "reference_path_preserved": arrow.get("reference_path_preserved"), + "path_percent": arrow.get("path_percent"), + "style_token_id": arrow.get("style_token_id"), + "stroke_width_pt": arrow.get("stroke_width_pt"), + "arrowhead_size": arrow.get("arrowhead_size"), + "line_cap": arrow.get("line_cap"), + "line_pattern": arrow.get("line_pattern", "solid"), + "routing_algorithm": arrow.get("routing_algorithm"), + "route_generation_status": arrow.get("route_generation_status"), + "reference_original_path_percent": arrow.get("reference_original_path_percent"), + "reference_tunnel_percent": arrow.get("reference_tunnel_percent"), + "reference_path_delta_max": arrow.get("reference_path_delta_max"), + "reference_tunnel_preserved": arrow.get("reference_tunnel_preserved"), + "lane_offset_percent": arrow.get("lane_offset_percent"), + "halo_width_pt": arrow.get("halo_width_pt"), + "halo_color": arrow.get("halo_color"), + "candidate_count": arrow.get("candidate_count"), + "candidate_score": arrow.get("candidate_score"), + "metrics": m, + "aesthetic_score": arrow["aesthetic_score"], + }) + + arrow_style_profile = { + "summary": "Reference-first arrow styling profile for editable PPT connectors.", + "mode": mode, + "reference_priority": "reference_image_hard_constraint", + "routing_principle": "preserve reference-derived source-target logic and path geometry; only synthesize missing fallback paths", + "routing_algorithm": ROUTER_VERSION, + "aesthetic_routing_algorithm": AESTHETIC_ROUTER_VERSION, + "reference_tunnel_percent": DEFAULT_REFERENCE_TUNNEL_PERCENT, + "fallback_routing_policy": "only arrows with missing paths or route_policy=fallback_reroute_allowed may use obstacle-aware orthogonal routing", + "aesthetic_mode_policy": "mode=aesthetic may adjust reference-locked paths only inside the reference tunnel and must keep reference_tunnel_preserved=true", + "style_rules": { + "main_flow": {"route_style": "soft_straight", "stroke_width_pt": 2.2, "arrowhead_size": "med"}, + "branch": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, + "convergence": {"route_style": "bundled_elbow", "stroke_width_pt": 1.55, "arrowhead_size": "sm"}, + "feedback_loop": {"route_style": "dashed_spline_like", "stroke_width_pt": 1.8, "arrowhead_size": "sm"}, + "module_flow": {"route_style": "soft_straight", "stroke_width_pt": 1.45, "arrowhead_size": "sm"}, + }, + "aesthetic_style_rules": { + "main_flow": {"route_style": "soft_curve", "stroke_width_pt": 2.35, "arrowhead_size": "med", "halo_width_pt": 4.4}, + "branch": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, + "convergence": {"route_style": "metro_bundle", "stroke_width_pt": 1.45, "arrowhead_size": "sm", "halo_width_pt": 3.2}, + "feedback_loop": {"route_style": "dashed_loop", "stroke_width_pt": 1.75, "arrowhead_size": "sm", "halo_width_pt": 3.0}, + "module_flow": {"route_style": "soft_curve", "stroke_width_pt": 1.55, "arrowhead_size": "sm", "halo_width_pt": 3.0}, + }, + "ppt_editability": "all arrows render as PPT connector shapes, not raster assets", + "rounded_line_policy": "use round line caps and editable connector segments; reference-locked paths are not geometrically rewritten", + } + selected_routes = { + "summary": "Selected reference-preserving arrow routes and style assignments.", + "mode": mode, + "route_count": len(routes), + "routes": routes, + } + tunnel_violations = [route["id"] for route in routes if route.get("reference_tunnel_preserved") is False] + quality_report = { + "summary": "Arrow routing and styling quality report.", + "mode": mode, + "status": "pass" if all(route["aesthetic_score"] >= 35 for route in routes) and not tunnel_violations else "needs_review", + "arrow_count": len(routes), + "total_crossing_count": total_crossings, + "total_obstacle_overlap_count": sum(int(route["metrics"].get("obstacle_overlap_count", 0)) for route in routes), + "fallback_route_count": sum(1 for route in routes if route.get("route_generation_status") == "fallback_route_selected"), + "aesthetic_tunnel_adjusted_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_tunnel_adjusted"), + "aesthetic_style_only_count": sum(1 for route in routes if route.get("route_generation_status") == "aesthetic_style_only"), + "reference_tunnel_violations": tunnel_violations, + "average_aesthetic_score": round(sum(float(route["aesthetic_score"]) for route in routes) / max(len(routes), 1), 2), + "reference_path_overrides": [route["id"] for route in routes if route.get("reference_locked") and not route.get("reference_path_preserved")], + "routes": routes, + } + program["arrows"] = enriched + program["control_shapes"] = [dict(item) for item in enriched] + program.setdefault("style", {})["arrow_style_profile_path"] = "arrow_style_profile.json" + write_json(out / "arrow_style_profile.json", arrow_style_profile) + write_json(out / "selected_arrow_routes.json", selected_routes) + write_json(out / "arrow_quality_report.json", quality_report) + write_json(out / "figure_program.json", program) + return program diff --git a/rfs/composition/rebuild/export.py b/rfs/composition/rebuild/export.py new file mode 100644 index 0000000..f4b8a7a --- /dev/null +++ b/rfs/composition/rebuild/export.py @@ -0,0 +1,112 @@ +"""Canonical rebuild compile-only and preview export operations.""" + +from __future__ import annotations + +import json +import platform +from pathlib import Path + +from ..pptx import compile_ppt +from ..preview import render_rebuild_preview +from ...utils import read_json, write_json, write_text + + +def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: + x = max(0.0, min(0.995, x)) + y = max(0.0, min(0.995, y)) + w = max(0.001, min(w, 1.0 - x)) + h = max(0.001, min(h, 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: + ratio = width_px / max(height_px, 1) + width = max(10.0, min(15.6, 7.5 * ratio)) + return round(width, 3), round(width / ratio, 3) + + +def _style_from_layout(layout: dict) -> dict: + palette = layout.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52", "#6B57C8", "#D44E5D"] + background = str(layout.get("canvas", {}).get("background") or "#FFFFFF") + style = { + "palette": palette, + "reference_palette": palette, + "color_tokens": [{"token_id": f"palette_{i:02d}", "hex": color} for i, color in enumerate(palette)], + "panel_styles": {"reference_canvas": {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}}, + "arrow_weight_pt": 1.7, + } + for panel in layout.get("panels", []): + if panel.get("id") and panel["id"] != "reference_canvas": + style["panel_styles"].setdefault(str(panel["id"]), {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}) + return style + + +def _export_preview(pptx_path: Path, out: Path) -> Path | None: + preview = out / "rebuild_preview.png" + if platform.system() != "Windows": + write_text(out / "preview_export_error.txt", "PowerPoint preview export is only available on Windows with Microsoft PowerPoint installed.") + program_path = out / "figure_program.json" + return render_rebuild_preview(read_json(program_path), out, preview) if program_path.exists() else None + try: + import win32com.client + app = win32com.client.Dispatch("PowerPoint.Application") + app.Visible = True + deck = app.Presentations.Open(str(pptx_path), WithWindow=False) + deck.Slides(1).Export(str(preview), "PNG") + deck.Close() + app.Quit() + return preview + except Exception as exc: + write_text(out / "preview_export_error.txt", str(exc)) + program_path = out / "figure_program.json" + return render_rebuild_preview(read_json(program_path), out, preview) if program_path.exists() else None + + +def _ppt_package_counts(pptx_path: Path, reference_path: Path) -> dict: + from pptx import Presentation + prs = Presentation(str(pptx_path)) + shapes = [shape for slide in prs.slides for shape in slide.shapes] + return {"ppt_page_count": len(prs.slides), "shape_count": len(shapes), "picture_count": sum(getattr(shape, "shape_type", None) == 13 for shape in shapes), "text_shape_count": sum(bool(getattr(shape, "has_text_frame", False) and shape.text_frame.text.strip()) for shape in shapes), "connector_count": sum("CONNECTOR" in str(getattr(shape, "shape_type", "")) or "LINE" in str(getattr(shape, "shape_type", "")) for shape in shapes), "contains_full_reference_image": False, "reference_path": str(reference_path)} + + +def _load_json_or_empty(path: Path) -> dict: + if not path.exists(): return {} + try: data = json.loads(path.read_text(encoding="utf-8")) + except Exception: return {} + return data if isinstance(data, dict) else {} + + +def _program_from_contracts(out: Path) -> dict: + geometry = _load_json_or_empty(out / "reference_geometry.json") + controls = _load_json_or_empty(out / "reference_controls.json") + text_program = _load_json_or_empty(out / "text_program.json") + canvas = geometry.get("canvas") if isinstance(geometry.get("canvas"), dict) else {"width_in": 10.0, "height_in": 5.625, "background": "#FFFFFF"} + if "width_in" not in canvas or "height_in" not in canvas: + width_px = int(canvas.get("width_px") or 1600) + height_px = int(canvas.get("height_px") or 900) + width_in, height_in = _canvas_inches(width_px, height_px) + canvas = {**canvas, "width_in": width_in, "height_in": height_in, "background": canvas.get("background") or "#FFFFFF"} + layout = {"canvas": canvas, "palette": geometry.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52"], "panels": geometry.get("panels") or [{"id": "reference_canvas", "title": "Editable Rebuild", "bbox_percent": _bbox(0.025, 0.065, 0.95, 0.84), "editable_in": "pptx"}]} + slots = geometry.get("slots") or _load_json_or_empty(out / "slot_inventory.json").get("slots") or [] + program = {"canvas": canvas, "style": _style_from_layout(layout), "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, "panels": layout["panels"], "cards": geometry.get("cards") or [], "slots": slots, "assets": [{"id": slot.get("asset_id") or slot.get("id"), "path": f"assets/{slot.get('asset_id') or slot.get('id')}.png", "source": "slot_asset"} for slot in slots], "arrows": controls.get("arrows") or [], "labels": [], "groups": [], "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}]} + if isinstance(text_program.get("items"), list): + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + return program + + +def _compile_existing_contracts(reference_path: Path, out: Path, export_preview: bool = False) -> dict: + program = _program_from_contracts(out) + write_json(out / "figure_program.json", program) + pptx_path = compile_ppt(program, out) + preview = _export_preview(pptx_path, out) if export_preview else None + counts = _ppt_package_counts(pptx_path, reference_path) + report_path = out / "composition_quality_report.json" + report = _load_json_or_empty(report_path) + report["rebuild_editable_summary"] = counts + report["compile_only"] = True + write_json(report_path, report) + return {"summary": "Editable rebuild recompiled from existing JSON contracts.", "ok": True, "out_dir": str(out), "reference": str(reference_path), "pptx": str(pptx_path), "preview": str(preview) if preview else None, "asset_count": len(program.get("assets", [])), "slot_count": len(program.get("slots", [])), "connector_count": len(program.get("arrows", [])), "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, "compile_only": True} + + +__all__ = ["_compile_existing_contracts", "_export_preview", "_load_json_or_empty", "_ppt_package_counts", "_program_from_contracts"] diff --git a/rfs/composition/rebuild/program.py b/rfs/composition/rebuild/program.py new file mode 100644 index 0000000..a6b9879 --- /dev/null +++ b/rfs/composition/rebuild/program.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from ...analysis.rebuild.layout import plan_reference_layout +from ...utils import ratio_string, write_json + + +def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: + x = max(0.0, min(0.995, x)) + y = max(0.0, min(0.995, y)) + w = max(0.001, min(w, 1.0 - x)) + h = max(0.001, min(h, 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _style_from_layout(layout: dict) -> dict: + palette = layout.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52", "#6B57C8", "#D44E5D"] + background = str(layout.get("canvas", {}).get("background") or "#FFFFFF") + style = { + "palette": palette, + "reference_palette": palette, + "color_tokens": [{"token_id": f"palette_{i:02d}", "hex": color} for i, color in enumerate(palette)], + "panel_styles": {"reference_canvas": {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}}, + "arrow_weight_pt": 1.7, + } + for panel in layout.get("panels", []): + if panel.get("id") and panel["id"] != "reference_canvas": + style["panel_styles"].setdefault(str(panel["id"]), {"fill_color": background, "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", "header_color": palette[1] if len(palette) > 1 else "#6C7A89"}) + return style + + +def _build_program(reference_path: Path, out: Path, layout_mode: str = "hybrid", vlm_layout_adapter: Callable | None = None, panel_layout_adapter: Callable | None = None, design_plan: dict | None = None, layout_adjudication_mode: str = "off", layout_adjudicator: Callable | None = None) -> tuple[dict, dict]: + """Create the editable-rebuild figure program from the layout contract.""" + layout = plan_reference_layout(reference_path, out, mode=layout_mode, vlm_adapter=vlm_layout_adapter, panel_layout_adapter=panel_layout_adapter, design_plan=design_plan, layout_adjudication_mode=layout_adjudication_mode, layout_adjudicator=layout_adjudicator) + return { + "canvas": layout["canvas"], "style": _style_from_layout(layout), + "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, + "panels": layout.get("panels", []), "cards": layout.get("cards", []), "slots": layout.get("slots", []), + "assets": [], "arrows": [], "labels": [], "groups": [], + "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}], + }, layout + + +def _center(bbox: dict) -> tuple[float, float]: + return float(bbox["x"]) + float(bbox["w"]) / 2, float(bbox["y"]) + float(bbox["h"]) / 2 + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _center(sbox) + tx, ty = _center(tbox) + dx = tx - sx + dy = ty - sy + if abs(dx) >= abs(dy): + start = [float(sbox["x"]) + (float(sbox["w"]) if dx >= 0 else 0.0), sy] + end = [float(tbox["x"]) if dx >= 0 else float(tbox["x"]) + float(tbox["w"]), ty] + else: + start = [sx, float(sbox["y"]) + (float(sbox["h"]) if dy >= 0 else 0.0)] + end = [tx, float(tbox["y"]) if dy >= 0 else float(tbox["y"]) + float(tbox["h"])] + return [[round(start[0], 4), round(start[1], 4)], [round(end[0], 4), round(end[1], 4)]] + + +def _control_record_from_arrow(arrow: dict) -> dict: + points = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + xs = [float(point[0]) for point in points if isinstance(point, list) and len(point) >= 2] + ys = [float(point[1]) for point in points if isinstance(point, list) and len(point) >= 2] + if not xs or not ys: + xs, ys = [0.0, 0.001], [0.0, 0.001] + x0, x1 = min(xs), max(xs) + y0, y1 = min(ys), max(ys) + w = max(0.001, x1 - x0) + h = max(0.001, y1 - y0) + return { + "id": arrow.get("id"), + "type": "ppt_control", + "control_kind": arrow.get("control_kind") or arrow.get("type") or "straight_arrow", + "bbox_percent": {"x": round(x0, 4), "y": round(y0, 4), "w": round(w, 4), "h": round(h, 4)}, + "center_percent": {"x": round(x0 + w / 2, 4), "y": round(y0 + h / 2, 4)}, + "width_percent": round(w, 4), + "height_percent": round(h, 4), + "aspect_ratio_decimal": round(w / max(h, 0.001), 3), + "aspect_ratio_w_h": ratio_string(w, h), + "source_id": arrow.get("source_id") or arrow.get("source"), + "target_id": arrow.get("target_id") or arrow.get("target"), + "source_anchor": arrow.get("source_anchor") or "auto", + "target_anchor": arrow.get("target_anchor") or "auto", + "path_percent": points, + "style_token_id": arrow.get("style_token_id"), + "candidate_label": arrow.get("candidate_label"), + "detected_by": arrow.get("detected_by"), + "binding_source": arrow.get("binding_source"), + "confidence": arrow.get("confidence"), + "editable_in": "pptx", + "render_policy": "ppt_shape_not_image_asset", + } + + +def build_figure_program(paper_brief: dict, inventory: dict, style: dict, out_dir: str | Path, layout_plan: dict) -> dict: + panels = layout_plan["panels"] + slots = layout_plan["slots"] + objects_by_id = {item.get("id"): item for item in panels + slots if isinstance(item, dict)} + color_tokens = style.get("color_tokens", []) if isinstance(style.get("color_tokens"), list) else [] + default_arrow_token = next((str(item.get("token_id")) for item in color_tokens if "arrow" in str(item.get("usage", "")).lower() or "connector" in str(item.get("usage", "")).lower()), "") + if not default_arrow_token and color_tokens: + default_arrow_token = str(color_tokens[0].get("token_id") or "") + arrows = [] + for item in layout_plan.get("arrows", []): + if not isinstance(item, dict): + continue + arrow = dict(item) + arrow["source"] = arrow.get("source") or arrow.get("source_id") or "" + arrow["target"] = arrow.get("target") or arrow.get("target_id") or "" + arrow["source_id"] = arrow.get("source_id") or arrow.get("source") + arrow["target_id"] = arrow.get("target_id") or arrow.get("target") + arrow["source_anchor"] = arrow.get("source_anchor") or "auto" + arrow["target_anchor"] = arrow.get("target_anchor") or "auto" + arrow["editable_in"] = "pptx" + arrow["render_policy"] = arrow.get("render_policy") or "ppt_shape_not_image_asset" + arrow["style_token_id"] = arrow.get("style_token_id") or default_arrow_token + if not arrow.get("path_percent"): + arrow["path_percent"] = _edge_path(objects_by_id.get(arrow["source"]), objects_by_id.get(arrow["target"])) + arrows.append(arrow) + + assets = [ + { + "id": slot["asset_id"], + "slot_id": slot["id"], + "path": f"assets/{slot['asset_id']}.png", + "source": "slot_image_generation", + "fit_policy": "contain_no_crop", + "z_index": slot.get("z_index", 20), + "reference_crop_path": slot.get("reference_crop_path"), + "local_color_token_ids": slot.get("local_color_token_ids", []), + "style_profile_path": style.get("reference_style_profile_path", "reference_style_profile.json"), + } + for slot in slots + ] + + labels = [] + for panel in panels: + labels.append({ + "id": f"label_{panel['id']}", + "text": panel["title"], + "target_id": panel["id"], + "role": "panel_title", + "editable_in": "pptx", + }) + for slot in slots: + labels.append({ + "id": f"label_{slot['id']}", + "text": slot["paper_concept"], + "target_id": slot["id"], + "role": "slot_caption", + "editable_in": "pptx", + }) + + canvas_ratio = float(inventory.get("canvas_aspect_ratio") or 16 / 9) + height_in = 7.5 + width_in = height_in * canvas_ratio + if width_in > 16.0: + width_in = 16.0 + height_in = width_in / canvas_ratio + program = { + "summary": "Structured figure program used as the only layout source for PPT composition.", + "canvas": {"width_in": round(width_in, 3), "height_in": height_in, "ratio": round(canvas_ratio, 4), "background": "#FFFFFF"}, + "paper_brief": { + "title_guess": paper_brief.get("title_guess"), + "figure_goal": paper_brief.get("figure_goal"), + "variables": paper_brief.get("variables", []), + }, + "style": { + "palette": style.get("palette", []), + "reference_palette": style.get("reference_palette", []), + "color_tokens": style.get("color_tokens", []), + "reference_style_profile_path": style.get("reference_style_profile_path", "reference_style_profile.json"), + "panel_styles": style.get("panel_styles", {}), + "slot_frame_policy": style.get("slot_frame_policy", "frameless_slot"), + "picture_fill_policy": style.get("picture_fill_policy", "direct_full_slot_contain_no_tile"), + }, + "locator": { + "mode": layout_plan.get("locator_mode"), + "reference_path": layout_plan.get("reference_path") or inventory.get("reference_path"), + }, + "panels": panels, + "slots": slots, + "assets": assets, + "labels": labels, + "arrows": arrows, + "control_shapes": layout_plan.get("control_shapes", []) or [_control_record_from_arrow(arrow) for arrow in arrows], + "ppt_shapes": layout_plan.get("ppt_shapes", []), + "text_regions": layout_plan.get("text_regions", []), + "groups": [ + { + "id": "top_pipeline", + "members": [p["id"] for p in panels if p["id"] != "shared_resource_library"], + "editable_in": "pptx", + }, + { + "id": "shared_resources", + "members": [p["id"] for p in panels if p["id"] == "shared_resource_library"], + "editable_in": "pptx", + }, + ], + "export_targets": [ + {"type": "pptx", "path": "editable_composition.pptx", "role": "main_editable_source"}, + {"type": "pdf", "path": "review.pdf", "role": "review_export"}, + {"type": "png", "path": "final_600dpi.png", "role": "publication_preview"}, + ], + } + controls_doc = { + "summary": "Reference arrows, connectors, loops, and other non-image controls measured for editable PPT rendering.", + "reference_path": layout_plan.get("reference_path") or inventory.get("reference_path"), + "candidate_path": inventory.get("reference_control_candidates_path") or "reference_control_candidates.json", + "slot_overlay_path": inventory.get("slot_overlay_path"), + "control_overlay_path": inventory.get("reference_control_overlay_path"), + "requested_mode": (inventory.get("control_localizer") or {}).get("requested_mode"), + "effective_mode": (inventory.get("control_localizer") or {}).get("effective_mode"), + "controls": [_control_record_from_arrow(arrow) for arrow in arrows], + "ppt_arrows": [_control_record_from_arrow(arrow) for arrow in arrows], + "render_policy": "ppt_shape_not_image_asset", + } + write_json(Path(out_dir) / "reference_controls.json", controls_doc) + write_json(Path(out_dir) / "figure_program.json", program) + return program diff --git a/rfs/composition/rebuild/text.py b/rfs/composition/rebuild/text.py new file mode 100644 index 0000000..dc0cc07 --- /dev/null +++ b/rfs/composition/rebuild/text.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import math +from pathlib import Path +from typing import Callable + +from PIL import Image + +from ...analysis.rebuild.reference_text import extract_reference_text +from ...text_grouping import group_text_regions, write_text_grouping_artifacts +from ...text_layer_ownership import apply_text_layer_ownership, write_text_layer_ownership_artifacts +from ...utils import write_json + + +def _bbox_center(bbox: dict) -> dict: + x = float(bbox["x"]) + y = float(bbox["y"]) + w = float(bbox["w"]) + h = float(bbox["h"]) + return {"x": round(x + w / 2, 4), "y": round(y + h / 2, 4)} + + +def _clamp_bbox(bbox: dict) -> dict: + x = max(0.0, min(0.995, float(bbox["x"]))) + y = max(0.0, min(0.995, float(bbox["y"]))) + w = max(0.001, min(float(bbox["w"]), 1.0 - x)) + h = max(0.001, min(float(bbox["h"]), 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _region_record(region_id: str, text: str, role: str, bbox: dict, color_hex: str, source: str, target_id: str) -> dict: + bbox = _clamp_bbox(bbox) + return { + "id": region_id, + "text": text, + "role": role, + "target_id": target_id, + "bbox_percent": bbox, + "center_percent": _bbox_center(bbox), + "width_percent": round(float(bbox["w"]), 4), + "height_percent": round(float(bbox["h"]), 4), + "estimated_font_ratio": round(float(bbox["h"]) * 0.62, 5), + "color_hex": color_hex, + "source": source, + "editable_in": "pptx", + } + + +def _rgb_tuple(hex_color: str) -> tuple[int, int, int]: + text = str(hex_color or "#000000").strip().lstrip("#") + if len(text) < 6: + return 0, 0, 0 + return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) + + +def _hex(rgb: tuple[int, int, int]) -> str: + return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" + + +def _luminance(rgb: tuple[int, int, int]) -> float: + return 0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2] + + +def _sample_text_color(reference_path: str | Path, bbox: dict, fallback: str = "#263747") -> str: + try: + with Image.open(reference_path) as image: + image = image.convert("RGB") + iw, ih = image.size + x0 = max(0, min(iw - 1, int(float(bbox["x"]) * iw))) + y0 = max(0, min(ih - 1, int(float(bbox["y"]) * ih))) + x1 = max(x0 + 1, min(iw, int((float(bbox["x"]) + float(bbox["w"])) * iw))) + y1 = max(y0 + 1, min(ih, int((float(bbox["y"]) + float(bbox["h"])) * ih))) + crop = image.crop((x0, y0, x1, y1)) + pixels = list(crop.getdata()) + except Exception: + return fallback + if not pixels: + return fallback + dark = [px for px in pixels if _luminance(px) < 215] + candidates = dark or pixels + candidates.sort(key=_luminance) + return _hex(candidates[len(candidates) // 3]) + + +def _nearest_token_id(style: dict, color_hex: str) -> str: + tokens = style.get("color_tokens", []) if isinstance(style.get("color_tokens"), list) else [] + if not tokens: + return "" + target = _rgb_tuple(color_hex) + best_id = "" + best_dist = math.inf + for token in tokens: + if not isinstance(token, dict): + continue + token_hex = str(token.get("hex") or "") + if not token_hex: + continue + rgb = _rgb_tuple(token_hex) + dist = sum((target[i] - rgb[i]) ** 2 for i in range(3)) + if dist < best_dist: + best_dist = dist + best_id = str(token.get("token_id") or "") + return best_id + + +def _short_label(text: str) -> str: + replacements = { + "ffmpeg video-to-audio extraction": "FFmpeg", + "funasr speech recognition microphone": "FunASR", + "funasr speech recognition": "FunASR", + "mtcnn face detection": "MTCNN", + "alphapose full-body skeleton extraction": "AlphaPose", + "frame sampling stack": "Frame sampling", + "frame sampling from video": "Frame sampling", + "audio waveform output": "audio waveform", + "spoken text with timestamps": "transcript + time", + "raw interview video recording": "Raw interview video", + "participant full-body video thumbnail grid": "Sampled full-body clips", + "face modality": "Face", + "frame modality": "Frame", + "pose modality": "Pose", + "audio modality": "Audio", + "text modality": "Text", + "neo-ffi-3 questionnaire checklist": "NEO-FFI-3 questionnaire", + "multimodal personality dataset database": "Multimodal signals + Big Five labels", + } + normalized = " ".join(str(text or "").strip().split()).lower() + if normalized in replacements: + return replacements[normalized] + raw = str(text or "").strip() + if len(raw) <= 24: + return raw + words = raw.replace("/", " ").split() + return " ".join(words[:3]) + + +def _slot_role(slot: dict) -> str: + text = f"{slot.get('id', '')} {slot.get('paper_concept', '')} {slot.get('display_label', '')}".lower() + if "legend" in text: + return "legend_label" + if "modality" in text: + return "modality_label" + if any(term in text for term in ("ffmpeg", "funasr", "mtcnn", "alphapose", "frame sampling", "waveform", "transcript")): + return "method_label" + if any(term in text for term in ("openness", "conscientious", "extraversion", "agreeableness", "neuroticism", "ocean")): + return "trait_label" + if bool(slot.get("show_slot_caption")) or str(slot.get("display_label", "")).strip(): + return "slot_caption" + return "hidden_slot_label" + + +def _slot_text_visible(slot: dict, role: str) -> bool: + if role == "hidden_slot_label": + return False + if bool(slot.get("show_slot_caption")) or str(slot.get("display_label", "")).strip(): + return True + return role in {"legend_label", "modality_label", "method_label", "trait_label"} + + +def _parent_panel(slot: dict, panels: dict[str, dict]) -> dict | None: + for key in ("parent_panel", "panel_id", "macro_panel_id"): + value = str(slot.get(key) or "") + if value in panels: + return panels[value] + return None + + +def _caption_bbox_for_slot(slot: dict, panels: dict[str, dict]) -> dict: + box = slot["bbox_percent"] + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + panel = _parent_panel(slot, panels) + role = _slot_role(slot) + if role in {"modality_label", "trait_label"} and panel: + pbox = panel["bbox_percent"] + label_x = x + w + 0.004 + panel_right = float(pbox["x"]) + float(pbox["w"]) + return _clamp_bbox({ + "x": label_x, + "y": y + h * 0.18, + "w": max(0.014, panel_right - label_x - 0.004), + "h": h * 0.58, + }) + label_h = max(0.012, min(0.026, h * 0.20)) + label_y = y + h + 0.006 + if panel: + pbox = panel["bbox_percent"] + panel_bottom = float(pbox["y"]) + float(pbox["h"]) + if label_y + label_h > panel_bottom - 0.004: + label_y = y - label_h - 0.004 + return _clamp_bbox({ + "x": x + w * 0.03, + "y": label_y, + "w": w * 0.94, + "h": label_h, + }) + + +def _panel_title_bbox(panel: dict, canvas_height_in: float) -> dict: + bbox = panel["bbox_percent"] + header_h = min(float(bbox["h"]) * 0.17, 0.34 / max(canvas_height_in, 0.001)) + return _clamp_bbox({"x": bbox["x"], "y": bbox["y"], "w": bbox["w"], "h": header_h}) + + +def _font_pt_from_reference_region(region: dict, canvas_height_in: float) -> float: + role = str(region.get("role") or "") + multiplier = 0.44 if role == "panel_title" else 0.62 + return round(max(1.0, float(region["height_percent"]) * canvas_height_in * 72 * multiplier), 2) + + +def _alignment_item(item: dict, region: dict) -> dict: + ib = item["bbox_percent"] + rb = region["bbox_percent"] + ic = item["center_percent"] + rc = region["center_percent"] + center_delta = math.sqrt((float(ic["x"]) - float(rc["x"])) ** 2 + (float(ic["y"]) - float(rc["y"])) ** 2) + width_delta = abs(float(ib["w"]) - float(rb["w"])) + height_delta = abs(float(ib["h"]) - float(rb["h"])) + font_delta = abs(float(item["estimated_font_ratio"]) - float(region["estimated_font_ratio"])) + status = "pass" + if center_delta > 0.05 or width_delta > 0.08 or height_delta > 0.06 or item.get("editable_in") != "pptx": + status = "fail" + return { + "label_id": item["id"], + "source_reference_text_id": region["id"], + "role": item["role"], + "center_delta_percent": round(center_delta, 5), + "width_delta_percent": round(width_delta, 5), + "height_delta_percent": round(height_delta, 5), + "font_ratio_delta": round(font_delta, 5), + "color_status": "reference_token" if item.get("color_token_id") else "sampled_reference_color", + "editable_in": item.get("editable_in"), + "status": status, + } + + +def _heuristic_regions(reference_path: str | Path, program: dict, canvas_height_in: float) -> list[dict]: + panels = {panel["id"]: panel for panel in program.get("panels", []) if isinstance(panel, dict) and panel.get("id")} + regions: list[dict] = [] + for panel in program.get("panels", []): + if not isinstance(panel, dict) or not panel.get("bbox_percent"): + continue + bbox = _panel_title_bbox(panel, canvas_height_in) + regions.append(_region_record( + f"ref_text_panel_{panel['id']}", + str(panel.get("title") or panel.get("paper_concept") or panel["id"]), + "panel_title", + bbox, + "#FFFFFF", + "reference_panel_header_geometry", + panel["id"], + )) + + for slot in program.get("slots", []): + if not isinstance(slot, dict) or not slot.get("bbox_percent"): + continue + role = _slot_role(slot) + if not _slot_text_visible(slot, role): + continue + label = str(slot.get("display_label") or "").strip() or _short_label(str(slot.get("paper_concept") or slot.get("id") or "")) + bbox = _caption_bbox_for_slot(slot, panels) + color = _sample_text_color(reference_path, bbox, fallback="#263747") + regions.append(_region_record( + f"ref_text_slot_{slot['id']}", + label, + role, + bbox, + color, + "reference_slot_text_region_inferred", + slot["id"], + )) + + for arrow in program.get("arrows", []): + label = str(arrow.get("label") or "").strip() + path = arrow.get("path_percent") if isinstance(arrow.get("path_percent"), list) else [] + if not label or len(path) < 2: + continue + xs = [float(point[0]) for point in path if isinstance(point, list) and len(point) >= 2] + ys = [float(point[1]) for point in path if isinstance(point, list) and len(point) >= 2] + if not xs or not ys: + continue + mid_x = (min(xs) + max(xs)) / 2 + mid_y = (min(ys) + max(ys)) / 2 + bbox = {"x": mid_x - 0.025, "y": mid_y - 0.01, "w": 0.05, "h": 0.02} + color = _sample_text_color(reference_path, bbox, fallback="#263747") + regions.append(_region_record( + f"ref_text_arrow_{arrow.get('id')}", + label, + "arrow_label", + bbox, + color, + "reference_arrow_label_region_inferred", + str(arrow.get("id") or ""), + )) + return regions + + +def _overlap(a: dict, b: dict) -> float: + ax0, ay0 = float(a["x"]), float(a["y"]) + ax1, ay1 = ax0 + float(a["w"]), ay0 + float(a["h"]) + bx0, by0 = float(b["x"]), float(b["y"]) + bx1, by1 = bx0 + float(b["w"]), by0 + float(b["h"]) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + return (ix * iy) / max(float(b["w"]) * float(b["h"]), 0.000001) + + +def _merge_ocr_with_fallback(ocr_regions: list[dict], fallback_regions: list[dict]) -> list[dict]: + if not ocr_regions: + return fallback_regions + merged = list(ocr_regions) + for fallback in fallback_regions: + target_id = str(fallback.get("target_id") or "") + role = str(fallback.get("role") or "") + duplicate = False + for region in ocr_regions: + if str(region.get("target_id") or "") == target_id and str(region.get("role") or "") == role: + duplicate = True + break + if _overlap(region["bbox_percent"], fallback["bbox_percent"]) > 0.45: + duplicate = True + break + if not duplicate and role == "panel_title": + fallback = dict(fallback) + fallback["source"] = "reference_panel_header_geometry_fallback_after_ocr" + merged.append(fallback) + return merged + + +def build_text_layer( + reference_path: str | Path, + program: dict, + style: dict, + out_dir: str | Path, + text_extractor_mode: str = "ocr", + ocr_engine: str = "easyocr", + ocr_lang: str = "en_ch", + ocr_adapter: Callable[[str | Path, str], list[dict]] | None = None, + text_grouping_mode: str = "heuristic", + text_grouping_model: str | None = None, + text_grouping_adapter: Callable | None = None, +) -> dict: + """Build reference-first editable text artifacts and attach them to the program.""" + + out = Path(out_dir) + canvas = program.get("canvas", {}) if isinstance(program.get("canvas"), dict) else {} + canvas_height_in = float(canvas.get("height_in") or 7.5) + + fallback_regions = _heuristic_regions(reference_path, program, canvas_height_in) + ocr_regions, ocr_report = extract_reference_text( + reference_path, + program, + mode=text_extractor_mode, + engine=ocr_engine, + lang=ocr_lang, + ocr_adapter=ocr_adapter, + ) + raw_geometry = { + "summary": "Raw line-level OCR geometry before paragraph grouping.", + "reference_path": str(reference_path), + "detection_mode": "ocr" if ocr_regions else "ocr_empty_or_unavailable", + "ocr_engine": ocr_engine, + "ocr_lang": ocr_lang, + "text_region_count": len(ocr_regions), + "text_regions": ocr_regions, + } + grouped_ocr_regions, grouping_plan, grouping_report = group_text_regions( + ocr_regions, + mode=text_grouping_mode if ocr_regions else "off", + adapter=text_grouping_adapter, + reference_path=reference_path, + program=program, + model=text_grouping_model, + ) + write_text_grouping_artifacts(out, raw_geometry, grouping_plan, grouping_report) + regions = _merge_ocr_with_fallback(grouped_ocr_regions, fallback_regions) + regions, ownership_plan, ownership_report = apply_text_layer_ownership(regions, program, mode="heuristic") + write_text_layer_ownership_artifacts(out, ownership_plan, ownership_report) + if not ocr_regions: + ocr_report["text_region_count"] = len(regions) + ocr_report.setdefault("warnings", []) + ocr_report["warnings"].append("using_heuristic_text_layer_fallback") + + items: list[dict] = [] + for region in regions: + if region.get("layer_ownership") != "editable_text_layer": + continue + font_size = float(region.get("font_size_pt") or _font_pt_from_reference_region(region, canvas_height_in)) + token_id = _nearest_token_id(style, region["color_hex"]) + item = { + "id": f"text_{region['id']}", + "text": region["text"], + "role": region["role"], + "target_id": region["target_id"], + "source_reference_text_id": region["id"], + "reference_binding": region["source"], + "bbox_percent": region["bbox_percent"], + "center_percent": region["center_percent"], + "width_percent": region["width_percent"], + "height_percent": region["height_percent"], + "estimated_font_ratio": region["estimated_font_ratio"], + "font_size_pt": font_size, + "color_hex": region["color_hex"], + "color_token_id": token_id, + "bold": str(region.get("font_weight_guess") or "").lower() == "bold" or region["role"] in {"panel_title", "method_label", "modality_label"}, + "align": "left" if region["role"] in {"modality_label", "trait_label", "arrow_label"} else "center", + "font_family_guess": region.get("font_family_guess") or ("Microsoft YaHei" if any(ord(char) > 127 for char in str(region.get("text", ""))) else "Arial"), + "font_weight_guess": region.get("font_weight_guess") or ("bold" if region["role"] in {"panel_title", "method_label", "modality_label"} else "regular"), + "fit_strategy": "ocr_bbox_exact" if str(region.get("source", "")).startswith("reference_ocr") else "reference_geometry_bbox", + "ocr_confidence": region.get("confidence"), + "editable_in": "pptx", + "visible": True, + "layer_ownership": region.get("layer_ownership"), + } + items.append(item) + + geometry = { + "summary": "Reference-first text geometry extracted or inferred from the reference layout; no publication font threshold is applied.", + "reference_path": str(reference_path), + "policy": "reference_image_is_highest_authority_for_text_size_position_color_and_hierarchy", + "detection_mode": "ocr" if ocr_regions else "reference_geometry_and_local_color_sampling", + "ocr_engine": ocr_engine, + "ocr_lang": ocr_lang, + "reference_text_geometry_raw_path": "reference_text_geometry_raw.json", + "text_grouping_plan_path": "text_grouping_plan.json", + "text_grouping_report_path": "text_grouping_report.json", + "text_grouping_mode": text_grouping_mode, + "text_grouping_status": grouping_report.get("status"), + "text_layer_ownership_plan_path": "text_layer_ownership_plan.json", + "text_layer_ownership_report_path": "text_layer_ownership_report.json", + "text_regions": regions, + } + text_program = { + "summary": "Reference-first editable PPT text program derived from reference_text_geometry.json.", + "policy": "match_reference_text_geometry; reference_layout_over_typography_defaults", + "reference_text_geometry_path": "reference_text_geometry.json", + "items": items, + } + region_by_id = {region["id"]: region for region in regions} + report_items = [_alignment_item(item, region_by_id[item["source_reference_text_id"]]) for item in items] + alignment_report = { + "summary": "Text alignment report comparing PPT text program geometry against reference-derived text geometry.", + "policy": "reference_alignment_only; no fixed typography failure condition", + "status": "pass" if all(item["status"] == "pass" for item in report_items) else "fail", + "max_center_delta_percent": max((item["center_delta_percent"] for item in report_items), default=0.0), + "max_width_delta_percent": max((item["width_delta_percent"] for item in report_items), default=0.0), + "max_height_delta_percent": max((item["height_delta_percent"] for item in report_items), default=0.0), + "items": report_items, + } + readability_note = { + "summary": "Optional readability note only; it does not override reference-first text sizing.", + "status": "informational_only", + "policy": "do_not_modify_text_size_unless_user_requests_readability_over_reference_matching", + "note": "Some labels may appear small after manuscript scaling if the reference figure uses small labels.", + } + + write_json(out / "reference_text_geometry.json", geometry) + write_json(out / "text_program.json", text_program) + write_json(out / "text_alignment_report.json", alignment_report) + write_json(out / "ocr_text_quality_report.json", ocr_report) + write_json(out / "publication_readability_note.json", readability_note) + program["text_program"] = text_program + program["text_program_path"] = "text_program.json" + program["reference_text_geometry_path"] = "reference_text_geometry.json" + program["text_alignment_report_path"] = "text_alignment_report.json" + write_json(out / "figure_program.json", program) + return program diff --git a/rfs/contracts/semantic.py b/rfs/contracts/semantic.py index 9cea2ca..9a9b6e7 100644 --- a/rfs/contracts/semantic.py +++ b/rfs/contracts/semantic.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +from itertools import combinations from pathlib import Path from typing import Any @@ -46,6 +47,10 @@ def _text(item: dict, fallback: str = "") -> str: return fallback +def _normalized_text(value: Any) -> str: + return re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "", str(value or "").casefold()) + + def _specification(contract: dict) -> dict: return contract.get("figure_specification") if isinstance(contract.get("figure_specification"), dict) else contract @@ -189,7 +194,7 @@ def compile_paper_brief_from_semantic_contract( for entity in entities: panel = entity.get("panel_label") or modules[0] neighbor_cues = [*incoming.get(entity["id"], []), *outgoing.get(entity["id"], [])] - must_show = [f"paper-grounded visual object for {entity['label']}", "local reference shape and visual density"] + must_show = [f"paper-grounded visual object for {entity['label']}", "visible mechanism supported by cited paper evidence"] if neighbor_cues: must_show.append("declared local context: " + ", ".join(list(dict.fromkeys(neighbor_cues))[:3])) suggestions.append({ @@ -197,7 +202,7 @@ def compile_paper_brief_from_semantic_contract( "macro_panel": panel, "paper_concept": entity["label"], "composition_type": _composition_type(entity["role"], entity["label"]), - "visual_metaphor": f"reference-grounded scientific visual for the paper term {entity['label']}", + "visual_metaphor": f"paper-grounded scientific visual for the paper term {entity['label']}", "must_show": must_show, "avoid_showing": ["unmentioned architecture modules", "fake formulas", "fake numeric chart text", "critical text baked into the image"], "semantic_instance_id": entity["id"], @@ -270,7 +275,10 @@ def _tokens(value: object) -> set[str]: def _object_text(obj: dict) -> str: - return " ".join(str(obj.get(key) or "") for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt")) + return " ".join( + str(obj.get(key) or "") + for key in ("id", "title", "label", "name", "semantic_type", "description", "prompt", "prompt_subject", "display_label", "paper_concept") + ) def _center(obj: dict) -> tuple[float, float]: @@ -278,21 +286,137 @@ def _center(obj: dict) -> tuple[float, float]: return float(box.get("x") or 0) + float(box.get("w") or 0) / 2, float(box.get("y") or 0) + float(box.get("h") or 0) / 2 -def _assign_entities(entities: list[dict], program: dict) -> tuple[dict[str, dict], list[dict]]: +def _semantic_panels(contract: dict) -> list[dict]: + spec = _specification(contract) + panels = [] + for index, raw in enumerate(spec.get("panel_graphs", []) if isinstance(spec.get("panel_graphs"), list) else []): + if not isinstance(raw, dict): + continue + panel_id = str(raw.get("id") or f"panel_{index + 1}").strip() + label = str(raw.get("label") or raw.get("description") or panel_id).strip() + if panel_id: + panels.append({"id": panel_id, "label": label, "order": index}) + return panels + + +def _visual_panel_id(obj: dict, panels: list[dict]) -> str | None: + direct = str(obj.get("panel_id") or obj.get("parent_panel") or "").strip() + if direct: + return direct + object_id = str(obj.get("id") or "").strip() + if any(str(panel.get("id") or "") == object_id for panel in panels): + return object_id + box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + containing = [ + panel for panel in panels + if isinstance(panel.get("bbox_percent"), dict) and _inside(box, panel["bbox_percent"]) + ] + if not containing: + return None + containing.sort(key=lambda panel: float(panel["bbox_percent"].get("w") or 0) * float(panel["bbox_percent"].get("h") or 0)) + return str(containing[0].get("id") or "").strip() or None + + +def _resolve_panel_mapping(semantic_panels: list[dict], program_panels: list[dict]) -> tuple[dict[str, str], list[dict]]: + visual_panels = sorted(program_panels, key=lambda item: (_center(item)[0], _center(item)[1], str(item.get("id") or ""))) + available = list(visual_panels) + mapping: dict[str, str] = {} + records: list[dict] = [] + for semantic_index, semantic in enumerate(semantic_panels): + semantic_id = str(semantic.get("id") or "").strip() + semantic_label = str(semantic.get("label") or semantic_id).strip() + semantic_tokens = _tokens(semantic_label) | _tokens(semantic_id) + semantic_compact = _normalized_text(semantic_label) + scored = [] + for visual in available: + visual_id = str(visual.get("id") or "").strip() + searchable = _object_text(visual) + visual_tokens = _tokens(searchable) + score = len(semantic_tokens & visual_tokens) / max(1, len(semantic_tokens | visual_tokens)) + if semantic_id and semantic_id == visual_id: + score += 10.0 + visual_compact = _normalized_text(searchable) + if semantic_compact and visual_compact: + if semantic_compact == visual_compact: + score += 4.0 + elif semantic_compact in visual_compact or visual_compact in semantic_compact: + score += 2.0 + scored.append((score, -_center(visual)[0], -_center(visual)[1], visual)) + if not scored: + records.append({ + "semantic_panel_id": semantic_id, + "semantic_panel_label": semantic_label, + "visual_panel_id": None, + "match_score": 0.0, + "status": "unmapped", + }) + continue + scored.sort(key=lambda value: (value[0], value[1], value[2]), reverse=True) + best_score, _, _, chosen = scored[0] + status = "mapped" + if best_score <= 0 and semantic_index < len(visual_panels): + ordered_candidate = visual_panels[semantic_index] + if ordered_candidate in available: + chosen = ordered_candidate + status = "mapped_spatial_fallback" + elif best_score <= 0: + records.append({ + "semantic_panel_id": semantic_id, + "semantic_panel_label": semantic_label, + "visual_panel_id": None, + "match_score": 0.0, + "status": "unmapped", + }) + continue + available.remove(chosen) + visual_id = str(chosen.get("id") or "").strip() + mapping[semantic_id] = visual_id + records.append({ + "semantic_panel_id": semantic_id, + "semantic_panel_label": semantic_label, + "visual_panel_id": visual_id, + "visual_panel_title": str(chosen.get("title") or chosen.get("label") or visual_id), + "match_score": round(max(0.0, best_score), 4), + "status": status, + }) + return mapping, records + + +def _assign_entities( + entities: list[dict], + program: dict, + semantic_panels: list[dict] | None = None, +) -> tuple[dict[str, dict], list[dict], dict[str, str], list[dict]]: slots = [item for item in program.get("slots", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] cards = [item for item in program.get("cards", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] panels = [item for item in program.get("panels", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] - objects = [*cards, *slots, *panels] + primary_objects = [*cards, *slots] seen: set[str] = set() - objects = [item for item in objects if not (str(item.get("id")) in seen or seen.add(str(item.get("id"))))] + primary_objects = [item for item in primary_objects if not (str(item.get("id")) in seen or seen.add(str(item.get("id"))))] text_items = [ item for item in (program.get("text_program", {}).get("items", []) if isinstance(program.get("text_program"), dict) else []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) and not str(item.get("id") or "").startswith("semantic_text_") - and not item.get("semantic_entity_id") + ] + text_objects = [ + { + "id": f"semantic_text_anchor_{str(item.get('id') or index)}", + "title": str(item.get("text") or ""), + "bbox_percent": dict(item["bbox_percent"]), + "panel_id": _visual_panel_id(item, panels), + "text_item_id": str(item.get("id") or ""), + "semantic_instance_id": str(item.get("semantic_entity_id") or "") or None, + "_semantic_text_anchor": True, + } + for index, item in enumerate(text_items, start=1) + if str(item.get("text") or "").strip() ] def object_search_text(obj: dict) -> str: + if obj.get("_semantic_text_anchor"): + return str(obj.get("title") or "") + if obj in panels: + return _object_text(obj) values = [_object_text(obj)] box = obj.get("bbox_percent") or {} for text_item in text_items: @@ -302,49 +426,218 @@ def object_search_text(obj: dict) -> str: def object_priority(obj: dict) -> int: if obj in cards: - return 2 + return 3 if obj in slots: + return 2 + if obj.get("_semantic_text_anchor"): return 1 return 0 - objects.sort(key=lambda item: (_center(item)[1], _center(item)[0], str(item.get("id") or ""))) - available = list(objects) - mapping: dict[str, dict] = {} - records: list[dict] = [] - for entity in entities: + def score_candidates(entity: dict, candidates: list[dict]) -> list[tuple]: entity_tokens = _tokens(entity["label"]) | _tokens(entity["id"]) + semantic_panel_id = str(entity.get("panel_id") or "").strip() scored = [] - for obj in available: + for obj in candidates: searchable = object_search_text(obj) obj_tokens = _tokens(searchable) score = len(entity_tokens & obj_tokens) / max(1, len(entity_tokens | obj_tokens)) semantic_instance_id = str(obj.get("semantic_instance_id") or "").strip() if semantic_instance_id and semantic_instance_id == entity["id"]: score += 10.0 - semantic_panel_id = str(obj.get("semantic_panel_id") or "").strip() - if semantic_panel_id and semantic_panel_id == str(entity.get("panel_id") or ""): + object_semantic_panel_id = str(obj.get("semantic_panel_id") or "").strip() + if object_semantic_panel_id and object_semantic_panel_id == semantic_panel_id: score += 0.5 - entity_compact = re.sub(r"[^a-z0-9]+", "", str(entity.get("label") or "").casefold()) - object_compact = re.sub(r"[^a-z0-9]+", "", searchable.casefold()) - if entity_compact and entity_compact in object_compact: + entity_compact = _normalized_text(entity.get("label")) + object_compact = _normalized_text(searchable) + if entity_compact and entity_compact == object_compact: + score += 2.0 + elif entity_compact and entity_compact in object_compact: score += 1.0 scored.append((score, object_priority(obj), -_center(obj)[1], -_center(obj)[0], obj)) - if not scored: + scored.sort(key=lambda value: (value[0], value[1], value[2], value[3]), reverse=True) + return scored + + def strong_primary_match(entity: dict, scored: list[tuple]) -> bool: + if not scored or scored[0][0] <= 0: + return False + entity_compact = _normalized_text(entity.get("label")) + return bool(entity_compact and entity_compact in _normalized_text(object_search_text(scored[0][4]))) + + def exact_text_match(entity: dict, scored: list[tuple]) -> bool: + if not scored or scored[0][0] <= 0: + return False + return _normalized_text(entity.get("label")) == _normalized_text(object_search_text(scored[0][4])) + + def composite_text_candidates(entity: dict, candidates: list[dict]) -> list[dict]: + """Build exact semantic anchors from adjacent OCR line fragments. + + OCR commonly returns a visible label such as Position Embedding as two + stacked regions. These composites are entity-specific and require an + exact normalized concatenation, so a short label such as Class still + cannot claim the Class Token region. + """ + + target = _normalized_text(entity.get("label")) + if not target or len(candidates) < 2: + return [] + + def local_sequence(items: tuple[dict, ...]) -> bool: + ordered = sorted(items, key=lambda item: (_center(item)[1], _center(item)[0])) + for previous, current in zip(ordered, ordered[1:]): + prev_box = previous["bbox_percent"] + cur_box = current["bbox_percent"] + prev_left = float(prev_box.get("x") or 0) + prev_top = float(prev_box.get("y") or 0) + prev_right = prev_left + float(prev_box.get("w") or 0) + prev_bottom = prev_top + float(prev_box.get("h") or 0) + cur_left = float(cur_box.get("x") or 0) + cur_top = float(cur_box.get("y") or 0) + cur_right = cur_left + float(cur_box.get("w") or 0) + cur_bottom = cur_top + float(cur_box.get("h") or 0) + max_height = max(float(prev_box.get("h") or 0), float(cur_box.get("h") or 0), 0.001) + vertical_gap = max(0.0, cur_top - prev_bottom) + horizontal_gap = max(0.0, cur_left - prev_right, prev_left - cur_right) + horizontal_overlap = min(prev_right, cur_right) - max(prev_left, cur_left) + vertical_overlap = min(prev_bottom, cur_bottom) - max(prev_top, cur_top) + stacked = horizontal_overlap >= -0.012 and vertical_gap <= max(0.025, max_height * 1.35) + inline = vertical_overlap >= -max_height * 0.35 and horizontal_gap <= max(0.035, max_height * 2.0) + if not (stacked or inline): + return False + return True + + results = [] + max_members = min(4, len(candidates)) + for member_count in range(2, max_members + 1): + for members in combinations(candidates, member_count): + panel_ids = {str(item.get("panel_id") or "") for item in members} + if len(panel_ids) > 1 or not local_sequence(members): + continue + ordered = sorted(members, key=lambda item: (_center(item)[1], _center(item)[0])) + if "".join(_normalized_text(item.get("title")) for item in ordered) != target: + continue + boxes = [item["bbox_percent"] for item in ordered] + left = min(float(box.get("x") or 0) for box in boxes) + top = min(float(box.get("y") or 0) for box in boxes) + right = max(float(box.get("x") or 0) + float(box.get("w") or 0) for box in boxes) + bottom = max(float(box.get("y") or 0) + float(box.get("h") or 0) for box in boxes) + member_ids = [str(item.get("id") or "") for item in ordered] + results.append({ + "id": "semantic_text_anchor_composite_" + "__".join(member_ids), + "title": " ".join(str(item.get("title") or "").strip() for item in ordered), + "bbox_percent": { + "x": round(left, 4), + "y": round(top, 4), + "w": round(right - left, 4), + "h": round(bottom - top, 4), + }, + "panel_id": str(ordered[0].get("panel_id") or "") or None, + "_semantic_text_anchor": True, + "_semantic_text_anchor_member_ids": member_ids, + }) + results.sort(key=lambda item: (len(item["_semantic_text_anchor_member_ids"]), _center(item)[1], _center(item)[0])) + return results + + panel_mapping, panel_records = _resolve_panel_mapping(semantic_panels or [], panels) + primary_objects.sort(key=lambda item: (_center(item)[1], _center(item)[0], str(item.get("id") or ""))) + available_primary = list(primary_objects) + available_text = list(text_objects) + available_panels = list(panels) + mapping: dict[str, dict] = {} + records: list[dict] = [] + for entity in entities: + semantic_panel_id = str(entity.get("panel_id") or "").strip() + expected_visual_panel_id = panel_mapping.get(semantic_panel_id) + eligible_primary = [ + obj for obj in available_primary + if not expected_visual_panel_id or _visual_panel_id(obj, panels) == expected_visual_panel_id + ] + eligible_text = [ + obj for obj in available_text + if not expected_visual_panel_id or _visual_panel_id(obj, panels) == expected_visual_panel_id + ] + eligible_panels = [ + obj for obj in available_panels + if not expected_visual_panel_id or _visual_panel_id(obj, panels) == expected_visual_panel_id + ] + if expected_visual_panel_id and not (eligible_primary or eligible_text or eligible_panels): + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "semantic_panel_id": semantic_panel_id, + "expected_visual_panel_id": expected_visual_panel_id, + "object_id": None, + "object_panel_id": None, + "panel_match": False, + "match_score": 0.0, + "status": "blocked_panel_constraint", + }) + continue + + scored_primary = score_candidates(entity, eligible_primary) + eligible_text_with_composites = [*eligible_text, *composite_text_candidates(entity, eligible_text)] + scored_text = score_candidates(entity, eligible_text_with_composites) + scored_panels = score_candidates(entity, eligible_panels) + scored: list[tuple] = [] + chosen_pool: list[dict] | None = None + if strong_primary_match(entity, scored_primary): + scored, chosen_pool = scored_primary, available_primary + elif exact_text_match(entity, scored_text): + scored, chosen_pool = scored_text, available_text + elif scored_primary and scored_primary[0][0] > 0: + scored, chosen_pool = scored_primary, available_primary + elif scored_panels and scored_panels[0][0] > 0: + scored, chosen_pool = scored_panels, available_panels + elif expected_visual_panel_id: + records.append({ + "entity_id": entity["id"], + "label": entity["label"], + "role": entity["role"], + "semantic_panel_id": semantic_panel_id, + "expected_visual_panel_id": expected_visual_panel_id, + "object_id": None, + "object_panel_id": None, + "panel_match": False, + "match_score": 0.0, + "status": "blocked_panel_constraint", + }) + continue + elif eligible_primary: + scored, chosen_pool = scored_primary, available_primary + elif eligible_panels: + scored, chosen_pool = scored_panels, available_panels + + if not scored or chosen_pool is None: records.append({"entity_id": entity["id"], "label": entity["label"], "status": "unmapped"}) continue - scored.sort(key=lambda value: (value[0], value[1], value[2], value[3]), reverse=True) - chosen = scored[0][4] if scored[0][0] > 0 else available[0] - available.remove(chosen) + + chosen = scored[0][4] + composite_member_ids = set(chosen.get("_semantic_text_anchor_member_ids") or []) + if composite_member_ids: + available_text[:] = [ + item for item in available_text + if str(item.get("id") or "") not in composite_member_ids + ] + else: + chosen_pool.remove(chosen) mapping[entity["id"]] = chosen + object_panel_id = _visual_panel_id(chosen, panels) + panel_match = not expected_visual_panel_id or object_panel_id == expected_visual_panel_id records.append({ "entity_id": entity["id"], "label": entity["label"], "role": entity["role"], "object_id": chosen.get("id"), + "semantic_panel_id": semantic_panel_id or None, + "expected_visual_panel_id": expected_visual_panel_id, + "object_panel_id": object_panel_id, + "object_kind": "text_anchor" if chosen.get("_semantic_text_anchor") else ("panel" if chosen in panels else ("card" if chosen in cards else "slot")), + "text_anchor_member_ids": list(chosen.get("_semantic_text_anchor_member_ids") or []), + "panel_match": panel_match, "match_score": round(max(0.0, scored[0][0]), 4), - "status": "mapped" if scored[0][0] > 0 else "mapped_spatial_fallback", + "status": ("mapped" if scored[0][0] > 0 else "mapped_spatial_fallback") if panel_match else "blocked_cross_panel", }) - return mapping, records + return mapping, records, panel_mapping, panel_records def _inside(inner: dict, outer: dict) -> bool: @@ -356,6 +649,14 @@ def _inside(inner: dict, outer: dict) -> bool: def _label_bbox(obj: dict) -> dict[str, float]: box = obj["bbox_percent"] x, y, w, h = (float(box[key]) for key in ("x", "y", "w", "h")) + if "paper_native" in str(obj.get("reference_crop_role") or "").casefold(): + label_h = min(0.05, max(0.035, h * 0.24)) + return { + "x": round(x, 4), + "y": round(max(0.0, y - label_h - 0.008), 4), + "w": round(w, 4), + "h": round(label_h, 4), + } label_h = min(0.055, max(0.026, h * 0.18)) return {"x": round(x, 4), "y": round(max(0.0, y), 4), "w": round(w, 4), "h": round(label_h, 4)} @@ -382,6 +683,96 @@ def _anchor(source: dict, target: dict) -> list[list[float]]: return clean +def _segment_hits_box(start: list[float], end: list[float], box: dict, margin: float = 0.004) -> bool: + left = float(box.get("x") or 0) + margin + top = float(box.get("y") or 0) + margin + right = float(box.get("x") or 0) + float(box.get("w") or 0) - margin + bottom = float(box.get("y") or 0) + float(box.get("h") or 0) - margin + if abs(start[0] - end[0]) < 1e-6: + x = start[0] + low, high = sorted((start[1], end[1])) + return left < x < right and max(low, top) < min(high, bottom) + if abs(start[1] - end[1]) < 1e-6: + y = start[1] + low, high = sorted((start[0], end[0])) + return top < y < bottom and max(low, left) < min(high, right) + return False + + +def _route_around_cards(source: dict, target: dict, cards: list[dict], panel_box: dict | None = None) -> tuple[list[list[float]], str]: + base = _anchor(source, target) + source_id = str(source.get("id") or "") + target_id = str(target.get("id") or "") + obstacles = [ + card for card in cards + if str(card.get("id") or "") not in {source_id, target_id} + and isinstance(card.get("bbox_percent"), dict) + and (not panel_box or _inside(card.get("bbox_percent") or {}, panel_box)) + ] + + def route_hits(path: list[list[float]]) -> bool: + return any( + _segment_hits_box(start, end, card["bbox_percent"]) + for start, end in zip(path, path[1:]) + for card in obstacles + ) + + if not route_hits(base): + return base, "direct_orthogonal" + start, end = base[0], base[-1] + vertical = abs(end[1] - start[1]) >= abs(end[0] - start[0]) + candidates: list[list[list[float]]] = [] + if vertical: + obstacle_left = min(float(card["bbox_percent"].get("x") or 0) for card in obstacles) + obstacle_right = max(float(card["bbox_percent"].get("x") or 0) + float(card["bbox_percent"].get("w") or 0) for card in obstacles) + lanes = [obstacle_left - 0.012, obstacle_right + 0.012] + if panel_box: + panel_left = float(panel_box.get("x") or 0) + 0.008 + panel_right = float(panel_box.get("x") or 0) + float(panel_box.get("w") or 0) - 0.008 + lanes = [lane for lane in lanes if panel_left <= lane <= panel_right] + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_y = float(source_box.get("y") or 0) + float(source_box.get("h") or 0) / 2 + target_center_y = float(target_box.get("y") or 0) + float(target_box.get("h") or 0) / 2 + for lane in lanes: + use_right = lane >= (start[0] + end[0]) / 2 + source_side = float(source_box.get("x") or 0) + (float(source_box.get("w") or 0) if use_right else 0.0) + target_side = float(target_box.get("x") or 0) + (float(target_box.get("w") or 0) if use_right else 0.0) + candidates.append([ + [round(source_side, 4), round(source_center_y, 4)], + [round(lane, 4), round(source_center_y, 4)], + [round(lane, 4), round(target_center_y, 4)], + [round(target_side, 4), round(target_center_y, 4)], + ]) + else: + obstacle_top = min(float(card["bbox_percent"].get("y") or 0) for card in obstacles) + obstacle_bottom = max(float(card["bbox_percent"].get("y") or 0) + float(card["bbox_percent"].get("h") or 0) for card in obstacles) + lanes = [obstacle_top - 0.012, obstacle_bottom + 0.012] + if panel_box: + panel_top = float(panel_box.get("y") or 0) + 0.008 + panel_bottom = float(panel_box.get("y") or 0) + float(panel_box.get("h") or 0) - 0.008 + lanes = [lane for lane in lanes if panel_top <= lane <= panel_bottom] + source_box = source["bbox_percent"] + target_box = target["bbox_percent"] + source_center_x = float(source_box.get("x") or 0) + float(source_box.get("w") or 0) / 2 + target_center_x = float(target_box.get("x") or 0) + float(target_box.get("w") or 0) / 2 + for lane in lanes: + use_bottom = lane >= (start[1] + end[1]) / 2 + source_side = float(source_box.get("y") or 0) + (float(source_box.get("h") or 0) if use_bottom else 0.0) + target_side = float(target_box.get("y") or 0) + (float(target_box.get("h") or 0) if use_bottom else 0.0) + candidates.append([ + [round(source_center_x, 4), round(source_side, 4)], + [round(source_center_x, 4), round(lane, 4)], + [round(target_center_x, 4), round(lane, 4)], + [round(target_center_x, 4), round(target_side, 4)], + ]) + valid = [path for path in candidates if not route_hits(path)] + if not valid: + return base, "direct_orthogonal_obstacle_unresolved" + valid.sort(key=lambda path: sum(abs(b[0] - a[0]) + abs(b[1] - a[1]) for a, b in zip(path, path[1:]))) + return valid[0], "obstacle_avoiding_orthogonal" + + def _preserve_local_reference_arrows(program: dict, mapping: dict[str, dict]) -> tuple[list[dict], list[str]]: mapped_objects = list(mapping.values()) all_objects = { @@ -423,24 +814,153 @@ def mapped_container(obj: dict | None) -> str | None: def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | Path) -> tuple[dict, dict]: out = Path(out_dir) entities = _entities(contract) + semantic_panels = _semantic_panels(contract) relations, relation_source = _relations(contract) - mapping, mapping_records = _assign_entities(entities, program) + mapping, mapping_records, panel_mapping, panel_mapping_records = _assign_entities(entities, program, semantic_panels) mapped_boxes = [obj["bbox_percent"] for obj in mapping.values()] text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {"items": []} existing = [item for item in text_program.get("items", []) if isinstance(item, dict)] - preserved = [ - item for item in existing - if not str(item.get("id") or "").startswith("semantic_text_") - and not item.get("semantic_entity_id") - and not any(_inside(item.get("bbox_percent") or {}, box) for box in mapped_boxes) - ] + preserved = [dict(item) for item in existing if not str(item.get("id") or "").startswith("semantic_text_")] semantic_text = [] semantic_reference_regions = [] + + def exact_local_fragment_sequence(items: tuple[dict, ...], expected_text: str) -> bool: + ordered = sorted(items, key=lambda item: (_center(item)[1], _center(item)[0])) + if "".join(_normalized_text(item.get("text")) for item in ordered) != expected_text: + return False + for previous, current in zip(ordered, ordered[1:]): + prev_box = previous.get("bbox_percent") or {} + cur_box = current.get("bbox_percent") or {} + prev_left = float(prev_box.get("x") or 0) + prev_top = float(prev_box.get("y") or 0) + prev_right = prev_left + float(prev_box.get("w") or 0) + prev_bottom = prev_top + float(prev_box.get("h") or 0) + cur_left = float(cur_box.get("x") or 0) + cur_top = float(cur_box.get("y") or 0) + cur_right = cur_left + float(cur_box.get("w") or 0) + cur_bottom = cur_top + float(cur_box.get("h") or 0) + max_height = max(float(prev_box.get("h") or 0), float(cur_box.get("h") or 0), 0.001) + vertical_gap = max(0.0, cur_top - prev_bottom) + horizontal_gap = max(0.0, cur_left - prev_right, prev_left - cur_right) + horizontal_overlap = min(prev_right, cur_right) - max(prev_left, cur_left) + vertical_overlap = min(prev_bottom, cur_bottom) - max(prev_top, cur_top) + stacked = horizontal_overlap >= -0.012 and vertical_gap <= max(0.025, max_height * 1.35) + inline = vertical_overlap >= -max_height * 0.35 and horizontal_gap <= max(0.035, max_height * 2.0) + if not (stacked or inline): + return False + return True + for entity in entities: obj = mapping.get(entity["id"]) if not obj: continue box = _label_bbox(obj) + object_box = obj.get("bbox_percent") if isinstance(obj.get("bbox_percent"), dict) else {} + target_ids = {str(obj.get("id") or ""), str(obj.get("panel_id") or "")} + target_ids.discard("") + entity_norm = _normalized_text(entity["label"]) + spatial_candidates = [ + item for item in preserved + if ( + str(item.get("target_id") or "") in target_ids + or _inside(item.get("bbox_percent") or {}, object_box) + ) + ] + reference_matches = [ + item for item in spatial_candidates + if entity_norm and _normalized_text(item.get("text")) + and (entity_norm in _normalized_text(item.get("text")) or _normalized_text(item.get("text")) in entity_norm) + ] + if reference_matches: + best_match = max( + reference_matches, + key=lambda item: ( + _normalized_text(item.get("text")) == entity_norm, + str(item.get("target_id") or "") == str(obj.get("id") or ""), + -abs(len(_normalized_text(item.get("text"))) - len(entity_norm)), + ), + ) + original_text = str(best_match.get("text") or "") + geometry_matches = [best_match] + if _normalized_text(best_match.get("text")) != entity_norm: + fragment_sequences = [] + max_members = min(4, len(reference_matches)) + for member_count in range(2, max_members + 1): + for members in combinations(reference_matches, member_count): + if best_match not in members: + continue + if exact_local_fragment_sequence(members, entity_norm): + fragment_sequences.append(members) + if fragment_sequences: + break + if fragment_sequences: + fragment_sequences.sort(key=lambda members: ( + len(members), + max(_center(item)[1] for item in members) - min(_center(item)[1] for item in members), + )) + geometry_matches = list(fragment_sequences[0]) + if len(geometry_matches) > 1: + reference_boxes = [item["bbox_percent"] for item in geometry_matches] + left = min(float(item.get("x") or 0) for item in reference_boxes) + top = min(float(item.get("y") or 0) for item in reference_boxes) + right = max(float(item.get("x") or 0) + float(item.get("w") or 0) for item in reference_boxes) + bottom = max(float(item.get("y") or 0) + float(item.get("h") or 0) for item in reference_boxes) + merged_box = { + "x": round(left, 4), + "y": round(top, 4), + "w": round(right - left, 4), + "h": round(bottom - top, 4), + } + best_match.update({ + "bbox_percent": merged_box, + "center_percent": { + "x": round(merged_box["x"] + merged_box["w"] / 2, 4), + "y": round(merged_box["y"] + merged_box["h"] / 2, 4), + }, + "width_percent": merged_box["w"], + "height_percent": merged_box["h"], + "semantic_merged_reference_text_ids": [ + str(item.get("id") or "") for item in geometry_matches + ], + }) + display_text = entity["label"] + if len(geometry_matches) > 1: + ordered_geometry_matches = sorted( + geometry_matches, + key=lambda item: (_center(item)[1], _center(item)[0]), + ) + display_text = "\n".join( + str(item.get("text") or "").strip() + for item in ordered_geometry_matches + if str(item.get("text") or "").strip() + ) + best_match.update({ + "text": display_text, + "semantic_contract_text": entity["label"], + "target_id": obj.get("id"), + "semantic_entity_id": entity["id"], + "semantic_panel_id": entity.get("panel_id"), + "semantic_evidence_ids": entity["evidence_ids"], + "semantic_original_ocr_text": original_text, + "reference_binding": "paper_semantic_contract_reference_text", + "fit_strategy": best_match.get("fit_strategy") or "reference_text_geometry_exact", + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + best_id = str(best_match.get("id") or "") + duplicate_ids = { + str(item.get("id") or "") for item in geometry_matches + if str(item.get("id") or "") != best_id + } + if duplicate_ids: + preserved = [item for item in preserved if str(item.get("id") or "") not in duplicate_ids] + continue + if str(obj.get("id") or "") not in {str(panel.get("id") or "") for panel in program.get("panels", []) if isinstance(panel, dict)}: + preserved = [ + item for item in preserved + if not _inside(item.get("bbox_percent") or {}, object_box) + ] reference_text_id = f"semantic_ref_text_{entity['id']}" center = {"x": round(box["x"] + box["w"] / 2, 4), "y": round(box["y"] + box["h"] / 2, 4)} estimated_font_ratio = round(max(0.006, box["h"] * 0.55), 6) @@ -490,6 +1010,89 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "source": "reference_slot_semantic_overlay_geometry", "editable_in": "pptx", }) + + panels_by_id = { + str(item.get("id") or ""): item + for item in program.get("panels", []) + if isinstance(item, dict) and item.get("id") and isinstance(item.get("bbox_percent"), dict) + } + panel_title_records = [] + for semantic_panel in semantic_panels: + semantic_panel_id = str(semantic_panel.get("id") or "") + visual_panel_id = panel_mapping.get(semantic_panel_id) + visual_panel = panels_by_id.get(str(visual_panel_id or "")) + if not visual_panel: + panel_title_records.append({ + "semantic_panel_id": semantic_panel_id, + "visual_panel_id": visual_panel_id, + "status": "unmapped", + }) + continue + label = str(semantic_panel.get("label") or semantic_panel_id) + panel_box = visual_panel["bbox_percent"] + header_limit = float(panel_box.get("y") or 0) + min(0.09, max(0.045, float(panel_box.get("h") or 0) * 0.08)) + candidates = [ + item for item in preserved + if str(item.get("target_id") or "") == visual_panel_id + and str(item.get("role") or "").casefold() == "panel_title" + and not item.get("semantic_entity_id") + and _center(item)[1] <= header_limit + ] + label_norm = _normalized_text(label) + fragments = [ + item for item in candidates + if _normalized_text(item.get("text")) and _normalized_text(item.get("text")) in label_norm + ] + best = max(fragments or candidates, key=lambda item: float((item.get("bbox_percent") or {}).get("w") or 0), default=None) + if best: + original_text = str(best.get("text") or "") + panel_title_box = { + "x": round(float(panel_box.get("x") or 0) + 0.008, 4), + "y": round(float(panel_box.get("y") or 0) + 0.004, 4), + "w": round(max(0.02, float(panel_box.get("w") or 0) - 0.016), 4), + "h": round(min(0.06, max(0.038, float(panel_box.get("h") or 0) * 0.07)), 4), + } + display_label = label + if len(label) > 34 and panel_title_box["w"] < 0.30 and " & " in label: + display_label = label.replace(" & ", " &\n", 1) + best.update({ + "text": display_label, + "target_id": visual_panel_id, + "semantic_panel_id": semantic_panel_id, + "semantic_panel_title": True, + "semantic_contract_text": label, + "semantic_original_ocr_text": original_text, + "reference_binding": "paper_semantic_contract_panel_title", + "bbox_percent": panel_title_box, + "center_percent": { + "x": round(panel_title_box["x"] + panel_title_box["w"] / 2, 4), + "y": round(panel_title_box["y"] + panel_title_box["h"] / 2, 4), + }, + "width_percent": panel_title_box["w"], + "height_percent": panel_title_box["h"], + "editable_in": "pptx", + "visible": True, + "layer_ownership": "editable_text_layer", + }) + best_id = str(best.get("id") or "") + duplicate_ids = { + str(item.get("id") or "") for item in fragments + if str(item.get("id") or "") != best_id + } + if duplicate_ids: + preserved = [item for item in preserved if str(item.get("id") or "") not in duplicate_ids] + panel_title_records.append({ + "semantic_panel_id": semantic_panel_id, + "visual_panel_id": visual_panel_id, + "text_id": best_id, + "status": "replaced_exact", + }) + else: + panel_title_records.append({ + "semantic_panel_id": semantic_panel_id, + "visual_panel_id": visual_panel_id, + "status": "missing_reference_title", + }) text_program = { **text_program, "summary": "Editable text with paper-grounded exact labels overriding image-derived labels in mapped semantic objects.", @@ -515,6 +1118,22 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | item for item in reference_geometry.get("text_regions", []) if isinstance(item, dict) and not str(item.get("id") or "").startswith("semantic_ref_text_") ] + exact_item_by_reference_id = { + str(item.get("source_reference_text_id") or ""): item + for item in text_program["items"] + if item.get("source_reference_text_id") and (item.get("semantic_entity_id") or item.get("semantic_panel_title")) + } + for region in geometry_regions: + region_id = str(region.get("id") or "") + if region_id in exact_item_by_reference_id: + exact_item = exact_item_by_reference_id[region_id] + region["semantic_original_reference_text"] = str(region.get("text") or region.get("raw_text") or "") + region["text"] = str(exact_item.get("text") or "") + region["raw_text"] = str(exact_item.get("text") or "") + for key in ("bbox_percent", "center_percent", "width_percent", "height_percent"): + if key in exact_item: + region[key] = exact_item[key] + region["semantic_text_replaced"] = True reference_geometry["text_regions"] = geometry_regions + semantic_reference_regions reference_geometry["semantic_overlay_policy"] = "exact paper terminology bound to reference-derived slot geometry" @@ -544,6 +1163,20 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | semantic_arrows = [] skipped_relations = [] + blueprint_routes = { + ( + str(item.get("semantic_source_id") or item.get("source_id") or ""), + str(item.get("semantic_target_id") or item.get("target_id") or ""), + ): item + for item in program.get("arrows", []) if isinstance(item, dict) + if str(item.get("binding_source") or "") == "generation_semantic_blueprint" + } + cards = [item for item in program.get("cards", []) if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict)] + panels_by_visual_id = { + str(item.get("id") or ""): item + for item in program.get("panels", []) + if isinstance(item, dict) and isinstance(item.get("bbox_percent"), dict) + } for index, relation in enumerate(relations, start=1): source_id = str(relation.get("source") or relation.get("source_id") or "") target_id = str(relation.get("target") or relation.get("target_id") or "") @@ -551,6 +1184,17 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | if not source or not target: skipped_relations.append({"source": source_id, "target": target_id, "reason": "unmapped_endpoint"}) continue + source_panel_id = _visual_panel_id(source, list(panels_by_visual_id.values())) + target_panel_id = _visual_panel_id(target, list(panels_by_visual_id.values())) + panel_box = panels_by_visual_id.get(str(source_panel_id or ""), {}).get("bbox_percent") if source_panel_id == target_panel_id else None + blueprint_route = blueprint_routes.get((source_id, target_id)) + if blueprint_route and isinstance(blueprint_route.get("path_percent"), list) and len(blueprint_route["path_percent"]) >= 2: + path_percent = blueprint_route["path_percent"] + route_generation_status = "generation_blueprint_preserved" + routing_algorithm = str(blueprint_route.get("routing_algorithm") or "generation_semantic_blueprint") + else: + path_percent, route_generation_status = _route_around_cards(source, target, cards, panel_box) + routing_algorithm = "panel_card_obstacle_aware_orthogonal" semantic_arrows.append({ "id": str(relation.get("id") or f"semantic_relation_{index:02d}"), "source_id": str(source.get("id")), @@ -561,19 +1205,31 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "relation_type": str(relation.get("type") or relation.get("relation_type") or "data_flow"), "label": str(relation.get("label") or relation.get("statement") or ""), "evidence_ids": list(relation.get("evidence_ids") or []), - "path_percent": _anchor(source, target), + "path_percent": path_percent, "source_anchor": "auto", "target_anchor": "auto", "control_kind": "elbow_connector", "render_style": "elbow_connector", "route_intent": "paper_semantic_relation", - "stroke_color": "#3F5063", - "stroke_width_pt": 1.8, - "arrowhead": "triangle", + "routing_algorithm": routing_algorithm, + "route_generation_status": route_generation_status, + "binding_source": "generation_semantic_blueprint" if blueprint_route else "paper_semantic_contract", + "reference_locked": bool(blueprint_route), + "reference_path_preserved": bool(blueprint_route), + "stroke_color": str((blueprint_route or {}).get("stroke_color") or "#3F5063"), + "stroke_width_pt": float((blueprint_route or {}).get("stroke_width_pt") or 1.8), + "arrowhead": str((blueprint_route or {}).get("arrowhead") or "triangle"), "editable_in": "pptx", "render_policy": "ppt_shape_not_image_asset", }) + paper_native = any( + "paper_native" in str(item.get("reference_crop_role") or "").casefold() + for item in program.get("slots", []) if isinstance(item, dict) + ) preserved_reference_arrows, replaced_reference_arrow_ids = _preserve_local_reference_arrows(program, mapping) + if paper_native: + replaced_reference_arrow_ids.extend(str(item.get("id") or "seed_control") for item in preserved_reference_arrows) + preserved_reference_arrows = [] if relation_source == "panel_graphs": heuristic_defaults = [ item for item in preserved_reference_arrows @@ -588,13 +1244,21 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | if mapping: program["arrows"] = [*preserved_reference_arrows, *semantic_arrows] - low_confidence_mappings = [item for item in mapping_records if float(item.get("match_score") or 0) <= 0] + low_confidence_mappings = [ + item for item in mapping_records + if float(item.get("match_score") or 0) <= 0 and str(item.get("status") or "") != "blocked_panel_constraint" + ] + cross_panel_mappings = [item for item in mapping_records if item.get("panel_match") is False and item.get("object_id")] + panel_constraint_failures = [item for item in mapping_records if item.get("status") == "blocked_panel_constraint"] + unmapped_panels = [item for item in panel_mapping_records if item.get("status") == "unmapped"] + blocked = bool(cross_panel_mappings or panel_constraint_failures) + complete = len(mapping) == len(entities) and not skipped_relations and not low_confidence_mappings and not unmapped_panels report = { "summary": "Paper semantic contract application report.", - "status": "pass" if len(mapping) == len(entities) and not skipped_relations and not low_confidence_mappings else "warning", + "status": "blocked" if blocked else ("pass" if complete else "warning"), "semantic_authority": "paper_ground_truth", "contract_graph_source": relation_source, - "visual_authority": "generated_reference_image", + "visual_authority": "paper_native_slot_assets" if paper_native else "generated_reference_image", "entity_count": len(entities), "mapped_entity_count": len(mapping), "relation_count": len(relations), @@ -602,9 +1266,18 @@ def apply_paper_semantic_contract(program: dict, contract: dict, out_dir: str | "preserved_reference_arrow_count": len(preserved_reference_arrows), "replaced_reference_arrow_ids": replaced_reference_arrow_ids, "mappings": mapping_records, + "panel_mappings": panel_mapping_records, + "panel_title_bindings": panel_title_records, + "cross_panel_mappings": cross_panel_mappings, + "panel_constraint_failures": panel_constraint_failures, + "unmapped_panels": unmapped_panels, "low_confidence_mappings": low_confidence_mappings, "skipped_relations": skipped_relations, - "policy": "exact labels and scientific relations come from the paper contract; the image supplies layout and style only", + "policy": ( + "exact labels, scientific objects, and relations come from the paper contract; the generated seed supplies geometry and style only" + if paper_native + else "exact labels and scientific relations come from the paper contract; the image supplies layout and style only; panel identity is a hard constraint for repeated labels" + ), } write_json(out / "paper_semantic_contract.json", contract) write_json(out / "semantic_binding_report.json", report) diff --git a/rfs/control_localizer.py b/rfs/control_localizer.py index 63b8f20..77ca9e2 100644 --- a/rfs/control_localizer.py +++ b/rfs/control_localizer.py @@ -1,13 +1,27 @@ from __future__ import annotations import math +import inspect from pathlib import Path from typing import Callable from PIL import Image, ImageDraw +from .utils import read_json, write_json from .cv_compat import hough_line_coordinates +VALID_RENDER_STYLES = { + "filled_block_arrow", + "line_connector", + "elbow_connector", + "branch_line_connector", + "dashed_loop_connector", +} +VALID_ROUTE_INTENTS = {"straight", "orthogonal", "branch", "loop"} +VALID_VISUAL_WEIGHTS = {"chunky", "normal", "thin"} +VALID_PREFERRED_AXES = {"horizontal", "vertical", "horizontal_first", "vertical_first"} +VALID_LINE_PATTERNS = {"solid", "dash", "dashed"} + def _center(box: dict) -> tuple[float, float]: return float(box["x"]) + float(box["w"]) / 2, float(box["y"]) + float(box["h"]) / 2 @@ -20,6 +34,136 @@ def _nearest_slot(point: tuple[float, float], slots: list[dict]) -> dict | None: return min(slots, key=lambda slot: (_center(slot["bbox_percent"])[0] - px) ** 2 + (_center(slot["bbox_percent"])[1] - py) ** 2) +def _distance_to_bbox(point: tuple[float, float], obj: dict | None) -> float: + if not obj or not isinstance(obj.get("bbox_percent"), dict): + return 1.0 + box = obj["bbox_percent"] + px, py = point + x = float(box["x"]) + y = float(box["y"]) + w = float(box["w"]) + h = float(box["h"]) + dx = max(x - px, 0.0, px - (x + w)) + dy = max(y - py, 0.0, py - (y + h)) + return math.hypot(dx, dy) + + +def _coerce_choice(value, allowed: set[str], default: str = "") -> str: + text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + aliases = { + "block_arrow": "filled_block_arrow", + "chunky_arrow": "filled_block_arrow", + "thick_arrow": "filled_block_arrow", + "straight_arrow": "line_connector", + "elbow": "elbow_connector", + "orthogonal_connector": "elbow_connector", + "branch_connector": "branch_line_connector", + "dashed_loop": "dashed_loop_connector", + "loop_connector": "dashed_loop_connector", + "dashed": "dash", + } + text = aliases.get(text, text) + return text if text in allowed else default + + +def _slot_by_id(slots: list[dict]) -> dict[str, dict]: + return {str(slot.get("id")): slot for slot in slots if isinstance(slot, dict) and slot.get("id")} + + +def _path_bbox(points: list[list[float]]) -> dict[str, float]: + xs = [float(point[0]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + ys = [float(point[1]) for point in points if isinstance(point, (list, tuple)) and len(point) >= 2] + if not xs or not ys: + return {"x": 0.0, "y": 0.0, "w": 0.0, "h": 0.0} + x0 = min(xs) + y0 = min(ys) + return {"x": x0, "y": y0, "w": max(xs) - x0, "h": max(ys) - y0} + + +def _bbox_iou(a: dict, b: dict) -> float: + ax0, ay0 = float(a.get("x", 0.0)), float(a.get("y", 0.0)) + ax1, ay1 = ax0 + float(a.get("w", 0.0)), ay0 + float(a.get("h", 0.0)) + bx0, by0 = float(b.get("x", 0.0)), float(b.get("y", 0.0)) + bx1, by1 = bx0 + float(b.get("w", 0.0)), by0 + float(b.get("h", 0.0)) + ix = max(0.0, min(ax1, bx1) - max(ax0, bx0)) + iy = max(0.0, min(ay1, by1) - max(ay0, by0)) + inter = ix * iy + union = float(a.get("w", 0.0)) * float(a.get("h", 0.0)) + float(b.get("w", 0.0)) * float(b.get("h", 0.0)) - inter + return inter / max(union, 0.000001) + + +def _point_distance(a: list[float], b: list[float]) -> float: + return math.hypot(float(a[0]) - float(b[0]), float(a[1]) - float(b[1])) + + +def _path_distance(a: list[list[float]], b: list[list[float]]) -> float: + if len(a) < 2 or len(b) < 2: + return 1.0 + return (_point_distance(a[0], b[0]) + _point_distance(a[-1], b[-1])) / 2 + + +def _control_signature(control: dict) -> tuple[str, str]: + return str(control.get("source_id") or control.get("source") or ""), str(control.get("target_id") or control.get("target") or "") + + +def _controls_are_duplicate(a: dict, b: dict) -> bool: + ap = a.get("path_percent") or [] + bp = b.get("path_percent") or [] + same_binding = _control_signature(a) == _control_signature(b) + if same_binding and _path_distance(ap, bp) <= 0.08: + return True + return _bbox_iou(_path_bbox(ap), _path_bbox(bp)) >= 0.55 and _path_distance(ap, bp) <= 0.12 + + +def _controls_match_for_disagreement(heuristic: dict, vlm: dict) -> bool: + if _controls_are_duplicate(heuristic, vlm): + return True + hp = heuristic.get("path_percent") or [] + vp = vlm.get("path_percent") or [] + return _control_signature(heuristic) == _control_signature(vlm) or _path_distance(hp, vp) <= 0.08 or _bbox_iou(_path_bbox(hp), _path_bbox(vp)) >= 0.35 + + +def _bindable_objects(out_dir: Path, slots: list[dict]) -> list[dict]: + bindable = [dict(slot) for slot in slots if isinstance(slot, dict) and isinstance(slot.get("bbox_percent"), dict)] + seen = {str(item.get("id")) for item in bindable} + try: + geometry = read_json(out_dir / "reference_geometry.json") + except Exception: + geometry = {} + for card in geometry.get("cards", []) if isinstance(geometry, dict) else []: + if not isinstance(card, dict) or not isinstance(card.get("bbox_percent"), dict): + continue + card_id = str(card.get("id") or "") + if not card_id or card_id in seen: + continue + enriched = dict(card) + enriched.setdefault("display_label", card.get("title") or card_id) + enriched.setdefault("paper_concept", card.get("title") or card_id) + enriched.setdefault("semantic_role", "reference_card") + enriched.setdefault("asset_type", "editable_card") + bindable.append(enriched) + seen.add(card_id) + return bindable + + +def _edge_path(source: dict | None, target: dict | None) -> list[list[float]]: + if not source or not target: + return [] + sbox = source.get("bbox_percent") if isinstance(source.get("bbox_percent"), dict) else None + tbox = target.get("bbox_percent") if isinstance(target.get("bbox_percent"), dict) else None + if not sbox or not tbox: + return [] + sx, sy = _center(sbox) + tx, ty = _center(tbox) + if abs(tx - sx) >= abs(ty - sy): + start = (float(sbox["x"]) + (float(sbox["w"]) if tx >= sx else 0.0), sy) + end = (float(tbox["x"]) + (0.0 if tx >= sx else float(tbox["w"])), ty) + else: + start = (sx, float(sbox["y"]) + (float(sbox["h"]) if ty >= sy else 0.0)) + end = (tx, float(tbox["y"]) + (0.0 if ty >= sy else float(tbox["h"]))) + return [[round(start[0], 4), round(start[1], 4)], [round(end[0], 4), round(end[1], 4)]] + + def _sequence_controls(slots: list[dict], palette: list[str]) -> list[dict]: ordered = sorted(slots, key=lambda slot: (float(slot["bbox_percent"]["y"]), float(slot["bbox_percent"]["x"]))) arrows = [] @@ -71,7 +215,10 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str for line in lines[:80]: coordinates = hough_line_coordinates(line) if coordinates is None: - continue + values = line.reshape(-1) if hasattr(line, "reshape") else line + if len(values) < 4: + continue + coordinates = [int(v) for v in values[:4]] x1, y1, x2, y2 = coordinates length = math.hypot(x2 - x1, y2 - y1) if length < min(width, height) * 0.06: @@ -82,6 +229,8 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str target = _nearest_slot(p2, slots) if not source or not target or source["id"] == target["id"]: continue + if _distance_to_bbox(p1, source) > 0.075 or _distance_to_bbox(p2, target) > 0.075: + continue direction = math.degrees(math.atan2(p2[1] - p1[1], p2[0] - p1[0])) candidates.append({ "line": [p1, p2], @@ -126,6 +275,7 @@ def _cv_line_controls(reference_path: Path, slots: list[dict], palette: list[str def _normalize_controls(raw: list[dict], slots: list[dict], palette: list[str]) -> list[dict]: slot_ids = {str(slot.get("id")) for slot in slots} + objects = _slot_by_id(slots) normalized = [] for idx, item in enumerate(raw or [], start=1): if not isinstance(item, dict): @@ -133,40 +283,359 @@ def _normalize_controls(raw: list[dict], slots: list[dict], palette: list[str]) source = str(item.get("source_id") or item.get("source") or "") target = str(item.get("target_id") or item.get("target") or "") path = item.get("path_percent") - if source not in slot_ids or target not in slot_ids or not isinstance(path, list) or len(path) < 2: + if source not in slot_ids or target not in slot_ids: continue points = [] - for point in path: - if not isinstance(point, (list, tuple)) or len(point) < 2: - continue - points.append([round(max(0.0, min(1.0, float(point[0]))), 4), round(max(0.0, min(1.0, float(point[1]))), 4)]) + if isinstance(path, list): + for point in path: + if not isinstance(point, (list, tuple)) or len(point) < 2: + continue + points.append([round(max(0.0, min(1.0, float(point[0]))), 4), round(max(0.0, min(1.0, float(point[1]))), 4)]) + has_reference_path = len(points) >= 2 + if len(points) < 2: + points = _edge_path(objects.get(source), objects.get(target)) if len(points) < 2: continue dx = points[-1][0] - points[0][0] dy = points[-1][1] - points[0][1] dash_style = str(item.get("dash_style") or item.get("line_pattern") or "solid") - normalized.append({ + line_pattern = _coerce_choice(dash_style, VALID_LINE_PATTERNS, "solid") + render_style = _coerce_choice(item.get("render_style"), VALID_RENDER_STYLES, "") + route_intent = _coerce_choice(item.get("route_intent"), VALID_ROUTE_INTENTS, "") + visual_weight = _coerce_choice(item.get("visual_weight"), VALID_VISUAL_WEIGHTS, "") + preferred_axis = _coerce_choice(item.get("preferred_axis"), VALID_PREFERRED_AXES, "") + explicit_route_policy = str(item.get("route_policy") or "").strip() + route_policy = explicit_route_policy or ("preserve_reference_path" if has_reference_path else "bbox_route_from_hint") + route_style = str(item.get("route_style") or ("dashed_loop" if str(item.get("control_kind") or "").lower() == "dashed_loop" else "")) + if line_pattern in {"dash", "dashed"} and len(points) >= 3 and not route_style: + route_style = "dashed_spline_like" + record = { "id": str(item.get("id") or f"arrow_{idx:02d}"), "source": source, "target": target, "source_id": source, "target_id": target, + "target_ids": [str(target_id) for target_id in item.get("target_ids", []) if str(target_id) in slot_ids] if isinstance(item.get("target_ids"), list) else [], "control_kind": str(item.get("control_kind") or "straight_arrow"), "path_percent": points, "arrowhead_direction": float(item.get("arrowhead_direction") if item.get("arrowhead_direction") is not None else math.degrees(math.atan2(dy, dx))), "stroke_color": str(item.get("stroke_color") or (palette[2] if len(palette) > 2 else "#4A6FA5")), "stroke_width_pt": float(item.get("stroke_width_pt") or 1.5), "dash_style": dash_style, - "line_pattern": "dash" if dash_style.lower() in {"dash", "dashed"} else "solid", - "route_style": str(item.get("route_style") or ("dashed_loop" if str(item.get("control_kind") or "").lower() == "dashed_loop" else "")), + "line_pattern": "dash" if line_pattern in {"dash", "dashed"} else "solid", + "route_style": route_style, + "render_style": render_style, + "route_intent": route_intent, + "visual_weight": visual_weight, + "preferred_axis": preferred_axis, + "bend_side": str(item.get("bend_side") or ""), + "fill_color": str(item.get("fill_color") or ""), + "outline_color": str(item.get("outline_color") or ""), + "outline_width_pt": item.get("outline_width_pt"), + "block_arrow_thickness_percent": item.get("block_arrow_thickness_percent"), + "line_cap": str(item.get("line_cap") or ""), + "arrowhead_size": str(item.get("arrowhead_size") or ""), + "binding_source": str(item.get("binding_source") or ("heuristic" if str(item.get("detected_by") or "").startswith(("cv_", "sequence_")) else "vlm")), + "route_policy": route_policy, "confidence": float(item.get("confidence") or 0.8), "detected_by": str(item.get("detected_by") or "vlm"), "editable_in": "pptx", "render_policy": "ppt_shape_not_image_asset", - }) + } + normalized.append({key: value for key, value in record.items() if value not in ("", [], None)}) return normalized +def _disagreement_brief(item: dict | None) -> dict | None: + if not isinstance(item, dict): + return None + return { + key: item.get(key) + for key in ( + "id", + "source_id", + "target_id", + "control_kind", + "render_style", + "route_intent", + "visual_weight", + "path_percent", + "stroke_color", + "stroke_width_pt", + "dash_style", + "line_pattern", + "confidence", + "detected_by", + ) + if key in item + } + + +def _control_disagreements(heuristic: list[dict], vlm: list[dict]) -> dict: + matched_heuristic: set[int] = set() + matched_vlm: set[int] = set() + disagreements: list[dict] = [] + duplicates: list[dict] = [] + for v_idx, vlm_item in enumerate(vlm): + best_idx = None + best_score = -1.0 + for h_idx, heuristic_item in enumerate(heuristic): + if h_idx in matched_heuristic: + continue + if not _controls_match_for_disagreement(heuristic_item, vlm_item): + continue + score = _bbox_iou(_path_bbox(heuristic_item.get("path_percent") or []), _path_bbox(vlm_item.get("path_percent") or [])) + if _control_signature(heuristic_item) == _control_signature(vlm_item): + score += 1.0 + score -= _path_distance(heuristic_item.get("path_percent") or [], vlm_item.get("path_percent") or []) + if score > best_score: + best_idx = h_idx + best_score = score + if best_idx is None: + continue + heuristic_item = heuristic[best_idx] + matched_heuristic.add(best_idx) + matched_vlm.add(v_idx) + same_style = ( + str(heuristic_item.get("render_style") or "") == str(vlm_item.get("render_style") or "") + or not heuristic_item.get("render_style") + or not vlm_item.get("render_style") + ) + same_route = ( + str(heuristic_item.get("route_intent") or "") == str(vlm_item.get("route_intent") or "") + or not heuristic_item.get("route_intent") + or not vlm_item.get("route_intent") + ) + if _controls_are_duplicate(heuristic_item, vlm_item) and same_style and same_route: + duplicates.append({ + "id": f"duplicate_{len(duplicates)+1:02d}", + "kind": "duplicate", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + else: + disagreements.append({ + "id": f"control_mismatch_{len(disagreements)+1:02d}", + "kind": "mismatch", + "heuristic": _disagreement_brief(heuristic_item), + "vlm": _disagreement_brief(vlm_item), + "default_decision": "keep_vlm", + }) + for idx, item in enumerate(heuristic): + if idx not in matched_heuristic: + disagreements.append({ + "id": f"heuristic_only_{len(disagreements)+1:02d}", + "kind": "heuristic_only", + "heuristic": _disagreement_brief(item), + "vlm": None, + "default_decision": "keep_heuristic", + }) + for idx, item in enumerate(vlm): + if idx not in matched_vlm: + disagreements.append({ + "id": f"vlm_only_{len(disagreements)+1:02d}", + "kind": "vlm_only", + "heuristic": None, + "vlm": _disagreement_brief(item), + "default_decision": "keep_vlm", + }) + return { + "summary": "Control disagreements between heuristic/CV and VLM arrow detection.", + "heuristic_count": len(heuristic), + "vlm_count": len(vlm), + "duplicate_count": len(duplicates), + "disagreement_count": len(disagreements), + "duplicates": duplicates, + "disagreements": disagreements, + } + + +def _call_adjudicator(adapter: Callable, reference: Path, bindable: list[dict], disagreements: dict, flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, disagreements, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, disagreements) + + +def _decision_by_id(raw: dict | list | None) -> dict[str, dict]: + decisions = raw.get("decisions", []) if isinstance(raw, dict) else raw + result = {} + for item in decisions or []: + if not isinstance(item, dict): + continue + disagreement_id = str(item.get("disagreement_id") or item.get("id") or "") + if disagreement_id: + result[disagreement_id] = item + return result + + +def _append_unique(arrows: list[dict], candidate: dict) -> bool: + if any(_controls_are_duplicate(existing, candidate) for existing in arrows): + return False + candidate_id = str(candidate.get("id") or f"arrow_{len(arrows)+1:02d}") + used_ids = {str(item.get("id") or "") for item in arrows} + if candidate_id in used_ids: + stem = candidate_id + suffix = 2 + while f"{stem}_{suffix}" in used_ids: + suffix += 1 + candidate = dict(candidate) + candidate["id"] = f"{stem}_{suffix}" + arrows.append(candidate) + return True + + +def _merge_with_adjudication( + heuristic: list[dict], + vlm: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + adjudicator: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict, dict]: + disagreements = _control_disagreements(heuristic, vlm) + write_json(out / "reference_control_disagreements.json", disagreements) + if mode == "off": + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(vlm), + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return vlm, disagreements, adjudication_doc + raw_adjudication: dict | list | None = None + decisions: dict[str, dict] = {} + status = "skipped" + if mode == "vlm" and adjudicator and disagreements["disagreement_count"]: + try: + raw_adjudication = _call_adjudicator(adjudicator, reference, bindable, disagreements, flow_graph) + decisions = _decision_by_id(raw_adjudication) + status = "used" + except Exception as exc: + status = "fallback" + warnings.append(f"control_adjudication_failed:{exc}") + elif mode == "vlm" and disagreements["disagreement_count"]: + status = "unavailable" + merged: list[dict] = [] + for duplicate in disagreements["duplicates"]: + vlm_item = duplicate.get("vlm") + normalized = _normalize_controls([vlm_item], bindable, palette) + if normalized: + _append_unique(merged, normalized[0]) + for item in disagreements["disagreements"]: + decision = decisions.get(str(item.get("id") or ""), {}) + action = str(decision.get("decision") or item.get("default_decision") or "").lower() + selected: list[dict] = [] + if action == "keep_heuristic": + if item.get("heuristic"): + selected = [item["heuristic"]] + elif action == "keep_both": + selected = [candidate for candidate in [item.get("vlm"), item.get("heuristic")] if candidate] + elif action == "drop_both": + selected = [] + elif action == "merge_or_adjust": + selected = [decision.get("merged_control") or decision.get("control") or item.get("vlm") or item.get("heuristic")] + else: + selected = [item.get("vlm") or item.get("heuristic")] + normalized = _normalize_controls(selected, bindable, palette) + if len(normalized) < len([candidate for candidate in selected if candidate]): + warnings.append(f"invalid_control_adjudication_item_dropped:{item.get('id')}") + for candidate in normalized: + _append_unique(merged, candidate) + if not merged: + merged = vlm or heuristic + adjudication_doc = { + "summary": "Control disagreement adjudication report.", + "mode": mode, + "status": status, + "decision_count": len(decisions), + "input_disagreement_count": disagreements["disagreement_count"], + "input_duplicate_count": disagreements["duplicate_count"], + "merged_arrow_count": len(merged), + "raw_adjudication": raw_adjudication, + } + write_json(out / "reference_control_adjudication.json", adjudication_doc) + return merged, disagreements, adjudication_doc + + +def _call_coverage_audit(adapter: Callable, reference: Path, bindable: list[dict], arrows: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, arrows, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, arrows) + + +def _apply_coverage_audit( + arrows: list[dict], + bindable: list[dict], + palette: list[str], + reference: Path, + out: Path, + flow_graph: dict | None, + mode: str, + coverage_audit_adapter: Callable | None, + warnings: list[str], +) -> tuple[list[dict], dict]: + audit_doc: dict = { + "summary": "Control coverage audit report.", + "mode": mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } + if mode != "vlm": + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + if not coverage_audit_adapter: + audit_doc["status"] = "unavailable" + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + try: + raw = _call_coverage_audit(coverage_audit_adapter, reference, bindable, arrows, flow_graph) + raw_items = raw.get("missing_controls") if isinstance(raw, dict) else raw + if raw_items is None and isinstance(raw, dict): + raw_items = raw.get("arrows", []) + normalized = _normalize_controls(raw_items or [], bindable, palette) + appended = 0 + for candidate in normalized: + candidate = dict(candidate) + candidate["detected_by"] = "coverage_audit_vlm" + candidate["binding_source"] = "coverage_audit_vlm" + if _append_unique(arrows, candidate): + appended += 1 + audit_doc.update({ + "status": "used", + "raw_audit": raw, + "missing_control_count": len(raw_items or []), + "valid_missing_control_count": len(normalized), + "appended_control_count": appended, + }) + invalid_count = max(0, len(raw_items or []) - len(normalized)) + if invalid_count: + warnings.append(f"{invalid_count}_invalid_coverage_audit_control(s)_dropped") + except Exception as exc: + audit_doc["status"] = "fallback" + audit_doc["fallback_reason"] = f"control_coverage_audit_failed:{exc}" + warnings.append(str(audit_doc["fallback_reason"])) + write_json(out / "reference_control_coverage_audit.json", audit_doc) + return arrows, audit_doc + + def _draw_overlay(reference_path: Path, out_path: Path, arrows: list[dict]) -> None: with Image.open(reference_path).convert("RGB") as image: draw = ImageDraw.Draw(image) @@ -190,6 +659,39 @@ def _draw_overlay(reference_path: Path, out_path: Path, arrows: list[dict]) -> N image.save(out_path) +def _flow_edges(flow_graph: dict | None) -> list[dict]: + if not isinstance(flow_graph, dict): + return [] + return [item for item in flow_graph.get("edges", []) or [] if isinstance(item, dict)] + + +def _flow_graph_stats(flow_graph: dict | None, objects: list[dict]) -> dict: + object_ids = {str(item.get("id") or "") for item in objects if isinstance(item, dict)} + edges = _flow_edges(flow_graph) + invalid = [] + for edge in edges: + source = str(edge.get("source_id") or edge.get("source") or "") + target = str(edge.get("target_id") or edge.get("target") or "") + if source not in object_ids or target not in object_ids: + invalid.append(str(edge.get("id") or f"{source}->{target}")) + return { + "flow_graph_used": bool(edges), + "flow_graph_edge_count": len(edges), + "invalid_flow_edge_count": len(invalid), + "invalid_flow_edge_ids": invalid, + } + + +def _call_control_adapter(adapter: Callable, reference: Path, bindable: list[dict], heuristic: list[dict], flow_graph: dict | None): + try: + signature = inspect.signature(adapter) + if len(signature.parameters) >= 4: + return adapter(reference, bindable, heuristic, flow_graph) + except (TypeError, ValueError): + pass + return adapter(reference, bindable, heuristic) + + def localize_reference_controls( reference_path: str | Path, slots: list[dict], @@ -197,46 +699,134 @@ def localize_reference_controls( out_dir: str | Path, mode: str = "hybrid", control_adapter: Callable[[str | Path, list[dict], list[dict]], dict | list[dict]] | None = None, + flow_graph: dict | None = None, + control_adjudication_mode: str = "off", + control_adjudicator: Callable | None = None, + control_coverage_audit_mode: str = "off", + coverage_audit_adapter: Callable | None = None, ) -> dict: reference = Path(reference_path) out = Path(out_dir) warnings = [] - heuristic = _cv_line_controls(reference, slots, palette) or _sequence_controls(slots, palette) + bindable = _bindable_objects(out, slots) + flow_stats = _flow_graph_stats(flow_graph, bindable) + if flow_stats["invalid_flow_edge_count"]: + warnings.append(f"{flow_stats['invalid_flow_edge_count']}_invalid_flow_graph_edge(s)") + cv_heuristic = _cv_line_controls(reference, bindable, palette) + heuristic = cv_heuristic or _sequence_controls(slots, palette) _draw_overlay(reference, out / "reference_controls_candidates_overlay.png", heuristic) arrows = heuristic vlm_status = "not_requested" + normalized: list[dict] = [] + raw: dict | list | None = None + disagreements = { + "summary": "Control disagreement detection skipped.", + "heuristic_count": len(heuristic), + "vlm_count": 0, + "duplicate_count": 0, + "disagreement_count": 0, + "duplicates": [], + "disagreements": [], + } + adjudication_doc = { + "summary": "Control disagreement adjudication skipped.", + "mode": control_adjudication_mode, + "status": "skipped", + "decision_count": 0, + "input_disagreement_count": 0, + "input_duplicate_count": 0, + "merged_arrow_count": len(arrows), + } + audit_doc = { + "summary": "Control coverage audit skipped.", + "mode": control_coverage_audit_mode, + "status": "skipped", + "missing_control_count": 0, + "appended_control_count": 0, + } if mode in {"vlm", "hybrid"}: if control_adapter: try: - raw = control_adapter(reference, slots, heuristic) + raw = _call_control_adapter(control_adapter, reference, bindable, heuristic, flow_graph) raw_arrows = raw.get("arrows", []) if isinstance(raw, dict) else raw - normalized = _normalize_controls(raw_arrows, slots, palette) + normalized = _normalize_controls(raw_arrows, bindable, palette) if normalized: - arrows = normalized vlm_status = "used" + if mode == "hybrid": + arrows, disagreements, adjudication_doc = _merge_with_adjudication( + heuristic, + normalized, + bindable, + palette, + reference, + out, + flow_graph, + control_adjudication_mode, + control_adjudicator, + warnings, + ) + else: + arrows = normalized + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) invalid_count = max(0, len(raw_arrows or []) - len(normalized)) if invalid_count: warnings.append(f"{invalid_count}_invalid_vlm_control_arrow(s)_dropped") else: vlm_status = "fallback" warnings.append("control_adapter_returned_no_valid_arrows") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) except Exception as exc: vlm_status = "fallback" warnings.append(f"vlm_control_failed:{exc}") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) else: vlm_status = "unavailable_fallback_to_heuristic" if mode == "vlm": warnings.append("control_mode_vlm_requested_without_adapter") + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) + else: + write_json(out / "reference_control_disagreements.json", disagreements) + write_json(out / "reference_control_adjudication.json", adjudication_doc) if mode == "manual": arrows = [] vlm_status = "manual_empty" _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) + if mode in {"vlm", "hybrid"}: + arrows, audit_doc = _apply_coverage_audit( + arrows, + bindable, + palette, + reference, + out, + flow_graph, + control_coverage_audit_mode, + coverage_audit_adapter, + warnings, + ) + _draw_overlay(reference, out / "reference_controls_overlay.png", arrows) return { "summary": "Reference control localization report.", "mode": mode, "status": "ok", "vlm_status": vlm_status, - "vlm_model": str(raw.get("_vlm_model")) if mode in {"vlm", "hybrid"} and isinstance(locals().get("raw"), dict) and raw.get("_vlm_model") else None, + "vlm_model": str(raw.get("_vlm_model")) if mode in {"vlm", "hybrid"} and isinstance(raw, dict) and raw.get("_vlm_model") else None, + "bindable_object_count": len(bindable), + "heuristic_arrow_count": len(heuristic), + "cv_heuristic_arrow_count": len(cv_heuristic), + "vlm_arrow_count": len(normalized), + "merged_arrow_count": len(arrows), + "control_adjudication_mode": control_adjudication_mode, + "control_adjudication_status": adjudication_doc.get("status"), + "control_disagreement_count": disagreements.get("disagreement_count", 0), + "control_duplicate_count": disagreements.get("duplicate_count", 0), + "control_coverage_audit_mode": control_coverage_audit_mode, + "control_coverage_audit_status": audit_doc.get("status"), + "control_coverage_appended_count": audit_doc.get("appended_control_count", 0), "warnings": warnings, "arrows": arrows, + **flow_stats, } diff --git a/rfs/editable_rebuild.py b/rfs/editable_rebuild.py index e2a388b..4106ea3 100644 --- a/rfs/editable_rebuild.py +++ b/rfs/editable_rebuild.py @@ -1,1148 +1,14 @@ -from __future__ import annotations - -import base64 -import json -import mimetypes -import os -import platform -import shutil -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path -from typing import Callable - -from PIL import Image, ImageChops, ImageDraw - -from .composition import compile_ppt, render_rebuild_preview -from .contracts import apply_paper_semantic_contract -from .control_localizer import localize_reference_controls -from .evaluation import run_rebuild_visual_quality_check -from .layout_planner import dominant_palette, estimate_background, plan_reference_layout -from .layout_semantic_planner import plan_slot_semantics -from .rebuild_design_planner import plan_rebuild_design -from .rebuild_vlm_validation import build_rebuild_vlm_validation_report -from .text_layer import build_text_layer -from .utils import ensure_dir, read_json, write_json, write_text - - -ASSET_THRESHOLDS = { - "character": (0.80, 0.95), - "document_stack": (0.75, 0.95), - "chart_card": (0.75, 0.95), - "inspection": (0.70, 0.95), - "tool_icon": (0.80, 0.95), - "tool_combo": (0.70, 0.95), - "device": (0.70, 0.95), - "screenshot_card": (0.75, 0.95), - "legend_marker": (0.80, 0.95), - "thin_tool": (0.50, 0.95), - "generic": (0.70, 0.95), -} - - -TEXT_ASSET_OVERLAP_THRESHOLD = 0.60 -SIMPLE_PRIMITIVE_TYPES = {"legend_marker", "simple_marker", "status_marker", "node_marker"} - - -def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: - x = max(0.0, min(0.995, x)) - y = max(0.0, min(0.995, y)) - w = max(0.001, min(w, 1.0 - x)) - h = max(0.001, min(h, 1.0 - y)) - return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} - - -def _canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: - ratio = width_px / max(height_px, 1) - width = max(10.0, min(15.6, 7.5 * ratio)) - return round(width, 3), round(width / ratio, 3) - - -def _rgb_hex(rgb: tuple[int, int, int]) -> str: - return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" - - -def _estimate_background(image: Image.Image) -> str: - rgb = image.convert("RGB") - w, h = rgb.size - samples = [] - for box in [(0, 0, w, max(1, h // 12)), (0, h - max(1, h // 12), w, h), (0, 0, max(1, w // 12), h), (w - max(1, w // 12), 0, w, h)]: - crop = rgb.crop(box).resize((1, 1)) - samples.append(crop.getpixel((0, 0))) - avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) - return _rgb_hex(avg) - - -def _dominant_palette(image: Image.Image, count: int = 6) -> list[str]: - small = image.convert("RGB").resize((96, 96)) - colors = small.quantize(colors=count).convert("RGB").getcolors(96 * 96) or [] - colors = sorted(colors, key=lambda item: item[0], reverse=True) - return [_rgb_hex(rgb) for _n, rgb in colors[:count]] - - -def _supported_aspect_ratio(width: float, height: float) -> str: - ratio = width / max(height, 0.001) - if ratio >= 1.55: - return "16:9" - if ratio >= 1.18: - return "4:3" - if ratio >= 0.85: - return "1:1" - if ratio >= 0.62: - return "3:4" - return "9:16" - - -def _slot_type_for_box(slot: dict) -> str: - semantic_type = str(slot.get("asset_type") or "").strip() - if semantic_type: - return semantic_type - ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) - if ratio >= 2.2: - return "thin_tool" - if ratio >= 1.35: - return "chart_card" - if ratio <= 0.78: - return "character" - return "generic" - - -def _threshold_for_type(slot_type: str) -> tuple[float, float]: - return ASSET_THRESHOLDS.get(slot_type, ASSET_THRESHOLDS["generic"]) - - -def _foreground_metrics(image_path: str | Path, background_hex: str | None = None) -> dict: - try: - image = Image.open(image_path).convert("RGB") - except Exception as exc: - return {"status": "error", "error": str(exc), "foreground_bbox_px": None, "foreground_bbox_fill_percent": 0.0} - w, h = image.size - bg = background_hex or _estimate_background(image) - bg_rgb = tuple(int(bg.strip("#")[i:i + 2], 16) for i in (0, 2, 4)) - bg_img = Image.new("RGB", image.size, bg_rgb) - diff = ImageChops.difference(image, bg_img).convert("L") - mask = diff.point(lambda p: 255 if p > 18 else 0) - box = mask.getbbox() - if not box: - return { - "status": "empty", - "foreground_bbox_px": None, - "foreground_bbox_fill_percent": 0.0, - "margin_left_percent": 0.5, - "margin_right_percent": 0.5, - "margin_top_percent": 0.5, - "margin_bottom_percent": 0.5, - } - left, top, right, bottom = box - fill = ((right - left) * (bottom - top)) / max(w * h, 1) - return { - "status": "ok", - "foreground_bbox_px": [left, top, right, bottom], - "foreground_bbox_fill_percent": round(fill, 4), - "margin_left_percent": round(left / w, 4), - "margin_right_percent": round((w - right) / w, 4), - "margin_top_percent": round(top / h, 4), - "margin_bottom_percent": round((h - bottom) / h, 4), - } - - -def economy_acceptance_decision(slot_type: str, fill_percent: float, strict: bool = False) -> dict: - low, high = (0.80, 0.95) if strict else _threshold_for_type(slot_type) - accepted = low <= float(fill_percent) <= high - if accepted: - reason = f"fill {fill_percent:.3f} within {slot_type} threshold {low:.2f}-{high:.2f}" - elif float(fill_percent) < low: - reason = f"fill {fill_percent:.3f} below {slot_type} threshold {low:.2f}" - else: - reason = f"fill {fill_percent:.3f} above tightness threshold {high:.2f}" - return {"accepted": accepted, "selected_reason": reason, "threshold_min": low, "threshold_max": high} - - -def _find_visual_components(reference_path: Path, canvas_w: int, canvas_h: int) -> list[dict]: - try: - import cv2 - import numpy as np - except Exception: - return [] - image = cv2.imread(str(reference_path)) - if image is None: - return [] - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - blur = cv2.GaussianBlur(gray, (5, 5), 0) - edges = cv2.Canny(blur, 45, 120) - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) - dilated = cv2.dilate(edges, kernel, iterations=2) - contours, _hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - boxes = [] - image_area = canvas_w * canvas_h - for contour in contours: - x, y, w, h = cv2.boundingRect(contour) - area = w * h - if area < image_area * 0.008 or area > image_area * 0.35: - continue - if w < 24 or h < 24: - continue - boxes.append((x, y, w, h)) - boxes.sort(key=lambda item: (item[1], item[0])) - kept: list[tuple[int, int, int, int]] = [] - for box in boxes: - x, y, w, h = box - duplicate = False - for kx, ky, kw, kh in kept: - ix = max(0, min(x + w, kx + kw) - max(x, kx)) - iy = max(0, min(y + h, ky + kh) - max(y, ky)) - overlap = ix * iy / max(w * h, 1) - if overlap > 0.55: - duplicate = True - break - if not duplicate: - kept.append(box) - if len(kept) >= 12: - break - slots = [] - for idx, (x, y, w, h) in enumerate(kept, start=1): - slots.append({ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": _bbox(x / canvas_w, y / canvas_h, w / canvas_w, h / canvas_h), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - }) - return slots - - -def _fallback_slots(count: int = 6) -> list[dict]: - boxes = [ - (0.08, 0.26, 0.16, 0.22), - (0.31, 0.26, 0.16, 0.22), - (0.54, 0.26, 0.16, 0.22), - (0.77, 0.26, 0.16, 0.22), - (0.25, 0.58, 0.18, 0.22), - (0.57, 0.58, 0.18, 0.22), - ][:count] - slots = [] - for idx, (x, y, w, h) in enumerate(boxes, start=1): - slots.append({ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": _bbox(x, y, w, h), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - }) - return slots - - -def _style_from_layout(layout: dict) -> dict: - palette = layout.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52", "#6B57C8", "#D44E5D"] - background = str(layout.get("canvas", {}).get("background") or "#FFFFFF") - style = { - "palette": palette, - "reference_palette": palette, - "color_tokens": [{"token_id": f"palette_{i:02d}", "hex": color} for i, color in enumerate(palette)], - "panel_styles": { - "reference_canvas": { - "fill_color": background, - "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", - "header_color": palette[1] if len(palette) > 1 else "#6C7A89", - } - }, - "arrow_weight_pt": 1.7, - } - for panel in layout.get("panels", []): - if panel.get("id") and panel["id"] != "reference_canvas": - style["panel_styles"].setdefault(str(panel["id"]), { - "fill_color": background, - "stroke_color": palette[1] if len(palette) > 1 else "#B8C0CC", - "header_color": palette[1] if len(palette) > 1 else "#6C7A89", - }) - return style - - -def _build_program( - reference_path: Path, - out: Path, - layout_mode: str = "hybrid", - vlm_layout_adapter: Callable | None = None, -) -> tuple[dict, dict]: - layout = plan_reference_layout(reference_path, out, mode=layout_mode, vlm_adapter=vlm_layout_adapter) - style = _style_from_layout(layout) - program = { - "canvas": layout["canvas"], - "style": style, - "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, - "panels": layout.get("panels", []), - "cards": layout.get("cards", []), - "slots": layout.get("slots", []), - "assets": [], - "arrows": [], - "labels": [], - "groups": [], - "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}], - } - return program, layout - - -def _crop_reference(reference_path: Path, out_path: Path, bbox_percent: dict) -> Path: - with Image.open(reference_path).convert("RGB") as image: - w, h = image.size - left = int(float(bbox_percent["x"]) * w) - top = int(float(bbox_percent["y"]) * h) - right = int((float(bbox_percent["x"]) + float(bbox_percent["w"])) * w) - bottom = int((float(bbox_percent["y"]) + float(bbox_percent["h"])) * h) - crop = image.crop((max(0, left), max(0, top), min(w, right), min(h, bottom))) - out_path.parent.mkdir(parents=True, exist_ok=True) - crop.save(out_path) - return out_path - - -def _placeholder_asset(slot: dict, out_path: Path, background: str) -> None: - out_path.parent.mkdir(parents=True, exist_ok=True) - ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) - if ratio >= 1.55: - size = (512, 288) - elif ratio <= 0.62: - size = (288, 512) - else: - size = (384, 384) - image = Image.new("RGB", size, background) - draw = ImageDraw.Draw(image) - pad = int(min(size) * 0.12) - draw.rounded_rectangle((pad, pad, size[0] - pad, size[1] - pad), radius=18, fill="#FFFFFF", outline="#6C7A89", width=3) - draw.line((pad * 1.5, size[1] * 0.45, size[0] - pad * 1.5, size[1] * 0.45), fill="#4A6FA5", width=5) - draw.ellipse((size[0] * 0.42, size[1] * 0.22, size[0] * 0.58, size[1] * 0.38), fill="#E17721") - draw.text((pad * 1.3, size[1] * 0.58), slot["id"], fill="#263747") - image.save(out_path) - - -def _api_generate_asset(spec: dict, crop_path: Path, out_path: Path) -> dict: - import requests - - api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") - url = os.getenv("GEMINI_GEN_IMG_URL") - if not api_key or not url: - raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode api") - mime = mimetypes.guess_type(crop_path.name)[0] or "image/png" - image_b64 = base64.b64encode(crop_path.read_bytes()).decode("ascii") - prompt = spec["prompt"] - payload = { - "contents": [{ - "role": "user", - "parts": [ - {"text": prompt}, - {"inlineData": {"mimeType": mime, "data": image_b64}}, - ], - }], - "generationConfig": { - "responseModalities": ["IMAGE"], - "imageConfig": {"imageSize": "1K", "aspectRatio": spec["generation_aspect_ratio"]}, - }, - } - response = requests.post(url, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), timeout=240) - response.raise_for_status() - data = response.json() - for candidate in data.get("candidates", []): - for part in candidate.get("content", {}).get("parts", []): - inline = part.get("inlineData") or part.get("inline_data") - if inline and inline.get("data"): - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_bytes(base64.b64decode(inline["data"])) - return {"status": "generated", "api_requests_attempted": 1} - raise RuntimeError("API returned no image data") +"""Compatibility alias for :mod:`rfs.workflows.rebuild_editable`. +The reference-image rebuild implementation now belongs to the workflows layer. +This alias preserves the historical import path and private monkeypatch targets. +""" -def _fill_target_for_type(slot_type: str) -> tuple[float, float]: - return _threshold_for_type(slot_type) - - -def _asset_prompt(slot: dict, slot_type: str, background: str, aspect_ratio: str) -> str: - low, high = _fill_target_for_type(slot_type) - subject = str(slot.get("prompt_subject") or slot.get("paper_concept") or slot.get("id")) - nearby = slot.get("nearby_text") or [] - nearby_text = " | ".join(nearby) if isinstance(nearby, list) else str(nearby or "") - orientation = "horizontal" if aspect_ratio in {"16:9", "4:3"} else "vertical" if aspect_ratio in {"3:4", "9:16"} else "square" - return ( - "Use the provided reference crop only as guidance for object identity, pose, colors, and diagram style. " - "Generate one clean standalone visual asset for an editable PowerPoint reconstruction. " - f"Slot type: {slot_type}. Canvas aspect ratio: {aspect_ratio}. Composition orientation: {orientation}. " - f"Main subject: {subject}. " - f"Nearby reference text for semantics only: {nearby_text}. " - f"The main subject should occupy {int(low * 100)}%-{int(high * 100)}% of the image canvas, with only narrow uniform margins. " - "For horizontal slots, make the subject span the width; for vertical slots, make the subject span the height. " - "Avoid a tiny object floating in a large colored field. " - f"Use a flat background matching {background}. " - "Do not include readable text, captions, watermarks, arrows, panel borders, or the whole source diagram. " - "Do not crop off important details." - ) - - -def _load_accepted_assets(path: Path) -> dict: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - if isinstance(data, dict): - return data - return {} - - -def _bbox_overlap_ratio(a: dict, b: dict) -> float: - ax1, ay1 = float(a["x"]), float(a["y"]) - ax2, ay2 = ax1 + float(a["w"]), ay1 + float(a["h"]) - bx1, by1 = float(b["x"]), float(b["y"]) - bx2, by2 = bx1 + float(b["w"]), by1 + float(b["h"]) - ix = max(0.0, min(ax2, bx2) - max(ax1, bx1)) - iy = max(0.0, min(ay2, by2) - max(ay1, by1)) - return (ix * iy) / max(float(a["w"]) * float(a["h"]), 0.000001) - - -def _text_program_boxes(program: dict) -> list[dict]: - text_program = program.get("text_program") if isinstance(program.get("text_program"), dict) else {} - boxes = [] - for item in text_program.get("items", []) or []: - if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): - continue - if not str(item.get("text") or "").strip(): - continue - boxes.append(item) - return boxes - - -def _normalized_reuse_key(spec: dict) -> str: - import re - - subject = str(spec.get("prompt_subject") or spec.get("semantic_role") or spec.get("slot_id") or "").lower() - subject = re.sub(r"\b(executor|branch|step|slot|icon)\s*[a-z0-9]*\b", " ", subject) - subject = re.sub(r"\b[a-d]\b|\b[0-9]+\b", " ", subject) - subject = re.sub(r"[^a-z0-9]+", " ", subject).strip() - if not subject: - subject = str(spec.get("asset_type") or spec.get("slot_type") or "generic") - return f"{spec.get('asset_type') or spec.get('slot_type') or 'generic'}::{subject}" - - -def _apply_smart_asset_policy(specs: list[dict], program: dict, out: Path, asset_policy: str) -> list[dict]: - text_items = _text_program_boxes(program) - filtered: list[dict] = [] - text_filter_report = [] - decision_report = [] - api_plan = { - "summary": "Smart API asset plan. Reference crops are only prompt/context inputs; they are not final PPT assets.", - "asset_policy": asset_policy, - "unique_api_assets": 0, - "reused_slots": 0, - "skipped_text_regions": 0, - "ppt_primitive_slots": 0, - "estimated_api_requests": 0, - "reuse_groups": [], - "recommended_api_slots": [], - } - - for spec in specs: - slot_id = str(spec["slot_id"]) - bbox = spec["slot_bbox_percent"] - overlaps = [ - { - "text_id": item.get("id"), - "text": item.get("text"), - "overlap_ratio": round(_bbox_overlap_ratio(bbox, item["bbox_percent"]), 4), - } - for item in text_items - ] - best = max(overlaps, key=lambda item: item["overlap_ratio"], default=None) - if best and float(best["overlap_ratio"]) >= TEXT_ASSET_OVERLAP_THRESHOLD: - spec["asset_decision"] = "text_region" - spec["asset_generation_mode"] = "skip_asset" - spec["asset_decision_reason"] = f"OCR/DSL text overlap {best['overlap_ratio']:.2f}; region is primarily editable text." - text_filter_report.append({ - "candidate_id": slot_id, - "rejected_as_asset": True, - "converted_to": "existing_editable_text", - "reason": spec["asset_decision_reason"], - "matched_text_id": best.get("text_id"), - "matched_text": best.get("text"), - "ocr_or_text_overlap": best.get("overlap_ratio"), - }) - decision_report.append({ - "slot_id": slot_id, - "decision": "text_region", - "needs_api": False, - "reason": spec["asset_decision_reason"], - }) - api_plan["skipped_text_regions"] += 1 - continue - if str(spec.get("asset_type") or spec.get("slot_type") or "").lower() in SIMPLE_PRIMITIVE_TYPES: - spec["asset_decision"] = "ppt_primitive" - spec["asset_generation_mode"] = "skip_asset" - spec["asset_decision_reason"] = "Simple marker should be represented by editable PPT primitives, not an image asset." - decision_report.append({ - "slot_id": slot_id, - "decision": "ppt_primitive", - "needs_api": False, - "reason": spec["asset_decision_reason"], - }) - api_plan["ppt_primitive_slots"] += 1 - continue - spec["asset_decision"] = "api_generated" - spec["asset_generation_mode"] = "api" - spec["asset_decision_reason"] = "Complex non-text visual slot; final asset should be API-generated from reference crop context." - spec["reuse_group_key"] = _normalized_reuse_key(spec) - filtered.append(spec) - - primary_by_key: dict[str, dict] = {} - reuse_groups: dict[str, list[str]] = {} - for spec in filtered: - key = str(spec["reuse_group_key"]) - reuse_groups.setdefault(key, []).append(str(spec["slot_id"])) - if key not in primary_by_key: - primary_by_key[key] = spec - decision = "api_generated" - api_plan["unique_api_assets"] += 1 - api_plan["recommended_api_slots"].append(str(spec["slot_id"])) - else: - primary = primary_by_key[key] - spec["asset_decision"] = "reuse_existing" - spec["asset_generation_mode"] = "reuse_existing" - spec["reuse_source_slot_id"] = primary["slot_id"] - spec["reuse_source_asset_id"] = primary["asset_id"] - decision = "reuse_existing" - api_plan["reused_slots"] += 1 - decision_report.append({ - "slot_id": spec["slot_id"], - "decision": decision, - "reused_asset_id": spec.get("reuse_source_asset_id"), - "reuse_group_key": key, - "needs_api": decision == "api_generated", - "reason": spec.get("asset_decision_reason"), - }) - api_plan["estimated_api_requests"] = api_plan["unique_api_assets"] - api_plan["reuse_groups"] = [ - {"reuse_group_key": key, "slots": slots} - for key, slots in sorted(reuse_groups.items()) - if len(slots) > 1 - ] - kept_slot_ids = {str(spec["slot_id"]) for spec in filtered} - program["slots"] = [slot for slot in program.get("slots", []) if str(slot.get("id")) in kept_slot_ids] - write_json(out / "asset_decision_report.json", { - "summary": "Slot-level asset decisions. In smart-api mode, final crop assets are disabled; crops are only API reference inputs.", - "asset_policy": asset_policy, - "decisions": decision_report, - }) - write_json(out / "text_asset_filter_report.json", { - "summary": "Text-like candidate slots rejected before asset generation.", - "overlap_threshold": TEXT_ASSET_OVERLAP_THRESHOLD, - "items": text_filter_report, - }) - write_json(out / "api_asset_plan.json", api_plan) - return filtered - - -def _make_asset_specs(program: dict, reference_path: Path, out: Path, asset_policy: str = "legacy") -> list[dict]: - specs = [] - background = str(program["canvas"].get("background") or "#FFFFFF") - for slot in program["slots"]: - bbox = slot["bbox_percent"] - slot_type = _slot_type_for_box(slot) - fill_min, fill_max = _fill_target_for_type(slot_type) - slot_ratio = float(bbox["w"]) / max(float(bbox["h"]), 0.001) - generation_ratio = str(slot.get("generation_aspect_ratio") or _supported_aspect_ratio(float(bbox["w"]), float(bbox["h"]))) - slot_background = str(slot.get("background_color_hex") or background) - crop_path = out / "reference_slot_crops" / f"{slot['id']}.png" - _crop_reference(reference_path, crop_path, bbox) - spec = { - "slot_id": slot["id"], - "asset_id": slot["asset_id"], - "slot_type": slot_type, - "asset_type": slot_type, - "semantic_role": slot.get("semantic_role"), - "nearby_text": slot.get("nearby_text"), - "panel_context": slot.get("panel_context"), - "upstream_ids": slot.get("upstream_ids", []), - "downstream_ids": slot.get("downstream_ids", []), - "prompt_subject": slot.get("prompt_subject") or slot.get("paper_concept") or slot.get("id"), - "slot_bbox_percent": bbox, - "slot_aspect_ratio": round(slot_ratio, 4), - "generation_aspect_ratio": generation_ratio, - "background_color_hex": slot_background, - "reference_crop_path": str(crop_path), - "prompt": _asset_prompt(slot, slot_type, slot_background, generation_ratio), - "main_subject_fill_target": round((fill_min + fill_max) / 2, 3), - "internal_content_fill_target": round((fill_min + fill_max) / 2, 3), - "max_margin_percent": 0.10, - "forbidden_elements": ["readable text", "captions", "watermarks", "arrows", "panel borders", "full source diagram"], - "style_reference": "provided reference crop", - "no_text": True, - "no_arrows": True, - "no_full_diagram": True, - "layout_bbox_locked": True, - "no_crop_postprocess": True, - } - specs.append(spec) - if asset_policy == "smart-api": - return _apply_smart_asset_policy(specs, program, out, asset_policy) - write_json(out / "asset_decision_report.json", { - "summary": "Legacy asset decision report.", - "asset_policy": asset_policy, - "decisions": [{"slot_id": spec["slot_id"], "decision": asset_policy, "needs_api": asset_policy == "api"} for spec in specs], - }) - write_json(out / "text_asset_filter_report.json", { - "summary": "Text asset filter disabled in legacy asset policy.", - "items": [], - }) - write_json(out / "api_asset_plan.json", { - "summary": "API asset plan disabled in legacy asset policy.", - "asset_policy": asset_policy, - "estimated_api_requests": len(specs) if asset_policy == "api" else 0, - }) - return specs - - -def _generate_assets( - specs: list[dict], - program: dict, - out: Path, - asset_mode: str, - asset_workers: int, - asset_retries: int, - economy_mode: bool, - regenerate_slots: set[str], - strict_asset_regeneration: bool, - asset_policy: str = "legacy", -) -> tuple[list[dict], dict]: - asset_dir = ensure_dir(out / "assets") - accepted = _load_accepted_assets(out / "accepted_assets.json") - reports: list[dict] = [] - api_requests = 0 - - def run_one(spec: dict) -> dict: - slot_id = spec["slot_id"] - out_path = asset_dir / f"{spec['asset_id']}.png" - if spec.get("asset_decision") == "reuse_existing": - source_asset = asset_dir / f"{spec['reuse_source_asset_id']}.png" - if source_asset.exists(): - shutil.copyfile(source_asset, out_path) - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) - return {**spec, **metrics, **decision, "status": "reused_from_group", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_group_asset", "fallback_used": False} - return {**spec, "status": "reuse_source_missing", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": f"reuse source missing: {source_asset}"} - locked = bool(accepted.get(slot_id, {}).get("accepted")) or bool(accepted.get(spec["asset_id"], {}).get("accepted")) - if out_path.exists() and economy_mode and slot_id not in regenerate_slots and not strict_asset_regeneration: - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) - if locked or decision["accepted"]: - return {**spec, **metrics, **decision, "status": "reused", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_existing_accepted_or_threshold_passed", "fallback_used": False} - crop_path = Path(spec["reference_crop_path"]) - attempts = max(1, int(asset_retries) + 1) if strict_asset_regeneration else 1 - last_error = None - for attempt in range(1, attempts + 1): - try: - if asset_mode == "placeholder": - _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) - request_count = 0 - status = "placeholder" - elif asset_mode == "crop" and asset_policy != "smart-api": - shutil.copyfile(crop_path, out_path) - request_count = 0 - status = "reference_crop" - elif asset_mode == "crop" and asset_policy == "smart-api": - _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) - request_count = 0 - status = "crop_disabled_by_smart_api_policy_placeholder" - else: - result = _api_generate_asset(spec, crop_path, out_path) - request_count = int(result.get("api_requests_attempted", 1)) - status = result.get("status", "generated") - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=strict_asset_regeneration) - return { - **spec, - **metrics, - **decision, - "status": status, - "asset_path": str(out_path), - "api_requests_attempted": request_count, - "attempt": attempt, - "economy_decision": "generated_once" if economy_mode else "generated", - "fallback_used": False, - } - except Exception as exc: - last_error = str(exc) - if asset_mode == "api" and asset_policy != "smart-api": - try: - shutil.copyfile(crop_path, out_path) - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) - return { - **spec, - **metrics, - **decision, - "status": "api_failed_reference_crop_fallback", - "asset_path": str(out_path), - "api_requests_attempted": 1, - "attempt": attempt, - "economy_decision": "fallback_after_api_error", - "fallback_used": True, - "error": last_error, - } - except Exception: - pass - if asset_mode == "api" and asset_policy == "smart-api": - try: - _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]) - metrics = _foreground_metrics(out_path, spec["background_color_hex"]) - decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0.0), strict=False) - return { - **spec, - **metrics, - **decision, - "status": "api_failed_placeholder_fallback", - "asset_path": str(out_path), - "api_requests_attempted": 1, - "attempt": attempt, - "economy_decision": "placeholder_after_api_error_crop_disabled", - "fallback_used": True, - "error": last_error, - } - except Exception: - pass - return {**spec, "status": "failed", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": last_error} - - primary_specs = [spec for spec in specs if spec.get("asset_decision") != "reuse_existing"] - reuse_specs = [spec for spec in specs if spec.get("asset_decision") == "reuse_existing"] - workers = max(1, min(int(asset_workers), 12)) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(run_one, spec) for spec in primary_specs] - for future in as_completed(futures): - report = future.result() - reports.append(report) - api_requests += int(report.get("api_requests_attempted") or 0) - for spec in reuse_specs: - report = run_one(spec) - reports.append(report) - api_requests += int(report.get("api_requests_attempted") or 0) - reports.sort(key=lambda item: str(item.get("slot_id"))) - program["assets"] = [{"id": spec["asset_id"], "path": f"assets/{spec['asset_id']}.png", "source": "slot_asset"} for spec in specs] - summary = { - "summary": "Asset generation report for reusable editable rebuild workflow.", - "asset_mode": asset_mode, - "asset_policy": asset_policy, - "final_crop_assets_allowed": asset_policy != "smart-api", - "economy_mode": economy_mode, - "asset_retries": asset_retries, - "strict_asset_regeneration": strict_asset_regeneration, - "api_requests_attempted": api_requests, - "assets": reports, - } - economy = { - "summary": "Economy policy decisions.", - "api_requests_attempted": api_requests, - "accepted_assets_file": str(out / "accepted_assets.json"), - "regenerate_slots": sorted(regenerate_slots), - "thresholds": {key: {"min": value[0], "max": value[1]} for key, value in ASSET_THRESHOLDS.items()}, - "asset_policy": asset_policy, - "assets": [{ - "slot_id": item.get("slot_id"), - "slot_type": item.get("slot_type"), - "economy_decision": item.get("economy_decision"), - "selected_reason": item.get("selected_reason"), - "api_requests_attempted": item.get("api_requests_attempted"), - "fallback_used": item.get("fallback_used"), - } for item in reports], - } - write_json(out / "asset_generation_report.json", summary) - write_json(out / "asset_economy_report.json", economy) - - def image_size(path_text: str | None) -> list[int] | None: - if not path_text or not Path(path_text).exists(): - return None - with Image.open(path_text) as img: - return list(img.size) - - write_json(out / "asset_ratio_fit_report.json", { - "summary": "Aspect-ratio fit report.", - "assets": [{ - "slot_id": item.get("slot_id"), - "slot_aspect_ratio": item.get("slot_aspect_ratio"), - "generation_aspect_ratio": item.get("generation_aspect_ratio"), - "generated_image_size": image_size(item.get("asset_path")), - "foreground_bbox_fill_percent": item.get("foreground_bbox_fill_percent"), - "slot_fit_waste_percent": None, - "aspect_ratio_match_status": "matched" if item.get("generation_aspect_ratio") else "unknown", - } for item in reports], - }) - return reports, summary - - -def _export_preview(pptx_path: Path, out: Path) -> Path | None: - preview = out / "rebuild_preview.png" - if platform.system() != "Windows": - write_text(out / "preview_export_error.txt", "PowerPoint preview export is only available on Windows with Microsoft PowerPoint installed.") - program_path = out / "figure_program.json" - if program_path.exists(): - return render_rebuild_preview(read_json(program_path), out, preview) - return None - try: - import win32com.client - app = win32com.client.Dispatch("PowerPoint.Application") - app.Visible = True - deck = app.Presentations.Open(str(pptx_path), WithWindow=False) - deck.Slides(1).Export(str(preview), "PNG") - deck.Close() - app.Quit() - return preview - except Exception as exc: - write_text(out / "preview_export_error.txt", str(exc)) - program_path = out / "figure_program.json" - if program_path.exists(): - return render_rebuild_preview(read_json(program_path), out, preview) - return None - - -def _ppt_package_counts(pptx_path: Path, reference_path: Path) -> dict: - from pptx import Presentation - - prs = Presentation(str(pptx_path)) - slide_count = len(prs.slides) - shape_count = 0 - picture_count = 0 - text_shape_count = 0 - connector_count = 0 - for slide in prs.slides: - for shape in slide.shapes: - shape_count += 1 - if getattr(shape, "shape_type", None) == 13: - picture_count += 1 - if getattr(shape, "has_text_frame", False) and shape.text_frame.text.strip(): - text_shape_count += 1 - shape_type_text = str(getattr(shape, "shape_type", "")) - if "CONNECTOR" in shape_type_text or "LINE" in shape_type_text: - connector_count += 1 - return { - "ppt_page_count": slide_count, - "shape_count": shape_count, - "picture_count": picture_count, - "text_shape_count": text_shape_count, - "connector_count": connector_count, - "contains_full_reference_image": False, - "reference_path": str(reference_path), - } - - -def _load_json_or_empty(path: Path) -> dict: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - return data if isinstance(data, dict) else {} - - -def _program_from_contracts(out: Path) -> dict: - geometry = _load_json_or_empty(out / "reference_geometry.json") - controls = _load_json_or_empty(out / "reference_controls.json") - text_program = _load_json_or_empty(out / "text_program.json") - canvas = geometry.get("canvas") if isinstance(geometry.get("canvas"), dict) else {"width_in": 10.0, "height_in": 5.625, "background": "#FFFFFF"} - if "width_in" not in canvas or "height_in" not in canvas: - width_px = int(canvas.get("width_px") or 1600) - height_px = int(canvas.get("height_px") or 900) - width_in, height_in = _canvas_inches(width_px, height_px) - canvas = {**canvas, "width_in": width_in, "height_in": height_in, "background": canvas.get("background") or "#FFFFFF"} - layout = { - "canvas": canvas, - "palette": geometry.get("palette") or ["#FFFFFF", "#4A90C2", "#E17721", "#4B9B52"], - "panels": geometry.get("panels") or [{ - "id": "reference_canvas", - "title": "Editable Rebuild", - "bbox_percent": _bbox(0.025, 0.065, 0.95, 0.84), - "editable_in": "pptx", - }], - } - slots = geometry.get("slots") or _load_json_or_empty(out / "slot_inventory.json").get("slots") or [] - program = { - "canvas": canvas, - "style": _style_from_layout(layout), - "title_block": {"title": "", "subtitle": "", "bbox_percent": _bbox(0.04, 0.02, 0.92, 0.04)}, - "panels": layout["panels"], - "cards": geometry.get("cards") or [], - "slots": slots, - "assets": [{"id": slot.get("asset_id") or slot.get("id"), "path": f"assets/{slot.get('asset_id') or slot.get('id')}.png", "source": "slot_asset"} for slot in slots], - "arrows": controls.get("arrows") or [], - "labels": [], - "groups": [], - "export_targets": [{"type": "pptx", "path": "editable_composition.pptx"}], - } - if isinstance(text_program.get("items"), list): - program["text_program"] = text_program - program["text_program_path"] = "text_program.json" - return program - - -def _compile_existing_contracts(reference_path: Path, out: Path, export_preview: bool = False) -> dict: - program = _program_from_contracts(out) - write_json(out / "figure_program.json", program) - pptx_path = compile_ppt(program, out) - preview = _export_preview(pptx_path, out) if export_preview else None - counts = _ppt_package_counts(pptx_path, reference_path) - report_path = out / "composition_quality_report.json" - report = _load_json_or_empty(report_path) - report["rebuild_editable_summary"] = counts - report["compile_only"] = True - write_json(report_path, report) - return { - "summary": "Editable rebuild recompiled from existing JSON contracts.", - "ok": True, - "out_dir": str(out), - "reference": str(reference_path), - "pptx": str(pptx_path), - "preview": str(preview) if preview else None, - "asset_count": len(program.get("assets", [])), - "slot_count": len(program.get("slots", [])), - "connector_count": len(program.get("arrows", [])), - "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, - "compile_only": True, - } - - -def rebuild_editable( - reference: str | Path, - out: str | Path, - asset_mode: str = "api", - asset_workers: int = 4, - asset_retries: int = 1, - economy_mode: bool = True, - text_mode: str = "ocr", - control_mode: str = "hybrid", - layout_mode: str = "hybrid", - export_preview: bool = False, - regenerate_slots: str | list[str] | None = None, - strict_asset_regeneration: bool = False, - skip_analysis: bool = False, - compile_only: bool = False, - ocr_engine: str = "paddle", - ocr_lang: str = "en_ch", - ocr_adapter: Callable | None = None, - vlm_layout_adapter: Callable | None = None, - control_adapter: Callable | None = None, - semantic_adapter: Callable | None = None, - asset_policy: str = "legacy", - design_plan_mode: str = "vlm", - design_plan_model: str | None = None, - design_adapter: Callable | None = None, - text_grouping_mode: str = "heuristic", - text_grouping_model: str | None = None, - text_grouping_adapter: Callable | None = None, - semantic_contract: dict | str | Path | None = None, -) -> dict: - reference_path = Path(reference) - if not reference_path.exists(): - raise FileNotFoundError(reference_path) - out_path = ensure_dir(out) - if compile_only: - return _compile_existing_contracts(reference_path, out_path, export_preview=export_preview) - input_dir = ensure_dir(out_path / "inputs") - archived_reference = input_dir / reference_path.name - shutil.copyfile(reference_path, archived_reference) - write_json(out_path / "input_manifest.json", { - "summary": "Input manifest for reference-only editable rebuild.", - "reference": str(reference_path), - "archived_reference": str(archived_reference), - "asset_mode": asset_mode, - "text_mode": text_mode, - "control_mode": control_mode, - "layout_mode": layout_mode, - "asset_policy": asset_policy, - "design_plan_mode": design_plan_mode, - "design_plan_model": design_plan_model, - "text_grouping_mode": text_grouping_mode, - "text_grouping_model": text_grouping_model, - "semantic_contract": str(semantic_contract) if isinstance(semantic_contract, (str, Path)) else ("inline" if semantic_contract else None), - "skip_analysis": skip_analysis, - "compile_only": compile_only, - }) - - if skip_analysis and (out_path / "reference_logic_plan.json").exists(): - design_bundle = { - "logic": _load_json_or_empty(out_path / "reference_logic_plan.json"), - "layer_plan": _load_json_or_empty(out_path / "reference_layer_plan.json"), - "generation_plan": _load_json_or_empty(out_path / "reference_generation_plan.json"), - "flow_graph": _load_json_or_empty(out_path / "reference_flow_graph.json"), - } - else: - design_bundle = plan_rebuild_design( - archived_reference, - out_path, - mode=design_plan_mode, - model=design_plan_model, - adapter=design_adapter, - fallback_on_error=True, - ) - - if skip_analysis and (out_path / "reference_geometry.json").exists(): - program = _program_from_contracts(out_path) - reference_geometry = _load_json_or_empty(out_path / "reference_geometry.json") - else: - program, reference_geometry = _build_program(archived_reference, out_path, layout_mode=layout_mode, vlm_layout_adapter=vlm_layout_adapter) - write_json(out_path / "reference_geometry.json", reference_geometry) - - if skip_analysis and (out_path / "reference_controls.json").exists(): - reference_controls = _load_json_or_empty(out_path / "reference_controls.json") - else: - reference_controls = localize_reference_controls( - archived_reference, - program.get("slots", []), - program.get("style", {}).get("palette", []), - out_path, - mode=control_mode, - control_adapter=control_adapter, - ) - program["arrows"] = reference_controls.get("arrows", []) - write_json(out_path / "reference_controls.json", reference_controls) - - reuse_text_layer = bool( - skip_analysis - and (out_path / "reference_text_geometry.json").exists() - and (out_path / "text_program.json").exists() - ) - if reuse_text_layer: - pass - elif text_mode != "off": - extractor_mode = "ocr" if text_mode == "ocr" else "heuristic" - build_text_layer( - archived_reference, - program, - program.get("style", {}), - out_path, - text_extractor_mode=extractor_mode, - ocr_engine=ocr_engine, - ocr_lang=ocr_lang, - ocr_adapter=ocr_adapter, - text_grouping_mode=text_grouping_mode, - text_grouping_model=text_grouping_model, - text_grouping_adapter=text_grouping_adapter, - ) - else: - write_json(out_path / "reference_text_geometry.json", {"summary": "Text extraction skipped.", "detection_mode": "off", "regions": []}) - write_json(out_path / "ocr_text_quality_report.json", {"summary": "Text extraction skipped.", "mode": "off", "status": "skipped"}) - - text_geometry = _load_json_or_empty(out_path / "reference_text_geometry.json") - semantic_report = _load_json_or_empty(out_path / "slot_semantic_report.json") if skip_analysis else {} - if not skip_analysis: - semantic_slots, semantic_report = plan_slot_semantics( - archived_reference, - program.get("slots", []), - program.get("panels", []), - program.get("arrows", []), - text_geometry, - semantic_adapter=semantic_adapter, - ) - program["slots"] = semantic_slots - reference_geometry["slots"] = semantic_slots - write_json(out_path / "reference_geometry.json", reference_geometry) - write_json(out_path / "slot_semantic_report.json", semantic_report) - elif not semantic_report: - semantic_report = {"summary": "Semantic planning skipped by --skip-analysis.", "semantic_vlm_status": "skipped", "slots": []} - write_json(out_path / "slot_semantic_report.json", semantic_report) - semantic_binding_report = {} - if semantic_contract: - contract = read_json(semantic_contract) if isinstance(semantic_contract, (str, Path)) else semantic_contract - program, semantic_binding_report = apply_paper_semantic_contract(program, contract, out_path) - write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) - - specs = _make_asset_specs(program, archived_reference, out_path, asset_policy=asset_policy) - if asset_policy == "smart-api": - reference_geometry["slots"] = program.get("slots", []) - write_json(out_path / "reference_geometry.json", reference_geometry) - write_json(out_path / "slot_inventory.json", {"summary": "Visual asset slot inventory.", "slots": program["slots"]}) - write_json(out_path / "asset_generation_specs.json", {"summary": "Slot-level asset generation specs.", "asset_mode": asset_mode, "specs": specs}) - regen = set() - if isinstance(regenerate_slots, str): - regen = {item.strip() for item in regenerate_slots.split(",") if item.strip()} - elif isinstance(regenerate_slots, list): - regen = {str(item).strip() for item in regenerate_slots if str(item).strip()} - asset_reports, asset_summary = _generate_assets(specs, program, out_path, asset_mode, asset_workers, asset_retries, economy_mode, regen, strict_asset_regeneration, asset_policy=asset_policy) - vlm_validation_report = build_rebuild_vlm_validation_report(out_path, reference_geometry, reference_controls, semantic_report, asset_summary) - visual_quality_report = run_rebuild_visual_quality_check( - out_path, - program, - reference_geometry=reference_geometry, - reference_controls=reference_controls, - ownership_report=_load_json_or_empty(out_path / "text_layer_ownership_report.json"), - ) +from __future__ import annotations - write_json(out_path / "figure_program.json", program) - pptx_path = compile_ppt(program, out_path) - preview = _export_preview(pptx_path, out_path) if export_preview else None +import importlib as _importlib +import sys as _sys - counts = _ppt_package_counts(pptx_path, archived_reference) - report_path = out_path / "composition_quality_report.json" - try: - report = json.loads(report_path.read_text(encoding="utf-8")) - except Exception: - report = {} - report["rebuild_editable_summary"] = counts - report["no_full_image_policy"] = { - "status": "pass", - "contains_full_reference_image": False, - "policy": "reference image is archived only; final PPT uses slot assets and editable objects", - } - write_json(report_path, report) +_implementation = _importlib.import_module(".workflows.rebuild_editable", __package__) - result = { - "summary": "Reusable image-to-editable-PPT rebuild complete.", - "ok": True, - "out_dir": str(out_path), - "reference": str(reference_path), - "pptx": str(pptx_path), - "preview": str(preview) if preview else None, - "asset_mode": asset_mode, - "asset_workers": asset_workers, - "asset_retries": asset_retries, - "economy_mode": economy_mode, - "asset_policy": asset_policy, - "design_plan_mode": design_plan_mode, - "design_plan_effective_mode": design_bundle.get("logic", {}).get("effective_mode"), - "design_plan_model": design_bundle.get("logic", {}).get("model"), - "text_grouping_mode": text_grouping_mode, - "api_requests_attempted": asset_summary.get("api_requests_attempted", 0), - "asset_count": len(asset_reports), - "slot_count": len(program.get("slots", [])), - "text_count": len(program.get("text_program", {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, - "connector_count": len(program.get("arrows", [])), - "text_mode": text_mode, - "text_layer_reused": reuse_text_layer, - "control_mode": control_mode, - "layout_mode": layout_mode, - "rebuild_visual_quality_status": visual_quality_report.get("status"), - "semantic_binding_status": semantic_binding_report.get("status") if semantic_binding_report else None, - "reports": { - "input_manifest": str(out_path / "input_manifest.json"), - "reference_logic_plan": str(out_path / "reference_logic_plan.json"), - "reference_layer_plan": str(out_path / "reference_layer_plan.json"), - "reference_generation_plan": str(out_path / "reference_generation_plan.json"), - "reference_flow_graph": str(out_path / "reference_flow_graph.json"), - "reference_geometry": str(out_path / "reference_geometry.json"), - "reference_text_geometry": str(out_path / "reference_text_geometry.json"), - "reference_controls": str(out_path / "reference_controls.json"), - "slot_inventory": str(out_path / "slot_inventory.json"), - "slot_semantic_report": str(out_path / "slot_semantic_report.json"), - "rebuild_vlm_validation_report": str(out_path / "rebuild_vlm_validation_report.json"), - "rebuild_visual_quality_report": str(out_path / "rebuild_visual_quality_report.json"), - "semantic_binding_report": str(out_path / "semantic_binding_report.json") if semantic_binding_report else None, - "asset_generation_specs": str(out_path / "asset_generation_specs.json"), - "asset_generation_report": str(out_path / "asset_generation_report.json"), - "asset_economy_report": str(out_path / "asset_economy_report.json"), - "asset_decision_report": str(out_path / "asset_decision_report.json"), - "text_asset_filter_report": str(out_path / "text_asset_filter_report.json"), - "api_asset_plan": str(out_path / "api_asset_plan.json"), - "figure_program": str(out_path / "figure_program.json"), - "composition_quality_report": str(out_path / "composition_quality_report.json"), - }, - } - write_json(out_path / "rebuild_result.json", result) - return result +_sys.modules[__name__] = _implementation diff --git a/rfs/evaluation/__init__.py b/rfs/evaluation/__init__.py index 8df7580..d049060 100644 --- a/rfs/evaluation/__init__.py +++ b/rfs/evaluation/__init__.py @@ -1,18 +1,22 @@ """Deterministic and model-assisted quality evaluation APIs.""" -from .rebuild_visual import run_rebuild_visual_quality_check +from .rebuild_visual import apply_rebuild_corrections, run_rebuild_visual_critic, run_rebuild_visual_quality_check from .benchmarking import audit_paper_image_run, fetch_benchmark_case, list_benchmark_cases, run_benchmark_case, run_fast_benchmark_case, run_fast_benchmark_suite, score_benchmark_case, validate_benchmark_case from .pdf_extraction_benchmark import run_pdf_extraction_stress_suite +from .outputs import validate_output __all__ = [ + "apply_rebuild_corrections", + "run_rebuild_visual_critic", + "run_rebuild_visual_quality_check", "list_benchmark_cases", "audit_paper_image_run", "fetch_benchmark_case", - "run_rebuild_visual_quality_check", "run_benchmark_case", "run_fast_benchmark_case", "run_fast_benchmark_suite", "run_pdf_extraction_stress_suite", "score_benchmark_case", "validate_benchmark_case", + "validate_output", ] diff --git a/rfs/evaluation/outputs.py b/rfs/evaluation/outputs.py new file mode 100644 index 0000000..29fd1e2 --- /dev/null +++ b/rfs/evaluation/outputs.py @@ -0,0 +1,5 @@ +"""Stable output-validation exports.""" + +from .validator import validate_output + +__all__ = ["validate_output"] diff --git a/rfs/evaluation/rebuild_coverage.py b/rfs/evaluation/rebuild_coverage.py new file mode 100644 index 0000000..d050f43 --- /dev/null +++ b/rfs/evaluation/rebuild_coverage.py @@ -0,0 +1,94 @@ +"""Canonical layout/control coverage audit for editable rebuilds.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from ..analysis.rebuild.controls import _append_unique as _append_unique_control, _normalize_controls +from ..utils import write_json + + +def _bbox(x: float, y: float, w: float, h: float) -> dict[str, float]: + x, y = max(0.0, min(0.995, x)), max(0.0, min(0.995, y)) + w, h = max(0.001, min(w, 1.0 - x)), max(0.001, min(h, 1.0 - y)) + return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} + + +def _collection(kind: str) -> str: + normalized = str(kind or "").strip().lower().replace("-", "_") + return {"panel": "panels", "panels": "panels", "card": "cards", "cards": "cards", "slot": "slots", "visual_slot": "slots", "slots": "slots", "legend": "legend_regions", "legend_region": "legend_regions", "legend_regions": "legend_regions"}.get(normalized, "") + + +def _exists(items: list[dict], candidate: dict) -> bool: + candidate_id = str(candidate.get("id") or "") + box = candidate.get("bbox_percent") if isinstance(candidate.get("bbox_percent"), dict) else None + for item in items: + if candidate_id and str(item.get("id") or "") == candidate_id: + return True + other = item.get("bbox_percent") if isinstance(item.get("bbox_percent"), dict) else None + if box and other: + ix = max(0.0, min(float(box["x"]) + float(box["w"]), float(other["x"]) + float(other["w"])) - max(float(box["x"]), float(other["x"]))) + iy = max(0.0, min(float(box["y"]) + float(box["h"]), float(other["y"]) + float(other["h"])) - max(float(box["y"]), float(other["y"]))) + if ix * iy / max(float(box["w"]) * float(box["h"]), 1e-9) > .72: + return True + return False + + +def _normalize_missing_layout_object(item: dict, program: dict, collection: str) -> dict | None: + if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): return None + prefix = {"panels": "panel", "cards": "card", "slots": "slot", "legend_regions": "legend"}.get(collection, "item") + used = {str(obj.get("id") or "") for key in ("panels", "cards", "slots", "legend_regions") for obj in program.get(key, []) or [] if isinstance(obj, dict)} + base, ident, suffix = str(item.get("id") or f"{prefix}_{len(used) + 1:02d}"), "", 2 + ident = base + while ident in used: ident, suffix = f"{base}_{suffix}", suffix + 1 + raw = item["bbox_percent"] + record = {**item, "id": ident, "bbox_percent": _bbox(float(raw.get("x", 0)), float(raw.get("y", 0)), float(raw.get("w", .001)), float(raw.get("h", .001))), "confidence": float(item.get("confidence") or .75), "detected_by": "combined_coverage_audit_vlm", "coverage_audit_added": True} + if collection == "slots": + record.setdefault("asset_id", ident); record.setdefault("paper_concept", record.get("prompt_subject") or record.get("label") or ident); record.setdefault("composition_type", "full_frame_icon"); record.setdefault("show_slot_caption", False); record.setdefault("panel_id", "reference_canvas"); record.setdefault("z_index", 20 + len(program.get("slots", []))) + elif collection == "cards": + record.setdefault("title", record.get("label") or ident); record.setdefault("semantic_role", "subcard_frame"); record.setdefault("shape_kind", "rounded_rect") + if str(record.get("stroke_color") or "").strip(): record.setdefault("color_status", "vlm_reference_estimated"); record.setdefault("color_source", "vlm") + else: record["stroke_color"] = "#59AFCB"; record.setdefault("color_status", "fallback_missing_vlm_color"); record.setdefault("color_source", "local_default") + record.setdefault("stroke_width_pt", 1.5); record["dash_style"] = "dash" if str(record.get("dash_style") or "solid").lower() == "dashed" else str(record.get("dash_style") or "solid"); record.setdefault("fill_color", "#FFFFFF"); record.setdefault("fill_transparency", 1.0); record.setdefault("corner_radius", .08); record.setdefault("z_index", 12); record.setdefault("editable_in", "pptx") + else: + record.setdefault("title", record.get("label") or ident); record.setdefault("editable_in", "pptx") + return record + + +def _combined_bindable_objects(program: dict) -> list[dict]: + result, seen = [], set() + for key in ("panels", "cards", "slots"): + for item in program.get(key, []) or []: + ident = str(item.get("id") or "") if isinstance(item, dict) else "" + if ident and ident not in seen and isinstance(item.get("bbox_percent"), dict): result.append(item); seen.add(ident) + return result + + +def _apply_combined_coverage_audit(reference: Path, out: Path, program: dict, reference_geometry: dict, reference_controls: dict, palette: list[str], flow_graph: dict | None, mode: str, adapter: Callable | None) -> tuple[dict, dict, dict, dict]: + audit = {"summary": "Combined layout/control coverage audit report.", "mode": mode, "status": "skipped", "missing_layout_count": 0, "appended_layout_count": 0, "missing_control_count": 0, "appended_control_count": 0} + if mode != "vlm" or not adapter: + audit["status"] = "skipped" if mode != "vlm" else "unavailable"; write_json(out / "reference_combined_coverage_audit.json", audit); return program, reference_geometry, reference_controls, audit + try: + raw = adapter(reference, reference_geometry, reference_controls, flow_graph) or {}; raw_layout, raw_controls = raw.get("missing_layout_objects", []), raw.get("missing_controls", []); invalid = appended = 0 + for item in raw_layout: + collection = _collection(item.get("kind") if isinstance(item, dict) else ""); candidate = _normalize_missing_layout_object(item, program, collection) if collection else None + if not candidate: invalid += 1; continue + target = program.setdefault(collection, []) + if _exists(target, candidate): continue + target.append(candidate); reference_geometry.setdefault(collection, []) + if not _exists(reference_geometry[collection], candidate): reference_geometry[collection].append(candidate) + appended += 1 + arrows, added = list(reference_controls.get("arrows", []) if isinstance(reference_controls.get("arrows"), list) else []), 0 + normalized = _normalize_controls(raw_controls, _combined_bindable_objects(program), palette) + for candidate in normalized: + candidate = {**candidate, "detected_by": "combined_coverage_audit_vlm", "binding_source": "combined_coverage_audit_vlm"} + if _append_unique_control(arrows, candidate): added += 1 + if added: reference_controls = {**reference_controls, "arrows": arrows, "merged_arrow_count": len(arrows), "combined_coverage_appended_count": added}; program["arrows"] = arrows + audit.update({"status": "used", "raw_audit": raw, "missing_layout_count": len(raw_layout), "valid_missing_layout_count": max(0, len(raw_layout) - invalid), "appended_layout_count": appended, "invalid_layout_count": invalid, "missing_control_count": len(raw_controls), "valid_missing_control_count": len(normalized), "appended_control_count": added}) + except Exception as exc: audit.update({"status": "fallback", "fallback_reason": f"combined_coverage_audit_failed:{exc}"}) + write_json(out / "reference_combined_coverage_audit.json", audit) + return program, reference_geometry, reference_controls, audit + + +__all__ = ["_apply_combined_coverage_audit", "_combined_bindable_objects", "_normalize_missing_layout_object"] diff --git a/rfs/evaluation/rebuild_visual.py b/rfs/evaluation/rebuild_visual.py index 1b10818..9b1537b 100644 --- a/rfs/evaluation/rebuild_visual.py +++ b/rfs/evaluation/rebuild_visual.py @@ -1,9 +1,12 @@ from __future__ import annotations +import copy +import json from pathlib import Path from typing import Any from ..utils import write_json +from ..providers import call_vlm_json, resolve_vlm_model def _bbox(item: dict) -> dict | None: @@ -193,3 +196,335 @@ def run_rebuild_visual_quality_check( } write_json(Path(out_dir) / "rebuild_visual_quality_report.json", report) return report + + +def _brief_text(program: dict) -> list[dict]: + return [{ + "id": item.get("id"), + "source_reference_text_id": item.get("source_reference_text_id"), + "text": item.get("text"), + "role": item.get("role"), + "target_id": item.get("target_id"), + "bbox_percent": item.get("bbox_percent"), + "font_size_pt": item.get("font_size_pt"), + "align": item.get("align"), + "visible": item.get("visible", True), + "z_index": item.get("z_index"), + "layer_ownership": item.get("layer_ownership"), + } for item in _visible_text_items(program)] + + +def _brief_objects(program: dict, key: str) -> list[dict]: + return [{ + "id": item.get("id"), + "bbox_percent": item.get("bbox_percent"), + "title": item.get("title"), + "paper_concept": item.get("paper_concept"), + } for item in program.get(key, []) or [] if isinstance(item, dict)] + + +def _brief_arrows(program: dict) -> list[dict]: + return [{ + "id": item.get("id"), + "source_id": item.get("source_id") or item.get("source"), + "target_id": item.get("target_id") or item.get("target"), + "path_percent": item.get("path_percent"), + "render_style": item.get("render_style"), + "stroke_width_pt": item.get("stroke_width_pt"), + "line_pattern": item.get("line_pattern"), + } for item in program.get("arrows", []) or [] if isinstance(item, dict)] + + +def run_rebuild_visual_critic( + out_dir: str | Path, + reference_path: str | Path, + preview_path: str | Path | None, + program: dict, + deterministic_report: dict, + mode: str = "heuristic", + model: str | None = None, + iteration: int = 0, + vlm_timeout: int | None = None, + vlm_retries: int | None = None, +) -> dict: + out = Path(out_dir) + effective_mode = str(mode or "off").lower() + if effective_mode == "off": + critic = {"summary": "Rebuild visual critic skipped.", "mode": "off", "status": "skipped", "patches": [], "warnings": []} + elif effective_mode == "heuristic": + critic = { + "summary": "Heuristic rebuild visual critic produced no mutating patches.", + "mode": "heuristic", + "status": deterministic_report.get("status", "pass"), + "patches": [], + "issues": deterministic_report.get("issues", []), + "warnings": [], + } + elif effective_mode == "vlm": + if not preview_path or not Path(preview_path).exists(): + critic = { + "summary": "VLM rebuild visual critic skipped because preview PNG is unavailable.", + "mode": "vlm", + "status": "skipped", + "patches": [], + "warnings": ["preview_png_unavailable"], + } + else: + model_name = resolve_vlm_model("RFS_REBUILD_CRITIC_MODEL", "RFS_CRITIC_MODEL", explicit_model=model) + prompt = f""" +You are a visual patch generator for an editable PowerPoint reconstruction. +Compare image 1 (reference) and image 2 (current PPT preview). +Only output JSON. Do not output markdown, prose, Python, SVG, or PPT code. + +Every fix must be a concrete patch operation with exact fields. Do not say "optimize", "improve", or "make better" unless you also provide exact patch fields. +Examples of concrete text instructions: +- update_text with font_size_pt: 10, z_index: 8 +- update_text with bbox_percent: {{"x":0.615,"y":0.338,"w":0.12,"h":0.03}}, align:"center" +- update_arrow with path_percent:[[0.1,0.2],[0.3,0.2]], render_style:"line_connector" + +Allowed patch operations: +- update_text: text_id required; may set bbox_percent, font_size_pt, align, visible, z_index, layer_ownership. +- update_slot: slot_id required; may set bbox_percent only. +- update_panel: panel_id required; may set bbox_percent only. +- update_arrow: arrow_id required; may set path_percent, render_style, stroke_width_pt, line_pattern. + +Forbidden: +- Do not change text content. +- Do not request crop masking or image pixel edits. +- Do not write PPT code. +- Do not invent new objects. + +Return schema: +{{ + "summary": "...", + "mode": "vlm", + "status": "pass|needs_patch|blocked", + "patches": [ + {{"op":"update_text","text_id":"text_ref","font_size_pt":10,"z_index":8,"reason":"specific reason"}}, + {{"op":"update_slot","slot_id":"slot_1","bbox_percent":{{"x":0.1,"y":0.2,"w":0.2,"h":0.1}},"reason":"specific reason"}}, + {{"op":"update_arrow","arrow_id":"arrow_1","path_percent":[[0.1,0.2],[0.3,0.2]],"reason":"specific reason"}} + ], + "warnings": [] +}} + +Deterministic issues: +{json.dumps(deterministic_report.get("issues", []), ensure_ascii=False)} + +Text items: +{json.dumps(_brief_text(program), ensure_ascii=False)} + +Panels: +{json.dumps(_brief_objects(program, "panels"), ensure_ascii=False)} + +Slots: +{json.dumps(_brief_objects(program, "slots"), ensure_ascii=False)} + +Arrows: +{json.dumps(_brief_arrows(program), ensure_ascii=False)} +""".strip() + critic = call_vlm_json(prompt, [reference_path, preview_path], model=model_name, timeout=vlm_timeout, retries=vlm_retries) + critic.setdefault("summary", "VLM rebuild visual critic returned concrete patch operations.") + critic.setdefault("mode", "vlm") + critic.setdefault("status", "needs_patch" if critic.get("patches") else "pass") + critic.setdefault("warnings", []) + critic["vlm_model"] = model_name + else: + raise ValueError(f"Unsupported rebuild visual critic mode: {mode}") + write_json(out / f"rebuild_visual_critic_iter_{iteration}.json", critic) + return critic + + +def _limit_bbox_change(original: dict, requested: dict, center_limit: float, size_limit: float) -> dict: + ox, oy, ow, oh = float(original["x"]), float(original["y"]), float(original["w"]), float(original["h"]) + rx = float(requested.get("x", ox)) + ry = float(requested.get("y", oy)) + rw = float(requested.get("w", ow)) + rh = float(requested.get("h", oh)) + ocx, ocy = ox + ow / 2, oy + oh / 2 + rcx, rcy = rx + rw / 2, ry + rh / 2 + dcx = max(-center_limit, min(center_limit, rcx - ocx)) + dcy = max(-center_limit, min(center_limit, rcy - ocy)) + min_w, max_w = ow * (1 - size_limit), ow * (1 + size_limit) + min_h, max_h = oh * (1 - size_limit), oh * (1 + size_limit) + nw = max(0.001, min(max_w, max(min_w, rw))) + nh = max(0.001, min(max_h, max(min_h, rh))) + nx = ocx + dcx - nw / 2 + ny = ocy + dcy - nh / 2 + clamped_x = max(0.0, min(0.995, nx)) + clamped_y = max(0.0, min(0.995, ny)) + return { + "x": round(clamped_x, 4), + "y": round(clamped_y, 4), + "w": round(max(0.001, min(nw, 1.0 - clamped_x)), 4), + "h": round(max(0.001, min(nh, 1.0 - clamped_y)), 4), + } + + +def _limit_font_change(original: object, requested: object) -> float | None: + try: + original_value = float(original) + requested_value = float(requested) + except Exception: + return None + if original_value <= 0: + return round(max(1.0, requested_value), 2) + return round(max(original_value * 0.8, min(original_value * 1.2, requested_value)), 2) + + +def _coerce_path(path: object) -> list[list[float]] | None: + if not isinstance(path, list) or len(path) < 2: + return None + points = [] + for item in path: + if not isinstance(item, list) or len(item) < 2: + return None + points.append([round(max(0.0, min(1.0, float(item[0]))), 4), round(max(0.0, min(1.0, float(item[1]))), 4)]) + return points + + +def _refresh_ownership_report(out: Path, program: dict) -> None: + path = out / "text_layer_ownership_report.json" + if not path.exists(): + return + try: + report = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return + text_program = program.get("text_program", {}) if isinstance(program.get("text_program"), dict) else {} + by_source = {str(item.get("source_reference_text_id") or ""): item for item in text_program.get("items", []) if isinstance(item, dict)} + changed = False + for item in report.get("items", []) or []: + if not isinstance(item, dict): + continue + text_id = str(item.get("text_id") or "") + if text_id in by_source: + item["layer_ownership"] = by_source[text_id].get("layer_ownership", item.get("layer_ownership")) + item["included_in_text_program"] = bool(by_source[text_id].get("visible", True)) and item["layer_ownership"] == "editable_text_layer" + changed = True + if changed: + write_json(path, report) + + +def apply_rebuild_corrections(out_dir: str | Path, program: dict, critic: dict, iteration: int = 0) -> tuple[dict, dict]: + out = Path(out_dir) + updated = copy.deepcopy(program) + patches = critic.get("patches", []) if isinstance(critic.get("patches"), list) else [] + applied = [] + rejected = [] + changed_slots: set[str] = set() + changed_arrows = False + text_program = updated.get("text_program") if isinstance(updated.get("text_program"), dict) else {"items": []} + text_by_id = {str(item.get("id") or ""): item for item in text_program.get("items", []) if isinstance(item, dict)} + slots_by_id = {str(item.get("id") or ""): item for item in updated.get("slots", []) if isinstance(item, dict)} + panels_by_id = {str(item.get("id") or ""): item for item in updated.get("panels", []) if isinstance(item, dict)} + arrows_by_id = {str(item.get("id") or ""): item for item in updated.get("arrows", []) if isinstance(item, dict)} + + for patch in patches: + if not isinstance(patch, dict): + rejected.append({"patch": patch, "reason": "patch_not_object"}) + continue + op = str(patch.get("op") or "").strip() + if op == "update_text": + text_id = str(patch.get("text_id") or "") + item = text_by_id.get(text_id) + if not item: + rejected.append({"patch": patch, "reason": "unknown_text_id"}) + continue + changed = {} + if isinstance(patch.get("bbox_percent"), dict) and isinstance(item.get("bbox_percent"), dict): + item["bbox_percent"] = _limit_bbox_change(item["bbox_percent"], patch["bbox_percent"], center_limit=0.03, size_limit=0.20) + item["center_percent"] = {"x": round(float(item["bbox_percent"]["x"]) + float(item["bbox_percent"]["w"]) / 2, 4), "y": round(float(item["bbox_percent"]["y"]) + float(item["bbox_percent"]["h"]) / 2, 4)} + item["width_percent"] = item["bbox_percent"]["w"] + item["height_percent"] = item["bbox_percent"]["h"] + changed["bbox_percent"] = item["bbox_percent"] + if patch.get("font_size_pt") is not None: + limited = _limit_font_change(item.get("font_size_pt"), patch.get("font_size_pt")) + if limited is not None: + item["font_size_pt"] = limited + changed["font_size_pt"] = limited + if str(patch.get("align") or "").lower() in {"left", "center", "right"}: + item["align"] = str(patch["align"]).lower() + changed["align"] = item["align"] + if patch.get("visible") is not None: + item["visible"] = bool(patch.get("visible")) + changed["visible"] = item["visible"] + if patch.get("z_index") is not None: + try: + item["z_index"] = int(max(0, min(1000, int(patch.get("z_index"))))) + changed["z_index"] = item["z_index"] + except Exception: + pass + ownership = str(patch.get("layer_ownership") or "") + if ownership in {"editable_text_layer", "raster_asset_layer", "decorative_asset_text", "ignore"}: + item["layer_ownership"] = ownership + if ownership != "editable_text_layer": + item["visible"] = False + changed["layer_ownership"] = ownership + if changed: + applied.append({"op": op, "text_id": text_id, "applied": changed, "reason": patch.get("reason")}) + else: + rejected.append({"patch": patch, "reason": "no_allowed_text_fields"}) + elif op in {"update_slot", "update_panel"}: + key = "slot_id" if op == "update_slot" else "panel_id" + object_id = str(patch.get(key) or "") + item = (slots_by_id if op == "update_slot" else panels_by_id).get(object_id) + if not item or not isinstance(patch.get("bbox_percent"), dict) or not isinstance(item.get("bbox_percent"), dict): + rejected.append({"patch": patch, "reason": f"unknown_or_invalid_{key}"}) + continue + item["bbox_percent"] = _limit_bbox_change(item["bbox_percent"], patch["bbox_percent"], center_limit=0.05, size_limit=0.20) + if op == "update_slot": + changed_slots.add(object_id) + applied.append({"op": op, key: object_id, "applied": {"bbox_percent": item["bbox_percent"]}, "reason": patch.get("reason")}) + elif op == "update_arrow": + arrow_id = str(patch.get("arrow_id") or "") + item = arrows_by_id.get(arrow_id) + if not item: + rejected.append({"patch": patch, "reason": "unknown_arrow_id"}) + continue + changed = {} + path = _coerce_path(patch.get("path_percent")) + if path: + item["path_percent"] = path + changed["path_percent"] = path + if str(patch.get("render_style") or "") in {"filled_block_arrow", "line_connector", "elbow_connector", "branch_line_connector", "dashed_loop_connector"}: + item["render_style"] = str(patch["render_style"]) + changed["render_style"] = item["render_style"] + if patch.get("stroke_width_pt") is not None: + try: + item["stroke_width_pt"] = round(max(0.25, min(12.0, float(patch["stroke_width_pt"]))), 2) + changed["stroke_width_pt"] = item["stroke_width_pt"] + except Exception: + pass + if str(patch.get("line_pattern") or "") in {"solid", "dash", "dashed"}: + item["line_pattern"] = "dash" if str(patch["line_pattern"]) == "dashed" else str(patch["line_pattern"]) + changed["line_pattern"] = item["line_pattern"] + if changed: + changed_arrows = True + applied.append({"op": op, "arrow_id": arrow_id, "applied": changed, "reason": patch.get("reason")}) + else: + rejected.append({"patch": patch, "reason": "no_allowed_arrow_fields"}) + else: + rejected.append({"patch": patch, "reason": "unsupported_operation"}) + + correction_report = { + "summary": "Applied controlled rebuild visual critic corrections.", + "iteration": iteration, + "status": "changed" if applied else "no_changes", + "applied_count": len(applied), + "rejected_count": len(rejected), + "changed_slot_ids": sorted(changed_slots), + "changed_arrows": changed_arrows, + "applied": applied, + "rejected": rejected, + "limits": { + "text_center_move_percent": 0.03, + "text_font_scale": "80%-120%", + "slot_panel_center_move_percent": 0.05, + "slot_panel_size_scale": "80%-120%", + }, + } + write_json(out / f"rebuild_corrections_iter_{iteration}.json", correction_report) + if isinstance(updated.get("text_program"), dict): + write_json(out / "text_program.json", updated["text_program"]) + _refresh_ownership_report(out, updated) + return updated, correction_report diff --git a/rfs/evaluation/validator.py b/rfs/evaluation/validator.py new file mode 100644 index 0000000..413a5c2 --- /dev/null +++ b/rfs/evaluation/validator.py @@ -0,0 +1,721 @@ +from __future__ import annotations + +"""Canonical output validation implementation.""" + +import json +from pathlib import Path + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff"} +REQUIRED = [ + "input_manifest.json", + "reference_geometry.json", + "reference_control_candidates.json", + "slot_overlay.png", + "reference_control_overlay.png", + "reference_controls.json", + "arrow_style_profile.json", + "selected_arrow_routes.json", + "arrow_quality_report.json", + "slot_inventory.json", + "reference_style_profile.json", + "style_sheet.md", + "layout_plan.json", + "figure_program.json", + "reference_text_geometry.json", + "text_program.json", + "ocr_text_quality_report.json", + "text_alignment_report.json", + "slot_visual_spec.json", + "reference_slot_prompt_brief.json", + "slot_prompt_plan.json", + "prompts.md", + "asset_quality_report.json", + "asset_complexity_report.json", + "composition_quality_report.json", + "asset_visual_review.json", + "asset_contact_sheet.png", + "asset_candidate_contact_sheet.png", + "editable_composition.pptx", + "alignment_review.md", + "critic_report.md", + "visual_critic_iter_0.json", +] +FORBIDDEN_MARKERS = ["ok_after_crop", "cover-crop", "fill-crop", "forced crop", "crop-to-ratio", "single full diagram", "vector-only", "svg-only", "baked labels"] +COARSE_RATIO_PRESETS = {"1:1", "4:3", "3:4", "16:9", "9:16", "2:1", "1:2", "3:2", "2:3"} +ARROW_ASSET_ID_TERMS = ("arrow", "dashed_arc", "dashed_arrows", "transition_arrow", "loop_dashed", "graph_connector") +COMPLEXITY_BLOCKERS = {"too_simple", "generic_icon", "reference_crop_ignored", "single_object_on_blank_background", "style_drift"} + + +def _front_summary(path: Path) -> bool: + if path.suffix.lower() == ".json": + try: + data = json.loads(path.read_text(encoding="utf-8")) + return isinstance(data, dict) and bool(str(data.get("summary", "")).strip()) + except Exception: + return False + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + text = line.strip() + if not text: + continue + return text.lstrip("#").strip().lower() == "summary" + return False + + +def _is_precise_ratio(value: object) -> bool: + text = str(value or "").strip() + if text in COARSE_RATIO_PRESETS: + return False + parts = text.split(":") + if len(parts) != 2: + return False + try: + return all("." in part and len(part.split(".", 1)[1]) >= 3 and float(part) > 0 for part in parts) + except Exception: + return False + + +def _validate_geometry_item(item: dict, label: str, errors: list[str]) -> None: + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "aspect_ratio_decimal", "aspect_ratio_w_h", "target_pixels_exact"]: + if key not in item: + errors.append(f"{label} missing {key}") + if not _is_precise_ratio(item.get("aspect_ratio_w_h")): + errors.append(f"{label} must use precise decimal aspect_ratio_w_h") + try: + if float(item.get("aspect_ratio_decimal", 0)) <= 0: + errors.append(f"{label} aspect_ratio_decimal must be positive") + except Exception: + errors.append(f"{label} aspect_ratio_decimal must be numeric") + + +def _pixel_pair(value: object) -> tuple[float, float] | None: + if not isinstance(value, dict): + return None + try: + return float(value["width"]), float(value["height"]) + except Exception: + return None + + +def _color_family(hex_color: str) -> str: + text = str(hex_color).strip().lstrip("#") + if len(text) < 6: + return "unknown" + try: + r, g, b = int(text[0:2], 16), int(text[2:4], 16), int(text[4:6], 16) + except Exception: + return "unknown" + if max(r, g, b) - min(r, g, b) < 24: + return "neutral" + if r >= g and r >= b: + return "warm" + if g >= r and g >= b: + return "green" + return "blue" + + +def _arrow_like_asset_id(value: object) -> bool: + text = str(value or "").lower() + return any(term in text for term in ARROW_ASSET_ID_TERMS) + + +def validate_output(out_dir: str | Path) -> dict: + root = Path(out_dir) + errors = [] + warnings = [] + if not root.exists(): + return {"summary": "Framework output validation result.", "ok": False, "errors": [f"Missing output directory: {root}"], "warnings": []} + + for name in REQUIRED: + path = root / name + if not path.exists() or path.stat().st_size == 0: + errors.append(f"Missing non-empty {name}") + elif path.suffix.lower() in {".md", ".json"} and not _front_summary(path): + errors.append(f"{name} must start with Summary or top-level summary") + + try: + manifest = json.loads((root / "input_manifest.json").read_text(encoding="utf-8")) + if not manifest.get("paper_archived"): + errors.append("input_manifest.json missing archived paper path") + if not manifest.get("reference_archived"): + errors.append("input_manifest.json missing archived reference path") + except Exception as exc: + errors.append(f"Invalid input_manifest.json: {exc}") + + try: + geometry = json.loads((root / "reference_geometry.json").read_text(encoding="utf-8")) + if not isinstance(geometry.get("slots"), list) or len(geometry.get("slots", [])) < 25: + errors.append("reference_geometry.json must contain at least 25 slot geometry entries") + if not isinstance(geometry.get("panels"), list) or not geometry.get("panels"): + errors.append("reference_geometry.json must contain panel geometry entries") + for item in geometry.get("slots", []): + if isinstance(item, dict): + _validate_geometry_item(item, f"reference_geometry slot {item.get('id')}", errors) + if _arrow_like_asset_id(item.get("id")): + errors.append(f"reference_geometry slot {item.get('id')} looks like an arrow/control and must be moved to reference_controls.json") + for item in geometry.get("panels", []): + if isinstance(item, dict): + _validate_geometry_item(item, f"reference_geometry panel {item.get('id')}", errors) + for item in geometry.get("controls", []): + if isinstance(item, dict): + _validate_geometry_item(item, f"reference_geometry control {item.get('id')}", errors) + if not str(item.get("source_id", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing source_id") + if not str(item.get("target_id", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing target_id") + if not str(item.get("source_anchor", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing source_anchor") + if not str(item.get("target_anchor", "")).strip(): + errors.append(f"reference_geometry control {item.get('id')} missing target_anchor") + if str(item.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"reference_geometry control {item.get('id')} must render as a PPT shape, not an image asset") + palette = geometry.get("reference_palette", []) + if not isinstance(palette, list) or len({str(c).upper() for c in palette}) < 4: + errors.append("reference_geometry.json reference_palette must contain at least 4 distinct colors") + if not isinstance(geometry.get("color_tokens"), list) or not geometry.get("color_tokens"): + errors.append("reference_geometry.json must contain color_tokens") + except Exception as exc: + errors.append(f"Invalid reference_geometry.json: {exc}") + + try: + candidates_doc = json.loads((root / "reference_control_candidates.json").read_text(encoding="utf-8")) + if not str(candidates_doc.get("effective_mode", "")).strip(): + errors.append("reference_control_candidates.json missing effective_mode") + if not isinstance(candidates_doc.get("candidates"), list): + errors.append("reference_control_candidates.json candidates must be a list") + if str(candidates_doc.get("slot_overlay_path", "")).strip() and not (root / str(candidates_doc.get("slot_overlay_path"))).exists(): + errors.append("reference_control_candidates.json slot_overlay_path does not exist") + if str(candidates_doc.get("control_overlay_path", "")).strip() and not (root / str(candidates_doc.get("control_overlay_path"))).exists(): + errors.append("reference_control_candidates.json control_overlay_path does not exist") + for item in candidates_doc.get("candidates", []) if isinstance(candidates_doc.get("candidates"), list) else []: + cid = item.get("id") + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "path_percent", "editable_in", "render_policy"]: + if key not in item: + errors.append(f"reference_control_candidates item {cid} missing {key}") + if _arrow_like_asset_id(item.get("asset_id")): + errors.append(f"reference_control_candidates item {cid} must not bind to an image asset") + except Exception as exc: + errors.append(f"Invalid reference_control_candidates.json: {exc}") + + try: + controls_doc = json.loads((root / "reference_controls.json").read_text(encoding="utf-8")) + controls = controls_doc.get("controls", []) + if not isinstance(controls, list): + errors.append("reference_controls.json controls must be a list") + for item in controls if isinstance(controls, list) else []: + cid = item.get("id") + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "source_id", "target_id", "source_anchor", "target_anchor", "path_percent", "style_token_id", "editable_in", "render_policy"]: + if key not in item: + errors.append(f"reference_controls item {cid} missing {key}") + if not str(item.get("source_id", "")).strip(): + errors.append(f"reference_controls item {cid} missing non-empty source_id") + if not str(item.get("target_id", "")).strip(): + errors.append(f"reference_controls item {cid} missing non-empty target_id") + if not isinstance(item.get("path_percent"), list) or len(item.get("path_percent", [])) < 2: + errors.append(f"reference_controls item {cid} must have at least 2 path_percent points") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"reference_controls item {cid} must be editable in PPTX") + if str(item.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"reference_controls item {cid} must use ppt_shape_not_image_asset") + except Exception as exc: + errors.append(f"Invalid reference_controls.json: {exc}") + + try: + arrow_style = json.loads((root / "arrow_style_profile.json").read_text(encoding="utf-8")) + if str(arrow_style.get("reference_priority", "")).lower() != "reference_image_hard_constraint": + errors.append("arrow_style_profile.json must keep reference_image_hard_constraint") + if not isinstance(arrow_style.get("style_rules"), dict) or not arrow_style.get("style_rules"): + errors.append("arrow_style_profile.json must contain style_rules") + if "reference" not in str(arrow_style.get("routing_principle", "")).lower(): + errors.append("arrow_style_profile.json routing_principle must explicitly preserve the reference image") + if not str(arrow_style.get("routing_algorithm", "")).strip(): + errors.append("arrow_style_profile.json must record routing_algorithm") + if "fallback" not in str(arrow_style.get("fallback_routing_policy", "")).lower(): + errors.append("arrow_style_profile.json must record fallback_routing_policy") + except Exception as exc: + errors.append(f"Invalid arrow_style_profile.json: {exc}") + + try: + routes_doc = json.loads((root / "selected_arrow_routes.json").read_text(encoding="utf-8")) + routes = routes_doc.get("routes", []) + if not isinstance(routes, list): + errors.append("selected_arrow_routes.json routes must be a list") + for item in routes if isinstance(routes, list) else []: + rid = item.get("id") + for key in ["source_id", "target_id", "semantic_role", "route_style", "path_percent", "style_token_id", "stroke_width_pt", "arrowhead_size", "line_cap", "routing_algorithm", "route_generation_status"]: + if key not in item or not str(item.get(key, "")).strip(): + errors.append(f"selected_arrow_routes item {rid} missing {key}") + if item.get("reference_locked") and item.get("reference_path_preserved") is not True: + errors.append(f"selected_arrow_routes item {rid} overrides a locked reference path") + if not isinstance(item.get("path_percent"), list) or len(item.get("path_percent", [])) < 2: + errors.append(f"selected_arrow_routes item {rid} must have at least 2 path_percent points") + if str(item.get("route_generation_status", "")).lower() == "aesthetic_tunnel_adjusted": + if not isinstance(item.get("reference_original_path_percent"), list) or len(item.get("reference_original_path_percent", [])) < 2: + errors.append(f"selected_arrow_routes item {rid} missing reference_original_path_percent for aesthetic adjustment") + if item.get("reference_tunnel_preserved") is not True: + errors.append(f"selected_arrow_routes item {rid} left its reference tunnel") + except Exception as exc: + errors.append(f"Invalid selected_arrow_routes.json: {exc}") + + try: + arrow_quality = json.loads((root / "arrow_quality_report.json").read_text(encoding="utf-8")) + if str(arrow_quality.get("status", "")).lower() in {"fail", "failed", "blocked"}: + errors.append("arrow_quality_report.json reports failed arrow quality") + if arrow_quality.get("reference_path_overrides"): + errors.append("arrow_quality_report.json contains reference_path_overrides") + if arrow_quality.get("reference_tunnel_violations"): + errors.append("arrow_quality_report.json contains reference_tunnel_violations") + if "total_crossing_count" not in arrow_quality: + errors.append("arrow_quality_report.json missing total_crossing_count") + if "average_aesthetic_score" not in arrow_quality: + errors.append("arrow_quality_report.json missing average_aesthetic_score") + except Exception as exc: + errors.append(f"Invalid arrow_quality_report.json: {exc}") + + try: + style_profile = json.loads((root / "reference_style_profile.json").read_text(encoding="utf-8")) + if not isinstance(style_profile.get("color_tokens"), list) or not style_profile.get("color_tokens"): + errors.append("reference_style_profile.json must contain color_tokens") + for key in ["style_summary", "illustration_style", "line_weight", "shadow_style", "corner_radius", "icon_detail_level", "visual_density", "text_policy", "reference_priority"]: + if not str(style_profile.get(key, "")).strip(): + errors.append(f"reference_style_profile.json missing {key}") + except Exception as exc: + errors.append(f"Invalid reference_style_profile.json: {exc}") + + try: + layout_plan = json.loads((root / "layout_plan.json").read_text(encoding="utf-8")) + if not isinstance(layout_plan.get("slots"), list) or len(layout_plan.get("slots", [])) < 25: + errors.append("layout_plan.json must contain at least 25 located slots") + if not isinstance(layout_plan.get("panels"), list) or not layout_plan.get("panels"): + errors.append("layout_plan.json must contain located panels") + except Exception as exc: + errors.append(f"Invalid layout_plan.json: {exc}") + + try: + program = json.loads((root / "figure_program.json").read_text(encoding="utf-8")) + for key in ["canvas", "panels", "slots", "assets", "labels", "arrows", "groups", "export_targets"]: + if key not in program: + errors.append(f"figure_program.json missing {key}") + slots = program.get("slots", []) + if len(slots) < 25: + errors.append(f"Too few slots: {len(slots)}") + if not any(str(t.get("type", "")).lower() == "pptx" or str(t.get("path", "")).lower().endswith(".pptx") for t in program.get("export_targets", []) if isinstance(t, dict)): + errors.append("figure_program.json export_targets must include pptx") + for slot in slots: + if _arrow_like_asset_id(slot.get("id")) or _arrow_like_asset_id(slot.get("asset_id")): + errors.append(f"Slot {slot.get('id')} looks like an arrow/control and must not be generated as an image asset") + fit_policy = str(slot.get("fit_policy", "")).lower() + if "crop" in fit_policy and "no_crop" not in fit_policy and "without_crop" not in fit_policy: + errors.append(f"Slot {slot.get('id')} uses a crop fit policy") + if not _is_precise_ratio(slot.get("target_canvas_ratio")): + errors.append(f"Slot {slot.get('id')} uses a coarse or non-decimal target_canvas_ratio") + for key in ["center_percent", "width_percent", "height_percent", "aspect_ratio_decimal", "aspect_ratio_w_h", "target_pixels", "target_pixels_exact", "generation_min_pixels"]: + if key not in slot: + errors.append(f"Slot {slot.get('id')} missing precise geometry key {key}") + target_pixels = _pixel_pair(slot.get("target_pixels")) + exact_pixels = _pixel_pair(slot.get("target_pixels_exact")) + if target_pixels and exact_pixels: + if abs(target_pixels[0] - exact_pixels[0]) > 0.01 or abs(target_pixels[1] - exact_pixels[1]) > 0.01: + errors.append(f"Slot {slot.get('id')} target_pixels must equal target_pixels_exact; use generation_min_pixels for minimum generation size") + if float(slot.get("min_content_fill_percent", 0)) < 85: + errors.append(f"Slot {slot.get('id')} has min fill below 85") + if float(slot.get("max_empty_margin_percent", 99)) > 10: + errors.append(f"Slot {slot.get('id')} allows too much empty margin") + if str(slot.get("slot_frame_policy", "")).lower() != "frameless_slot": + errors.append(f"Slot {slot.get('id')} must use frameless_slot") + crop_path = str(slot.get("reference_crop_path", "")).strip() + if not crop_path: + errors.append(f"Slot {slot.get('id')} missing reference_crop_path") + elif not (root / crop_path).exists(): + errors.append(f"Slot {slot.get('id')} reference crop does not exist: {crop_path}") + if not str(slot.get("reference_style_profile_path", "")).strip(): + errors.append(f"Slot {slot.get('id')} missing reference_style_profile_path") + if not isinstance(slot.get("local_color_token_ids"), list) or not slot.get("local_color_token_ids"): + errors.append(f"Slot {slot.get('id')} missing local_color_token_ids") + for key in ["visual_spec_id", "complexity_profile", "complexity_kind", "foreground_subject", "secondary_objects", "micro_details", "background_fill_elements", "scientific_mechanism_detail", "required_visual_complexity", "forbidden_simplification"]: + if key not in slot: + errors.append(f"Slot {slot.get('id')} missing visual complexity key {key}") + if str(slot.get("complexity_kind", "")).lower() != "legend_icon": + if not isinstance(slot.get("secondary_objects"), list) or len(slot.get("secondary_objects", [])) < 2: + errors.append(f"Slot {slot.get('id')} needs at least 2 secondary_objects") + if not isinstance(slot.get("micro_details"), list) or len(slot.get("micro_details", [])) < 2: + errors.append(f"Slot {slot.get('id')} needs at least 2 micro_details") + safe_area = float(slot.get("safe_area_percent", 0)) + if safe_area < 88: + errors.append(f"Slot {slot.get('id')} safe area below 88") + for asset in program.get("assets", []): + if _arrow_like_asset_id(asset.get("id")) or _arrow_like_asset_id(asset.get("slot_id")): + errors.append(f"Asset {asset.get('id')} looks like an arrow/control and must not be generated by image2") + if not str(asset.get("reference_crop_path", "")).strip(): + errors.append(f"Asset {asset.get('id')} missing reference_crop_path") + if not str(asset.get("visual_spec_id", "")).strip(): + errors.append(f"Asset {asset.get('id')} missing visual_spec_id") + for key in ["panels", "labels", "arrows", "groups"]: + for item in program.get(key, []): + if isinstance(item, dict) and str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"{key} item {item.get('id')} is not marked editable in PPTX") + for arrow in program.get("arrows", []): + if not str(arrow.get("source_id") or arrow.get("source") or "").strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing source_id") + if not str(arrow.get("target_id") or arrow.get("target") or "").strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing target_id") + if not isinstance(arrow.get("path_percent"), list) or len(arrow.get("path_percent", [])) < 2: + errors.append(f"Arrow/control {arrow.get('id')} must have at least 2 path_percent points") + if not str(arrow.get("source_anchor", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing source_anchor") + if not str(arrow.get("target_anchor", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing target_anchor") + if not str(arrow.get("style_token_id", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing style_token_id") + if str(arrow.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"Arrow/control {arrow.get('id')} must render as PPT shape, not image asset") + if not str(arrow.get("semantic_role", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing semantic_role") + if not str(arrow.get("route_style", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing route_style") + if not str(arrow.get("routing_algorithm", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing routing_algorithm") + if not str(arrow.get("route_generation_status", "")).strip(): + errors.append(f"Arrow/control {arrow.get('id')} missing route_generation_status") + if arrow.get("reference_locked") and arrow.get("reference_path_preserved") is not True: + errors.append(f"Arrow/control {arrow.get('id')} overrides a locked reference path") + style = program.get("style", {}) if isinstance(program.get("style"), dict) else {} + if not isinstance(style.get("color_tokens"), list) or not style.get("color_tokens"): + errors.append("figure_program.json style must include color_tokens") + if not str(style.get("reference_style_profile_path", "")).strip(): + errors.append("figure_program.json style must include reference_style_profile_path") + if not str(style.get("arrow_style_profile_path", "")).strip(): + errors.append("figure_program.json style must include arrow_style_profile_path") + if not str(program.get("text_program_path", "")).strip(): + errors.append("figure_program.json must include text_program_path") + if not str(program.get("reference_text_geometry_path", "")).strip(): + errors.append("figure_program.json must include reference_text_geometry_path") + if not str(program.get("text_alignment_report_path", "")).strip(): + errors.append("figure_program.json must include text_alignment_report_path") + text_program_inline = program.get("text_program") + if not isinstance(text_program_inline, dict) or not isinstance(text_program_inline.get("items"), list) or not text_program_inline.get("items"): + errors.append("figure_program.json must embed non-empty text_program items for PPT text rendering") + palette = style.get("reference_palette") or style.get("palette") or [] + distinct = {str(c).upper() for c in palette if str(c).strip()} + if len(distinct) < 4: + errors.append("figure_program.json style must preserve reference-derived palette tokens, not a hardcoded fallback template") + token_ids = {str(item.get("token_id")) for item in style.get("color_tokens", []) if isinstance(item, dict)} + for arrow in program.get("arrows", []): + if str(arrow.get("style_token_id")) not in token_ids: + errors.append(f"Arrow/control {arrow.get('id')} style_token_id is not present in reference color_tokens") + except Exception as exc: + errors.append(f"Invalid figure_program.json: {exc}") + + reference_text_ids: set[str] = set() + try: + text_geometry = json.loads((root / "reference_text_geometry.json").read_text(encoding="utf-8")) + if "publication" in str(text_geometry.get("policy", "")).lower() and "reference" not in str(text_geometry.get("policy", "")).lower(): + errors.append("reference_text_geometry.json policy must be reference-first, not publication-scale-first") + regions = text_geometry.get("text_regions", []) + if not isinstance(regions, list) or not regions: + errors.append("reference_text_geometry.json must contain non-empty text_regions") + for item in regions if isinstance(regions, list) else []: + rid = str(item.get("id") or "") + reference_text_ids.add(rid) + for key in ["bbox_percent", "center_percent", "width_percent", "height_percent", "estimated_font_ratio", "color_hex", "role", "target_id"]: + if key not in item: + errors.append(f"reference_text_geometry item {rid} missing {key}") + try: + if float(item.get("width_percent", 0)) <= 0 or float(item.get("height_percent", 0)) <= 0: + errors.append(f"reference_text_geometry item {rid} has non-positive size") + if float(item.get("estimated_font_ratio", 0)) <= 0: + errors.append(f"reference_text_geometry item {rid} has non-positive estimated_font_ratio") + except Exception: + errors.append(f"reference_text_geometry item {rid} size/font fields must be numeric") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"reference_text_geometry item {rid} must be intended for editable PPTX text") + color = str(item.get("color_hex", "")) + if not color.startswith("#") or len(color.strip()) < 7: + errors.append(f"reference_text_geometry item {rid} must record reference color_hex") + if str(text_geometry.get("detection_mode", "")).lower() == "ocr": + if "confidence" not in item: + errors.append(f"reference_text_geometry OCR item {rid} missing confidence") + if not str(item.get("raw_text", "")).strip(): + errors.append(f"reference_text_geometry OCR item {rid} missing raw_text") + except Exception as exc: + errors.append(f"Invalid reference_text_geometry.json: {exc}") + + try: + ocr_report = json.loads((root / "ocr_text_quality_report.json").read_text(encoding="utf-8")) + if not str(ocr_report.get("mode", "")).strip(): + errors.append("ocr_text_quality_report.json missing mode") + if not str(ocr_report.get("status", "")).strip(): + errors.append("ocr_text_quality_report.json missing status") + if str(ocr_report.get("status", "")).lower() not in {"pass", "fallback", "not_run"}: + errors.append("ocr_text_quality_report.json status must be pass, fallback, or not_run") + if "text_region_count" not in ocr_report: + errors.append("ocr_text_quality_report.json missing text_region_count") + except Exception as exc: + errors.append(f"Invalid ocr_text_quality_report.json: {exc}") + + try: + text_program = json.loads((root / "text_program.json").read_text(encoding="utf-8")) + policy = str(text_program.get("policy", "")).lower() + if "paper_double_column" in policy or "minimum_effective_font" in policy: + errors.append("text_program.json must not apply fixed publication-scale font thresholds by default") + items = text_program.get("items", []) + if not isinstance(items, list) or not items: + errors.append("text_program.json must contain non-empty items") + for item in items if isinstance(items, list) else []: + tid = str(item.get("id") or "") + for key in ["text", "role", "target_id", "source_reference_text_id", "reference_binding", "bbox_percent", "center_percent", "width_percent", "height_percent", "estimated_font_ratio", "font_size_pt", "color_hex", "editable_in"]: + if key not in item: + errors.append(f"text_program item {tid} missing {key}") + for key in ["font_family_guess", "font_weight_guess", "fit_strategy"]: + if key not in item: + errors.append(f"text_program item {tid} missing {key}") + source_id = str(item.get("source_reference_text_id") or "") + if source_id not in reference_text_ids: + errors.append(f"text_program item {tid} source_reference_text_id does not exist in reference_text_geometry.json") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"text_program item {tid} must render as editable PPTX text") + try: + if float(item.get("font_size_pt", 0)) <= 0: + errors.append(f"text_program item {tid} font_size_pt must be positive") + if float(item.get("estimated_font_ratio", 0)) <= 0: + errors.append(f"text_program item {tid} estimated_font_ratio must be positive") + except Exception: + errors.append(f"text_program item {tid} font fields must be numeric") + binding = str(item.get("reference_binding", "")).lower() + if "reference" not in binding: + errors.append(f"text_program item {tid} must bind to a reference-derived text region") + except Exception as exc: + errors.append(f"Invalid text_program.json: {exc}") + + try: + text_alignment = json.loads((root / "text_alignment_report.json").read_text(encoding="utf-8")) + policy = str(text_alignment.get("policy", "")).lower() + if "fixed publication" in policy or "paper_double_column" in policy: + errors.append("text_alignment_report.json must check reference alignment, not fixed publication font thresholds") + if str(text_alignment.get("status", "")).lower() in {"fail", "failed", "blocked"}: + errors.append("text_alignment_report.json reports failed text alignment") + items = text_alignment.get("items", []) + if not isinstance(items, list) or not items: + errors.append("text_alignment_report.json must contain non-empty items") + for item in items if isinstance(items, list) else []: + tid = str(item.get("label_id") or "") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"text_alignment_report item {tid} is not editable in PPTX") + if str(item.get("status", "")).lower() in {"fail", "failed", "blocked"}: + errors.append(f"text_alignment_report item {tid} failed reference alignment") + try: + if float(item.get("center_delta_percent", 0)) > 0.05: + errors.append(f"text_alignment_report item {tid} center drift exceeds reference tolerance") + if float(item.get("width_delta_percent", 0)) > 0.08: + errors.append(f"text_alignment_report item {tid} width drift exceeds reference tolerance") + if float(item.get("height_delta_percent", 0)) > 0.06: + errors.append(f"text_alignment_report item {tid} height drift exceeds reference tolerance") + except Exception: + errors.append(f"text_alignment_report item {tid} delta fields must be numeric") + if not str(item.get("source_reference_text_id", "")).strip(): + errors.append(f"text_alignment_report item {tid} missing source_reference_text_id") + except Exception as exc: + errors.append(f"Invalid text_alignment_report.json: {exc}") + + try: + visual_spec = json.loads((root / "slot_visual_spec.json").read_text(encoding="utf-8")) + items = visual_spec.get("slots", []) + if not isinstance(items, list) or len(items) < 25: + errors.append("slot_visual_spec.json must contain at least 25 slot entries") + for item in items: + sid = item.get("slot_id") + for key in ["reference_crop_objects", "foreground_subject", "secondary_objects", "micro_details", "background_fill_elements", "scientific_mechanism_detail", "required_visual_complexity", "forbidden_simplification"]: + if key not in item: + errors.append(f"slot_visual_spec item {sid} missing {key}") + if str(item.get("complexity_kind", "")).lower() != "legend_icon": + if not isinstance(item.get("secondary_objects"), list) or len(item.get("secondary_objects", [])) < 2: + errors.append(f"slot_visual_spec item {sid} needs at least 2 secondary_objects") + if not isinstance(item.get("micro_details"), list) or len(item.get("micro_details", [])) < 2: + errors.append(f"slot_visual_spec item {sid} needs at least 2 micro_details") + forbidden_items = item.get("forbidden_simplification", []) if isinstance(item.get("forbidden_simplification"), list) else [] + forbidden = " ".join(str(x).lower() for x in forbidden_items) + if str(item.get("complexity_kind", "")).lower() != "legend_icon" and "simple icon" not in forbidden: + errors.append(f"slot_visual_spec item {sid} must forbid simple icon simplification") + except Exception as exc: + errors.append(f"Invalid slot_visual_spec.json: {exc}") + + try: + prompt_brief = json.loads((root / "reference_slot_prompt_brief.json").read_text(encoding="utf-8")) + brief_items = prompt_brief.get("slots", []) + if not isinstance(brief_items, list) or len(brief_items) < 25: + errors.append("reference_slot_prompt_brief.json must contain at least 25 slot brief entries") + for item in brief_items: + sid = item.get("slot_id") + if not str(item.get("slot_function", "")).strip(): + errors.append(f"reference_slot_prompt_brief item {sid} missing slot_function") + crop_path = str(item.get("reference_crop_path", "")).strip() + if not crop_path: + errors.append(f"reference_slot_prompt_brief item {sid} missing reference_crop_path") + elif not (root / crop_path).exists(): + errors.append(f"reference_slot_prompt_brief item {sid} crop file missing: {crop_path}") + if not str(item.get("reference_style_profile_path", "")).strip(): + errors.append(f"reference_slot_prompt_brief item {sid} missing reference_style_profile_path") + except Exception as exc: + errors.append(f"Invalid reference_slot_prompt_brief.json: {exc}") + + try: + prompt_plan = json.loads((root / "slot_prompt_plan.json").read_text(encoding="utf-8")) + items = prompt_plan.get("slots", []) + if not isinstance(items, list) or len(items) < 25: + errors.append("slot_prompt_plan.json must contain at least 25 slot prompt plans") + for item in items: + sid = item.get("slot_id") + for key in ["slot_function", "reference_slot_role", "reference_shape_language", "reference_local_style", "reference_prompt_hint", "visual_metaphor", "image_prompt_core"]: + if not str(item.get(key, "")).strip(): + errors.append(f"slot_prompt_plan item {sid} missing {key}") + for key in ["center_percent", "width_percent", "height_percent", "aspect_ratio_decimal", "aspect_ratio_w_h", "target_canvas_ratio", "target_pixels", "target_pixels_exact", "generation_min_pixels"]: + if key not in item: + errors.append(f"slot_prompt_plan item {sid} missing precise geometry key {key}") + crop_path = str(item.get("reference_crop_path", "")).strip() + if not crop_path: + errors.append(f"slot_prompt_plan item {sid} missing reference_crop_path") + elif not (root / crop_path).exists(): + errors.append(f"slot_prompt_plan item {sid} crop file missing: {crop_path}") + if not item.get("reference_crop_used"): + errors.append(f"slot_prompt_plan item {sid} must record reference_crop_used=true") + if not str(item.get("reference_style_profile_path", "")).strip(): + errors.append(f"slot_prompt_plan item {sid} missing reference_style_profile_path") + if not isinstance(item.get("local_color_token_ids"), list) or not item.get("local_color_token_ids"): + errors.append(f"slot_prompt_plan item {sid} missing local_color_token_ids") + for key in ["visual_spec_id", "complexity_kind", "foreground_subject", "secondary_objects", "micro_details", "background_fill_elements", "scientific_mechanism_detail", "required_visual_complexity", "forbidden_simplification"]: + if key not in item: + errors.append(f"slot_prompt_plan item {sid} missing visual complexity key {key}") + if not _is_precise_ratio(item.get("target_canvas_ratio")): + errors.append(f"slot_prompt_plan item {sid} uses coarse target_canvas_ratio") + core = str(item.get("image_prompt_core", "")).lower() + if "90-97" not in core or "10%" not in core: + errors.append(f"slot_prompt_plan item {sid} image_prompt_core must mention 90-97% fill and 10% margin") + if "reference_style_profile" not in core or "crop" not in core: + errors.append(f"slot_prompt_plan item {sid} image_prompt_core must mention reference_style_profile and local crop grounding") + if str(item.get("complexity_kind", "")).lower() != "legend_icon": + for phrase in ["dense mini scientific", "2-5", "standalone pictogram"]: + if phrase not in core: + errors.append(f"slot_prompt_plan item {sid} image_prompt_core must mention {phrase}") + if not isinstance(item.get("must_show"), list) or not item.get("must_show"): + errors.append(f"slot_prompt_plan item {sid} missing must_show") + except Exception as exc: + errors.append(f"Invalid slot_prompt_plan.json: {exc}") + + try: + report = json.loads((root / "asset_quality_report.json").read_text(encoding="utf-8")) + items = report.get("assets", []) + if len(items) < 25: + errors.append(f"Too few asset QA entries: {len(items)}") + for item in items: + aid = item.get("asset_id") or item.get("slot_id") + if "selected_candidate_index" not in item: + errors.append(f"Asset {aid} missing selected_candidate_index") + fill = float(item.get("content_fill_percent", 0)) + min_fill = float(item.get("min_content_fill_percent", 85)) + margin = float(item.get("empty_margin_percent", 100)) + max_margin = float(item.get("max_empty_margin_percent", 10)) + if fill < min_fill: + errors.append(f"Asset {aid} fill {fill} below {min_fill}") + if margin > max_margin: + errors.append(f"Asset {aid} margin {margin} above {max_margin}") + if str(item.get("edge_cutoff_status", "")).lower() not in {"ok", "none", "complete", "no_cutoff"}: + errors.append(f"Asset {aid} edge cutoff status is not ok") + except Exception as exc: + errors.append(f"Invalid asset_quality_report.json: {exc}") + + try: + complexity = json.loads((root / "asset_complexity_report.json").read_text(encoding="utf-8")) + items = complexity.get("assets", []) + if len(items) < 25: + errors.append(f"Too few asset complexity entries: {len(items)}") + for item in items: + aid = item.get("asset_id") or item.get("slot_id") + for key in ["detail_score", "detail_score_target", "object_count_estimate", "object_count_target", "simple_icon_risk", "reference_crop_match", "style_match", "selected_reason"]: + if key not in item: + errors.append(f"Asset complexity {aid} missing {key}") + try: + if float(item.get("detail_score", 0)) < float(item.get("detail_score_target", 0)): + errors.append(f"Asset complexity {aid} detail_score below target") + except Exception: + errors.append(f"Asset complexity {aid} detail_score fields must be numeric") + if item.get("simple_icon_risk") is True: + errors.append(f"Asset complexity {aid} has simple_icon_risk=true") + tags = item.get("complexity_issue_tags", []) if isinstance(item.get("complexity_issue_tags"), list) else [] + unresolved = sorted(COMPLEXITY_BLOCKERS.intersection(str(tag) for tag in tags)) + if unresolved: + errors.append(f"Asset complexity {aid} has unresolved complexity issue(s): {', '.join(unresolved)}") + if str(item.get("reference_crop_match", "")).startswith("missing"): + errors.append(f"Asset complexity {aid} missing reference crop match") + if str(item.get("style_match", "")).startswith("missing"): + errors.append(f"Asset complexity {aid} missing style match") + except Exception as exc: + errors.append(f"Invalid asset_complexity_report.json: {exc}") + + try: + comp = json.loads((root / "composition_quality_report.json").read_text(encoding="utf-8")) + items = comp.get("slots", []) + if not isinstance(items, list) or len(items) < 25: + errors.append("composition_quality_report.json must contain at least 25 slot entries") + for item in items: + sid = item.get("slot_id") + if item.get("tile_frame_added") is not False: + errors.append(f"Composition slot {sid} added an extra tile frame") + if item.get("caption_inside_image_slot") is not False: + errors.append(f"Composition slot {sid} has caption inside image slot") + if str(item.get("slot_frame_policy", "")).lower() != "frameless_slot": + errors.append(f"Composition slot {sid} must use frameless_slot") + fill = float(item.get("image_slot_area_fill_percent", 0)) + if fill < 95: + errors.append(f"Composition slot {sid} image fills {fill}% of slot, below 95%") + for item in comp.get("arrows", []) if isinstance(comp.get("arrows"), list) else []: + aid = item.get("arrow_id") + if str(item.get("editable_in", "")).lower() != "pptx": + errors.append(f"Composition arrow {aid} is not editable in PPTX") + if str(item.get("render_policy", "")).lower() != "ppt_shape_not_image_asset": + errors.append(f"Composition arrow {aid} must render as a PPT shape") + if int(item.get("segment_count", 0)) < 1: + errors.append(f"Composition arrow {aid} rendered no connector segments") + if not str(item.get("route_style", "")).strip(): + errors.append(f"Composition arrow {aid} missing route_style") + if not str(item.get("line_cap", "")).strip(): + errors.append(f"Composition arrow {aid} missing line_cap") + if not str(item.get("routing_algorithm", "")).strip(): + errors.append(f"Composition arrow {aid} missing routing_algorithm") + except Exception as exc: + errors.append(f"Invalid composition_quality_report.json: {exc}") + + try: + asset_review = json.loads((root / "asset_visual_review.json").read_text(encoding="utf-8")) + if str(asset_review.get("status", "")).lower() in {"needs_regeneration", "blocked", "fail", "failed"}: + issues = asset_review.get("issues", []) + errors.append(f"asset_visual_review.json has unresolved asset issues: {len(issues)}") + except Exception as exc: + errors.append(f"Invalid asset_visual_review.json: {exc}") + + try: + visual_critic = json.loads((root / "visual_critic_iter_0.json").read_text(encoding="utf-8")) + if str(visual_critic.get("status", "")).lower() in {"blocked", "needs_layout_refinement", "fail", "failed"}: + errors.append("visual_critic_iter_0.json reports unresolved layout or blocking issues") + if visual_critic.get("blocking_issues"): + errors.append(f"visual_critic_iter_0.json has blocking issues: {len(visual_critic.get('blocking_issues', []))}") + except Exception as exc: + errors.append(f"Invalid visual_critic_iter_0.json: {exc}") + + asset_count = len([p for p in (root / "assets").glob("*") if p.suffix.lower() in IMAGE_EXTS]) if (root / "assets").exists() else 0 + if asset_count < 25: + errors.append(f"Too few generated assets: {asset_count}") + + text_blob = "" + for name in ["prompts.md", "style_sheet.md", "reference_style_profile.json", "arrow_style_profile.json", "selected_arrow_routes.json", "arrow_quality_report.json", "figure_program.json", "reference_geometry.json", "reference_controls.json", "reference_text_geometry.json", "text_program.json", "ocr_text_quality_report.json", "text_alignment_report.json", "slot_visual_spec.json", "reference_slot_prompt_brief.json", "slot_prompt_plan.json", "asset_quality_report.json", "asset_complexity_report.json", "composition_quality_report.json", "asset_visual_review.json", "alignment_review.md", "critic_report.md", "visual_critic_iter_0.json"]: + path = root / name + if path.exists(): + text_blob += "\n" + path.read_text(encoding="utf-8", errors="ignore").lower() + hits = [m for m in FORBIDDEN_MARKERS if m in text_blob] + if hits: + errors.append("Forbidden marker(s) found: " + ", ".join(hits)) + + if not (root / "review.pdf").exists(): + warnings.append("review.pdf missing; PPTX may still be editable but export failed") + if not (root / "final_600dpi.png").exists(): + warnings.append("final_600dpi.png missing; PDF-to-PNG export may have failed") + + return {"summary": "Framework output validation result.", "ok": not errors, "errors": errors, "warnings": warnings, "asset_count": asset_count} diff --git a/rfs/evolution_contracts.py b/rfs/evolution_contracts.py new file mode 100644 index 0000000..f36f5c8 --- /dev/null +++ b/rfs/evolution_contracts.py @@ -0,0 +1,162 @@ +"""Cross-stage contracts for the Paperimage evolution pipeline. + +This module intentionally contains deterministic, inspectable checks only. A +trained Judge may later add scores and advice, but cannot replace the semantic +grounding gate maintained here. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .utils import write_json + + +def _items(value: object) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def _spec(contract: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(contract, dict): + return {} + value = contract.get("figure_specification") + return value if isinstance(value, dict) else contract + + +def _entities(contract: dict[str, Any] | None) -> list[dict[str, Any]]: + spec = _spec(contract) + direct = _items(spec.get("entities")) or _items(spec.get("modules")) + if direct: + return direct + entities: list[dict[str, Any]] = [] + for panel in _items(spec.get("panel_graphs")): + entities.extend(_items(panel.get("entities")) or _items(panel.get("modules"))) + return entities + + +def _relations(contract: dict[str, Any] | None) -> list[dict[str, Any]]: + spec = _spec(contract) + direct = _items(spec.get("relations")) + if direct: + return direct + relations: list[dict[str, Any]] = [] + for panel in _items(spec.get("panel_graphs")): + relations.extend(_items(panel.get("relations"))) + return relations + + +def build_scientific_ground_truth(contract: dict[str, Any] | None) -> dict[str, Any]: + """Normalize an existing paper contract into a Gate-friendly snapshot.""" + entities = _entities(contract) + relations = _relations(contract) + must_show = [] + normalized_entities = [] + for index, entity in enumerate(entities, start=1): + entity_id = str(entity.get("id") or f"entity_{index:02d}") + label = str(entity.get("label") or entity.get("name") or entity_id) + required = bool(entity.get("must_show", True)) + if required: + must_show.append(entity_id) + normalized_entities.append({ + "id": entity_id, + "label": label, + "must_show": required, + "evidence_ids": list(entity.get("evidence_ids") or []), + }) + return { + "summary": "Normalized scientific Ground Truth derived from the paper semantic contract.", + "schema_version": "1.0", + "entities": normalized_entities, + "relations": [{ + "id": str(item.get("id") or f"relation_{index:02d}"), + "source": str(item.get("source") or item.get("source_id") or ""), + "target": str(item.get("target") or item.get("target_id") or ""), + "type": str(item.get("type") or item.get("relation_type") or "data_flow"), + "evidence_ids": list(item.get("evidence_ids") or []), + } for index, item in enumerate(relations, start=1)], + "must_show": must_show, + "must_not_invent": list(_spec(contract).get("must_not_invent") or []), + "terminology": list(_spec(contract).get("terminology") or []), + "source": "paper_semantic_contract" if contract else "reference_only", + } + + +def build_scientific_grounding_report( + ground_truth: dict[str, Any], semantic_binding: dict[str, Any] | None, +) -> dict[str, Any]: + """Apply a deterministic semantic gate to the PPT semantic bindings.""" + if ground_truth.get("source") == "reference_only": + return { + "summary": "Scientific grounding skipped: reference-only rebuild has no paper contract.", + "status": "warning", + "gate_pass": True, + "judge_role": "bootstrap", + "trained": False, + "blocking_issues": [], + "missing_required_entities": [], + "incorrect_relations": [], + } + binding = semantic_binding if isinstance(semantic_binding, dict) else {} + mapped_entities = int(binding.get("mapped_entity_count") or 0) + entities = ground_truth.get("entities") or [] + skipped = binding.get("skipped_relations") or [] + missing = [] + if mapped_entities < len(entities): + mapped_ids = {str(item.get("entity_id") or item.get("semantic_entity_id") or "") for item in binding.get("mappings", []) if isinstance(item, dict) and float(item.get("match_score") or 0) > 0} + missing = [str(item.get("id")) for item in entities if str(item.get("id")) not in mapped_ids] + blockers = [] + if missing: + blockers.append("missing_required_entities") + if skipped: + blockers.append("unmapped_required_relations") + return { + "summary": "Deterministic scientific grounding gate for editable rebuild.", + "status": "pass" if not blockers else "fail", + "gate_pass": not blockers, + "judge_role": "bootstrap", + "trained": False, + "grounded_entities": [item.get("semantic_entity_id") for item in binding.get("mappings", []) if isinstance(item, dict)], + "missing_required_entities": missing, + "incorrect_relations": skipped, + "terminology_errors": [], + "blocking_issues": blockers, + } + + +def build_joint_judgement( + grounding: dict[str, Any], segmentation_validation: dict[str, Any], visual_quality: dict[str, Any], + program: dict[str, Any], +) -> dict[str, Any]: + """Build a model-independent joint evaluation artifact for later Judge training.""" + editability = { + "text_items": len((program.get("text_program") or {}).get("items", [])) if isinstance(program.get("text_program"), dict) else 0, + "connectors": len(program.get("arrows", [])), + "slots": len(program.get("slots", [])), + } + blockers = list(grounding.get("blocking_issues") or []) + return { + "summary": "Bootstrap joint judgement for scientific preservation, segmentation, and editable rebuild quality.", + "judge_role": "bootstrap", + "trained": False, + "scientific": grounding, + "segmentation": segmentation_validation, + "visual": {"status": visual_quality.get("status"), "report": "rebuild_visual_quality_report.json"}, + "editability": editability, + "overall_status": "pass" if not blockers and segmentation_validation.get("status") == "pass" else "warning", + "blocking_issues": blockers, + } + + +def write_pipeline_contracts( + out_dir: str | Path, contract: dict[str, Any] | None, semantic_binding: dict[str, Any] | None, + segmentation_plan: dict[str, Any], segmentation_validation: dict[str, Any], visual_quality: dict[str, Any], program: dict[str, Any], +) -> dict[str, Any]: + out = Path(out_dir) + ground_truth = build_scientific_ground_truth(contract) + grounding = build_scientific_grounding_report(ground_truth, semantic_binding) + judgement = build_joint_judgement(grounding, segmentation_validation, visual_quality, program) + write_json(out / "scientific_ground_truth.json", ground_truth) + write_json(out / "scientific_grounding_report.json", grounding) + write_json(out / "joint_judgement.json", judgement) + return {"ground_truth": ground_truth, "grounding": grounding, "judgement": judgement} diff --git a/rfs/exporter.py b/rfs/exporter.py index 8b45444..f6a0ac7 100644 --- a/rfs/exporter.py +++ b/rfs/exporter.py @@ -1,27 +1,145 @@ -from __future__ import annotations +from __future__ import annotations +import multiprocessing import os from pathlib import Path +import queue +import signal +import time -def _export_pdf_with_powerpoint(pptx_path: Path, pdf_path: Path) -> bool: - if os.name != "nt": - return False +def _powerpoint_export_worker(pptx_path: str, pdf_path: str, png_path: str, dpi: int, result_queue) -> None: + powerpoint = None + presentation = None try: import win32com.client # type: ignore - powerpoint = win32com.client.Dispatch("PowerPoint.Application") - presentation = powerpoint.Presentations.Open(str(pptx_path), WithWindow=False) + import win32process # type: ignore + + powerpoint = win32com.client.DispatchEx("PowerPoint.Application") + try: + _thread_id, process_id = win32process.GetWindowThreadProcessId(int(powerpoint.HWND)) + result_queue.put({"event": "powerpoint_pid", "pid": int(process_id)}) + except Exception: + pass + presentation = powerpoint.Presentations.Open(str(pptx_path), ReadOnly=True, Untitled=False, WithWindow=False) presentation.SaveAs(str(pdf_path), 32) - presentation.Close() - powerpoint.Quit() - return pdf_path.exists() + width_px = max(1200, int(float(presentation.PageSetup.SlideWidth) / 72.0 * dpi)) + height_px = max(700, int(float(presentation.PageSetup.SlideHeight) / 72.0 * dpi)) + presentation.Slides(1).Export(str(png_path), "PNG", width_px, height_px) + result_queue.put({ + "event": "complete", + "pdf_ok": Path(pdf_path).exists(), + "png_ok": Path(png_path).exists(), + "width_px": width_px, + "height_px": height_px, + }) + except Exception as exc: + result_queue.put({"event": "error", "error": f"{type(exc).__name__}: {exc}"}) + finally: + try: + if presentation is not None: + presentation.Close() + except Exception: + pass + try: + if powerpoint is not None: + powerpoint.Quit() + except Exception: + pass + + +def _export_with_powerpoint( + pptx_path: Path, + pdf_path: Path, + png_path: Path, + *, + dpi: int = 600, + timeout_seconds: float = 60.0, +) -> dict: + if os.name != "nt": + return {"status": "unavailable", "pdf_ok": False, "png_ok": False, "error": "PowerPoint export requires Windows"} + started = time.monotonic() + context = multiprocessing.get_context("spawn") + result_queue = context.Queue() + process = context.Process( + target=_powerpoint_export_worker, + args=(str(pptx_path.resolve()), str(pdf_path.resolve()), str(png_path.resolve()), int(dpi), result_queue), + daemon=True, + ) + powerpoint_pid = None + try: + process.start() + deadline = time.monotonic() + max(5.0, float(timeout_seconds)) + terminal = None + while process.is_alive() and time.monotonic() < deadline: + try: + message = result_queue.get(timeout=0.25) + except queue.Empty: + continue + if message.get("event") == "powerpoint_pid": + powerpoint_pid = message.get("pid") + elif message.get("event") in {"complete", "error"}: + terminal = message + process.join(timeout=1.0) + while terminal is None: + try: + message = result_queue.get_nowait() + except queue.Empty: + break + if message.get("event") == "powerpoint_pid": + powerpoint_pid = message.get("pid") + elif message.get("event") in {"complete", "error"}: + terminal = message + if process.is_alive(): + process.terminate() + process.join(timeout=2.0) + if powerpoint_pid: + try: + os.kill(int(powerpoint_pid), signal.SIGTERM) + except Exception: + pass + return { + "status": "timeout", + "pdf_ok": pdf_path.exists(), + "png_ok": png_path.exists(), + "error": f"PowerPoint export exceeded {max(5.0, float(timeout_seconds)):.1f}s", + "elapsed_seconds": round(time.monotonic() - started, 3), + } + if terminal and terminal.get("event") == "complete": + return {**terminal, "status": "ok", "elapsed_seconds": round(time.monotonic() - started, 3)} + return { + "status": "failed", + "pdf_ok": pdf_path.exists(), + "png_ok": png_path.exists(), + "error": (terminal or {}).get("error") or f"PowerPoint export worker exited with code {process.exitcode}", + "elapsed_seconds": round(time.monotonic() - started, 3), + } + except Exception as exc: + if process.is_alive(): + process.terminate() + return { + "status": "failed", + "pdf_ok": pdf_path.exists(), + "png_ok": png_path.exists(), + "error": f"{type(exc).__name__}: {exc}", + "elapsed_seconds": round(time.monotonic() - started, 3), + } + + +def _export_pdf_with_powerpoint(pptx_path: Path, pdf_path: Path) -> bool: + png_path = pdf_path.with_suffix(".preview.png") + result = _export_with_powerpoint(pptx_path, pdf_path, png_path, dpi=144, timeout_seconds=60) + try: + png_path.unlink(missing_ok=True) except Exception: - return False + pass + return bool(result.get("pdf_ok")) def _render_pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 600) -> bool: try: import fitz + doc = fitz.open(str(pdf_path)) page = doc.load_page(0) scale = dpi / 72.0 @@ -39,8 +157,16 @@ def export_outputs(pptx_path: str | Path, out_dir: str | Path) -> dict: review_pdf = out / "review.pdf" final_png = out / "final_600dpi.png" - if not _export_pdf_with_powerpoint(pptx, review_pdf): - return {"pdf": None, "png": None, "status": "pptx_only_export_failed"} - - png_ok = _render_pdf_to_png(review_pdf, final_png, dpi=600) - return {"pdf": str(review_pdf) if review_pdf.exists() else None, "png": str(final_png) if png_ok else None, "status": "ok" if png_ok else "pdf_only"} + timeout_seconds = float(os.getenv("RFS_POWERPOINT_EXPORT_TIMEOUT", "60") or 60) + result = _export_with_powerpoint(pptx, review_pdf, final_png, dpi=600, timeout_seconds=timeout_seconds) + png_ok = bool(result.get("png_ok")) + if not png_ok and review_pdf.exists(): + png_ok = _render_pdf_to_png(review_pdf, final_png, dpi=600) + pdf_ok = review_pdf.exists() + status = "ok" if pdf_ok and png_ok else "pdf_only" if pdf_ok else "png_only" if png_ok else f"pptx_only_export_{result.get('status', 'failed')}" + return { + "pdf": str(review_pdf) if pdf_ok else None, + "png": str(final_png) if png_ok else None, + "status": status, + "powerpoint_export": result, + } diff --git a/rfs/generation/__init__.py b/rfs/generation/__init__.py index d3a97a5..c1b1fe2 100644 --- a/rfs/generation/__init__.py +++ b/rfs/generation/__init__.py @@ -1,6 +1,15 @@ """Raster reference and slot-asset generation entry points.""" -from ..asset_generator import generate_assets -from ..paper_to_image.generator import generate_and_select +from .assets import build_slot_prompt, generate_assets -__all__ = ["generate_and_select", "generate_assets"] +__all__ = ["build_slot_prompt", "generate_and_select", "generate_assets"] + + +def __getattr__(name: str): + """Defer paper-image imports to avoid a slot-asset provider cycle.""" + + if name == "generate_and_select": + from ..paper_to_image.generator import generate_and_select + + return generate_and_select + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/rfs/generation/assets.py b/rfs/generation/assets.py new file mode 100644 index 0000000..276137d --- /dev/null +++ b/rfs/generation/assets.py @@ -0,0 +1,974 @@ +from __future__ import annotations + +import base64 +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import math +import os +import shutil +import time +from pathlib import Path +from statistics import median +from typing import Iterable + +import requests +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +from ..utils import ensure_dir, write_json, write_text + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".tif", ".tiff"} + +CANDIDATE_DIRECTIONS = [ + "dense mechanism-first scientific mini scene with foreground, secondary objects, and micro-detail", + "reference-crop reconstruction with layered local objects and edge-to-edge support detail", + "compact paper-figure mini panel with concrete objects, internal mechanism, and background cues", + "structured process card with visible before-and-after relation plus local reference texture", + "high-density method card focused on the named operation, not a standalone pictogram", +] + +NEGATIVE_PROMPT = ( + "Avoid: large empty margin, tiny centered object, sparse white background, isolated small icon on blank canvas, " + "cropped subject, cut-off edges, fake formulas, fake axes, fake metrics, readable paragraphs, paper title text, " + "watermarks, logos, screenshots, low-resolution blur, poster style, decorative clutter without scientific meaning, " + "extra white presentation tile, white matting, large white border, generic blue-green dashboard styling, " + "simple centered icon, single object on clean blank background, standalone pictogram." +) + + +def _parse_ratio(ratio: str) -> tuple[float, float]: + try: + left, right = ratio.split(":", 1) + return max(0.001, float(left)), max(0.001, float(right)) + except Exception: + return 1.0, 1.0 + + +def _ratio_value(ratio: str) -> float: + rw, rh = _parse_ratio(ratio) + return rw / max(rh, 1) + + +def _target_size(ratio: str, max_side: int = 768, min_side: int = 96) -> tuple[int, int]: + rw, rh = _parse_ratio(ratio) + if rw >= rh: + return max_side, max(min_side, int(max_side * rh / rw)) + return max(min_side, int(max_side * rw / rh)), max_side + + +def _as_list_text(value) -> str: + if isinstance(value, list): + items = [str(item).strip() for item in value if str(item).strip()] + return "; ".join(items) + return str(value or "").strip() + + +def _fallback_visual_metaphor(slot: dict) -> str: + concept = str(slot.get("paper_concept", "scientific method component")) + panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "") + low = f"{concept} {panel}".lower() + if "sparse" in low or "nonzero" in low: + return "draw a matrix grid with many pale empty cells and a few highlighted trainable nonzero cells" + if "empty" in low or "zero" in low: + return "draw a mostly empty matrix grid with muted zero cells and clear blank positions" + if "frozen" in low or "pre-trained" in low or "pretrained" in low: + return "draw a large locked frozen weight matrix slab, with icy muted cells and no trainable highlights" + if "lora" in low: + return "draw two slim low-rank matrix factors forming a compact adapter update beside a larger frozen weight matrix" + if "ift" in low or "fourier" in low or "frequency" in low: + return "draw a frequency-domain sparse grid transforming into a smooth spatial-domain matrix through wave arcs" + if "split" in low or "submat" in low: + return "draw one matrix split into multiple smaller aligned submatrices with visible separation cuts" + if "concat" in low: + return "draw multiple transformed matrix blocks stacked vertically into one taller concatenated matrix" + if "adaptor" in low or "adapter" in low: + return "draw a frozen shared projection block receiving matrix features and emitting a compact update" + if "multiplication" in low: + return "draw two matrix blocks combining through a multiplication junction into a compact output block" + if "budget" in low or "parameter" in low: + return "draw a tiny trainable budget highlighted against a large muted frozen parameter field" + if "benchmark" in low or "evaluation" in low or "glue" in low: + return "draw a benchmark suite board with multiple task tiles and abstract bars without readable numbers" + return f"draw a concrete paper-specific visual object for {concept}, showing its role inside {panel}" + + +def build_slot_prompt(slot: dict, style: dict, candidate_index: int = 1) -> str: + palette = ", ".join(style.get("palette", [])[:5]) + direction = CANDIDATE_DIRECTIONS[(candidate_index - 1) % len(CANDIDATE_DIRECTIONS)] + composition = slot.get("composition_type", "full_frame_icon") + visual_metaphor = str(slot.get("visual_metaphor") or "").strip() or _fallback_visual_metaphor(slot) + slot_function = str(slot.get("slot_function") or "").strip() + image_prompt_core = str(slot.get("image_prompt_core") or "").strip() + reference_content_priority = str(slot.get("reference_content_priority") or "reference_primary_visual_object").strip() + reference_visual_object = str(slot.get("reference_visual_object") or "").strip() + reference_visual_elements = _as_list_text(slot.get("reference_visual_elements")) + reference_color_palette = str(slot.get("reference_color_palette") or "").strip() + paper_label_mapping = str(slot.get("paper_label_mapping") or "").strip() + must_show = _as_list_text(slot.get("must_show")) or "the concrete scientific object, its internal structure, and its input-output relation" + avoid_showing = _as_list_text(slot.get("avoid_showing")) or "generic sci-fi dashboard; unrelated robot or brain; fake formulas; fake numeric chart text" + panel = str(slot.get("macro_panel") or slot.get("parent_panel") or "paper method") + reference_slot_role = str(slot.get("reference_slot_role") or "slot in the user-provided reference layout").strip() + reference_shape_language = str(slot.get("reference_shape_language") or slot.get("target_canvas_ratio") or "").strip() + reference_local_style = str(slot.get("reference_local_style") or "match the reference figure's local card/icon discipline").strip() + reference_prompt_hint = str(slot.get("reference_prompt_hint") or "use the reference image as local layout/style guidance, not as scientific content").strip() + reference_crop_path = str(slot.get("reference_crop_path") or "").strip() + reference_style_profile_path = str(slot.get("reference_style_profile_path") or "reference_style_profile.json").strip() + local_color_token_ids = ", ".join(str(item) for item in slot.get("local_color_token_ids", []) if str(item).strip()) + complexity_kind = str(slot.get("complexity_kind") or "pipeline_module").strip() + required_visual_complexity = str(slot.get("required_visual_complexity") or "dense").strip() + foreground_subject = str(slot.get("foreground_subject") or reference_visual_object or visual_metaphor).strip() + secondary_objects = _as_list_text(slot.get("secondary_objects")) or "2-5 supporting local objects from the reference crop" + micro_details = _as_list_text(slot.get("micro_details")) or "small non-critical marks, separators, texture, and internal details" + background_fill_elements = _as_list_text(slot.get("background_fill_elements")) or "edge-to-edge local reference-colored support detail" + scientific_mechanism_detail = str(slot.get("scientific_mechanism_detail") or "show the paper concept as a visible mechanism, not a generic icon").strip() + forbidden_simplification = _as_list_text(slot.get("forbidden_simplification")) or "simple icon; centered icon; clean blank background; single object on white canvas" + is_simple_allowed = complexity_kind == "legend_icon" + + if composition == "scene_thumbnail": + type_clause = ( + "Make it a dense small mechanism thumbnail: one concrete foreground scientific object, multiple visible process cues, secondary context objects, and enough environment to show what changes. Use edge-to-edge supporting scene detail so the crop is visually full." + ) + elif composition == "full_bleed_card": + type_clause = ( + "Make it a compact dense mechanism card: the central object fills the card, with internal subparts, small supporting objects, and micro process cues that clarify this exact operation. The card surface should extend almost to the image edge, with no separate white mat." + ) + elif composition == "symbol_cutout": + type_clause = ( + "Make it a large complete symbolic operation object plus local reference-colored support detail; the operation symbol or object should be visually distinct from other slots. If it is a line icon, thicken the strokes, enlarge it to nearly touch the safe area, and add compact supporting shapes so it is not sparse." + ) + else: + type_clause = ( + "Make it a full-frame scientific mechanism object with a large distinct subject, mechanism-specific internal details, and supporting local objects. If the subject is a waveform, face, pose skeleton, document, database, badge, or legend icon, make it bold, oversized, and backed by local reference-colored detail rather than a tiny object on a blank white field." + ) + + exact_ratio = float(slot.get("aspect_ratio_decimal") or _ratio_value(slot.get("target_canvas_ratio", "1.000:1.000"))) + center = slot.get("center_percent") if isinstance(slot.get("center_percent"), dict) else {} + center_text = f"x={float(center.get('x', 0.5)):.3f}, y={float(center.get('y', 0.5)):.3f}" + width_percent = float(slot.get("width_percent", slot.get("bbox_percent", {}).get("w", 0))) + height_percent = float(slot.get("height_percent", slot.get("bbox_percent", {}).get("h", 0))) + + return ( + f"Create candidate {candidate_index} for one slot-level image block in a publication-quality AI/ML research framework figure. " + f"This block belongs to macro panel: {panel}. Slot id: {slot['id']}. Paper concept: {slot['paper_concept']}. " + f"Slot function: {slot_function or 'visualize this paper concept as one small editable figure asset'}. " + f"Model-planned image prompt core: {image_prompt_core or visual_metaphor}. " + f"Reference content priority: {reference_content_priority}. " + f"Reference visual object to recreate: {reference_visual_object or visual_metaphor}. " + f"Reference visual elements to preserve: {reference_visual_elements or must_show}. " + f"Reference local color palette to preserve: {reference_color_palette or 'inherit colors from the local reference slot, not one global palette'}. " + f"Reference style profile source: {reference_style_profile_path}. Local reference crop used by the prompt planner: {reference_crop_path or 'missing crop path - preserve described local crop anyway'}. " + f"Local reference color token ids to inherit: {local_color_token_ids or 'none recorded; use local reference colors described in prompt plan'}. " + f"Paper label mapping: {paper_label_mapping or 'paper terminology is added later as editable PPT text'}. " + f"Main visual metaphor: {visual_metaphor}. " + f"Must visibly show: {must_show}. " + f"Reference-local slot role: {reference_slot_role}. " + f"Reference shape language: {reference_shape_language}. " + f"Reference-local style: {reference_local_style}. " + f"Reference prompt hint: {reference_prompt_hint}. " + f"Slot visual complexity contract: complexity_profile={slot.get('complexity_profile', 'reference-dense')}; complexity_kind={complexity_kind}; required_visual_complexity={required_visual_complexity}. " + f"Foreground subject must be large: {foreground_subject}. " + f"Secondary objects to include: {secondary_objects}. " + f"Micro details to include: {micro_details}. " + f"Background fill elements: {background_fill_elements}. " + f"Scientific mechanism detail: {scientific_mechanism_detail}. " + f"Forbidden simplifications: {forbidden_simplification}. " + "Use the local reference slot object first; do not invent a different abstract paper-internal object when the reference shows a concrete object. " + "The goal is to redraw/recreate the reference slot's object as a clean high-resolution asset, then adapt only minor details to the paper concept. " + f"Reference geometry is mandatory: canvas aspect ratio = exact {exact_ratio:.3f}:1, target_canvas_ratio = {slot['target_canvas_ratio']}, " + f"slot center in full figure = {center_text}, slot width_percent = {width_percent:.3f}, slot height_percent = {height_percent:.3f}. " + "Generate this asset for the exact reference slot shape, not a generic preset ratio such as 1:1, 4:3, 3:4, 16:9, or 9:16. " + "No semantic cropping: the whole subject must be visible, no object extending outside the frame, no cut-off hands/edges/bottom. " + "Safe area means important subject remains complete inside the frame, but supporting background details should extend close to edges. " + "Useful visual content should fill 90-97% of the canvas; minimum 85%; maximum empty margin is strictly below 10% on every edge. " + "Use a full-frame composition, large subject, maximal detail density, rich scientific visual texture, minimal blank canvas, no tiny centered object. " + + ("" if is_simple_allowed else "This must be a dense mini scientific scene/card with 2-5 layered objects, edge-to-edge supporting detail, and not a standalone pictogram. ") + + "The subject plus its local reference-colored support surface should occupy almost the entire frame; avoid isolated cutout icons floating on white or transparent-looking empty space. " + "For thin symbols such as pose skeletons, waveforms, clocks, microphones, documents, badges, and legend icons, use thick readable strokes, enlarged scale, and compact supporting shapes to raise visual fill while keeping the subject complete. " + f"Candidate visual direction: {direction}. {type_clause} " + f"Style constraints: polished academic paper illustration, crisp edges, consistent line quality, clean high-resolution vector-illustration / 2.5D hybrid only when useful. " + f"Global palette may include {palette}, but local reference colors override the global palette. Preserve color variety across slots. " + "Style consistency matters, but visual identity must vary by the reference crop: monitors look like monitors, video strips look like video strips, microphones look like microphones, waveforms look like waveforms, face cards look like face cards, pose skeletons look like pose skeletons, clocks look like clocks, clipboards look like clipboards, databases look like databases. " + "Allow only very small decorative non-critical UI marks or blurred pseudo-labels. " + "Critical scientific labels, formulas, variables, panel IDs, arrow labels, and metric values will be added later in PowerPoint and must not be generated in the image. " + f"Do not show: {avoid_showing}. " + "Do not add a white presentation tile, white mat, extra card background, or artificial border unless that exact border exists in the local reference crop. " + f"{NEGATIVE_PROMPT}" + ) + + +def _draw_placeholder(slot: dict, path: Path, style: dict, candidate_index: int = 1) -> None: + width, height = _target_size(slot["target_canvas_ratio"], max_side=512) + palette = style.get("palette", ["#EAF5FF", "#DFF7EF", "#8EC9E8", "#66C6A4", "#163B4D"]) + bg = palette[(candidate_index - 1) % 2] + line = palette[4] if len(palette) > 4 else "#163B4D" + accent = palette[2 + ((candidate_index + len(slot["id"])) % max(1, min(2, len(palette) - 2)))] if len(palette) >= 4 else "#8EC9E8" + img = Image.new("RGB", (width, height), bg) + draw = ImageDraw.Draw(img) + + margin = max(8, int(min(width, height) * 0.035)) + draw.rounded_rectangle([margin, margin, width - margin, height - margin], radius=max(14, margin * 2), fill=bg, outline=line, width=max(3, margin // 4)) + + # Dense edge-to-edge detail so placeholder QA exercises high-fill layouts. + for i in range(8 + candidate_index): + x = int(width * (0.035 + i * 0.12)) % width + draw.line([x, margin, min(width - margin, x + int(width * 0.22)), height - margin], fill=accent, width=max(1, margin // 6)) + for i in range(6): + y = int(height * (0.10 + i * 0.15)) + draw.arc([margin + i * 5, y, width - margin - i * 4, y + int(height * 0.26)], 180, 360, fill=accent, width=max(1, margin // 7)) + + cx, cy = width // 2, height // 2 + main_w = int(width * (0.72 + 0.03 * ((candidate_index - 1) % 3))) + main_h = int(height * (0.64 + 0.03 * ((candidate_index - 1) % 2))) + kind = slot.get("composition_type", "full_frame_icon") + if kind in {"scene_thumbnail", "full_bleed_card"}: + draw.rounded_rectangle([cx - main_w // 2, cy - main_h // 2, cx + main_w // 2, cy + main_h // 2], radius=max(12, margin), fill="#FFFFFF", outline=line, width=max(3, margin // 4)) + panel_gap = max(4, margin // 3) + for row in range(2): + for col in range(2): + px0 = cx - main_w // 2 + margin + col * ((main_w - margin * 2) // 2) + py0 = cy - main_h // 2 + margin + row * ((main_h - margin * 2) // 2) + px1 = px0 + (main_w - margin * 2) // 2 - panel_gap + py1 = py0 + (main_h - margin * 2) // 2 - panel_gap + draw.rounded_rectangle([px0, py0, px1, py1], radius=max(6, margin // 2), fill=accent if (row + col + candidate_index) % 2 == 0 else "#DFF7EF", outline=line, width=max(1, margin // 8)) + draw.polygon([(px0 + panel_gap, py1 - panel_gap), (px0 + (px1 - px0) // 2, py0 + panel_gap), (px1 - panel_gap, py1 - panel_gap)], fill="#9AD7C4") + else: + radius = min(main_w, main_h) // 2 + draw.ellipse([cx - radius, cy - radius, cx + radius, cy + radius], fill="#FFFFFF", outline=line, width=max(4, margin // 3)) + for angle_idx in range(12): + ang = angle_idx * math.pi / 6 + candidate_index * 0.08 + x1 = cx + int(math.cos(ang) * radius * 0.18) + y1 = cy + int(math.sin(ang) * radius * 0.18) + x2 = cx + int(math.cos(ang) * radius * 0.82) + y2 = cy + int(math.sin(ang) * radius * 0.82) + draw.line([x1, y1, x2, y2], fill=line, width=max(3, margin // 4)) + draw.ellipse([x2 - margin // 2, y2 - margin // 2, x2 + margin // 2, y2 + margin // 2], fill=accent, outline=line) + inner = int(radius * 0.36) + draw.rounded_rectangle([cx - inner, cy - inner, cx + inner, cy + inner], radius=max(8, margin // 2), fill=accent, outline=line, width=max(2, margin // 6)) + + try: + font = ImageFont.truetype("arial.ttf", max(9, int(min(width, height) * 0.045))) + except Exception: + font = ImageFont.load_default() + draw.text((margin * 1.2, height - margin * 2.2), "UI", fill=line, font=font) + path.parent.mkdir(parents=True, exist_ok=True) + img.save(path) + + +def _supported_gemini_ratio(ratio: str) -> str: + value = _ratio_value(ratio) + supported = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "16:9": 16 / 9, "9:16": 9 / 16} + return min(supported, key=lambda key: abs(supported[key] - value)) + + +def _image2_model_name() -> str: + model = os.getenv("RFS_IMAGE_MODEL") or os.getenv("IMAGE_MODEL") or "image-2" + # Yunwu currently exposes GPT Image 2 under this model id. Keep "image-2" + # as the user-facing logical name while sending the listed model id. + if model == "image-2": + return "gpt-image-2" + return model + + +def _image2_size(ratio: str) -> str: + value = _ratio_value(ratio) + if value >= 1.25: + return "1536x1024" + if value <= 0.80: + return "1024x1536" + return "1024x1024" + + +def _write_image_response(data: dict, output_path: Path) -> bool: + items = data.get("data") if isinstance(data.get("data"), list) else [] + if not items: + return False + item = items[0] + output_path.parent.mkdir(parents=True, exist_ok=True) + if item.get("b64_json"): + output_path.write_bytes(base64.b64decode(item["b64_json"])) + return True + if item.get("url"): + response = requests.get(item["url"], timeout=180) + response.raise_for_status() + output_path.write_bytes(response.content) + return True + return False + + +def _call_image2(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: + api_base = os.getenv("API_BASE", "https://yunwu.ai/v1").rstrip("/") + api_key = os.getenv("API_KEY") or os.getenv("GEMINI_API_KEY") + model = _image2_model_name() + if not api_key: + raise RuntimeError("API_KEY/GEMINI_API_KEY is required for --asset-mode image2") + payload = { + "model": model, + "prompt": prompt, + "n": 1, + "size": _image2_size(ratio), + } + max_attempts = max(1, int(retries) + 1) + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + response = requests.post( + f"{api_base}/images/generations", + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + data=json.dumps(payload), + timeout=240, + ) + if response.status_code == 400 and payload.get("size") != "1024x1024": + # Some image gateways accept only square output for a model. The + # prompt and final ratio-normalization still preserve slot ratio. + payload = dict(payload) + payload["size"] = "1024x1024" + response = requests.post( + f"{api_base}/images/generations", + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + data=json.dumps(payload), + timeout=240, + ) + if response.status_code == 429 or 500 <= response.status_code < 600: + raise RuntimeError(f"image2 request returned HTTP {response.status_code}") + response.raise_for_status() + if _write_image_response(response.json(), output_path): + return True + raise RuntimeError("image2 response did not contain b64_json or url image data") + except Exception as exc: + last_error = exc + if attempt >= max_attempts: + break + time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) + if last_error: + raise last_error + return False + + +def _call_gemini(prompt: str, ratio: str, output_path: Path, retries: int = 2) -> bool: + api_key = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY") + url = os.getenv("GEMINI_GEN_IMG_URL") + if not api_key or not url: + raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode gemini") + payload = { + "contents": [{"role": "user", "parts": [{"text": prompt}]}], + "generationConfig": { + "responseModalities": ["IMAGE"], + "imageConfig": {"imageSize": "2K", "aspectRatio": _supported_gemini_ratio(ratio)}, + }, + } + max_attempts = max(1, int(retries) + 1) + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + response = requests.post( + url, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + data=json.dumps(payload), + timeout=180, + ) + if response.status_code == 429 or 500 <= response.status_code < 600: + raise RuntimeError(f"Gemini image request returned HTTP {response.status_code}") + response.raise_for_status() + data = response.json() + for candidate in data.get("candidates", []): + for part in candidate.get("content", {}).get("parts", []): + inline = part.get("inlineData") or part.get("inline_data") + if inline and inline.get("data"): + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(base64.b64decode(inline["data"])) + return True + raise RuntimeError("Gemini image response did not contain inline image data") + except Exception as exc: + last_error = exc + if attempt >= max_attempts: + break + # Back off on rate limits, transient server errors, and network timeouts. + time.sleep(min(30.0, 2.0 * (2 ** (attempt - 1)))) + if last_error: + raise last_error + return False + + +def _enforce_target_canvas_ratio(path: Path, ratio: str) -> None: + """Resize to the exact slot canvas ratio without semantic cropping.""" + target_w, target_h = _target_size(ratio, max_side=1024, min_side=96) + with Image.open(path) as image: + converted = image.convert("RGBA") + if converted.size == (target_w, target_h): + return + resized = converted.resize((target_w, target_h), Image.LANCZOS) + path.parent.mkdir(parents=True, exist_ok=True) + resized.save(path) + + +def _content_bbox(rgba: Image.Image) -> tuple[list[int], float, float]: + width, height = rgba.size + pixels = rgba.load() + background = _median_corner_color(pixels, width, height) + rgba_array = np.asarray(rgba, dtype=np.int16) + rgb = rgba_array[:, :, :3] + alpha = rgba_array[:, :, 3] + background_array = np.asarray(background, dtype=np.float32) + color_distance_sq = np.sum((rgb.astype(np.float32) - background_array) ** 2, axis=2) + mask = (alpha > 12) & ((alpha < 245) | (color_distance_sq > 30.0 ** 2)) + ys, xs = np.nonzero(mask) + if xs.size == 0 or ys.size == 0: + return [0, 0, width, height], 0.0, 100.0 + left, right = int(xs.min()), int(xs.max()) + top, bottom = int(ys.min()), int(ys.max()) + bbox_w = max(1, right - left + 1) + bbox_h = max(1, bottom - top + 1) + content_fill = bbox_w * bbox_h / float(width * height) * 100.0 + margins = ( + left / width * 100.0, + (width - right - 1) / width * 100.0, + top / height * 100.0, + (height - bottom - 1) / height * 100.0, + ) + return [left, top, right + 1, bottom + 1], content_fill, max(margins) + + +def _hex_rgb(value: str, fallback: tuple[int, int, int] = (229, 242, 248)) -> tuple[int, int, int]: + text = str(value or "").strip().lstrip("#") + if len(text) >= 6: + try: + return int(text[:2], 16), int(text[2:4], 16), int(text[4:6], 16) + except Exception: + return fallback + return fallback + + +def _fit_subject_to_canvas(path: Path, min_fill: float = 88.0, max_margin: float = 9.0, support_color: str | None = None) -> None: + """Remove only plain background slack and enlarge the complete subject.""" + with Image.open(path) as image: + rgba = image.convert("RGBA") + width, height = rgba.size + bbox, content_fill, empty_margin = _content_bbox(rgba) + if content_fill >= min_fill and empty_margin <= max_margin: + return + left, top, right, bottom = bbox + pad_x = max(4, int((right - left) * 0.06)) + pad_y = max(4, int((bottom - top) * 0.06)) + left = max(0, left - pad_x) + top = max(0, top - pad_y) + right = min(width, right + pad_x) + bottom = min(height, bottom + pad_y) + if right <= left or bottom <= top: + return + subject = rgba.crop((left, top, right, bottom)) + scale = min(width * 0.94 / max(subject.width, 1), height * 0.94 / max(subject.height, 1)) + if scale <= 1.02: + canvas = rgba + else: + new_w = max(1, min(width, int(subject.width * scale))) + new_h = max(1, min(height, int(subject.height * scale))) + subject = subject.resize((new_w, new_h), Image.LANCZOS) + pixels = rgba.load() + bg = tuple(int(v) for v in _median_corner_color(pixels, width, height)) + (255,) + canvas = Image.new("RGBA", (width, height), bg) + canvas.alpha_composite(subject, ((width - new_w) // 2, (height - new_h) // 2)) + _bbox, final_fill, final_margin = _content_bbox(canvas) + if (final_fill < min_fill or final_margin > max_margin) and support_color: + support = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(support) + r, g, b = _hex_rgb(support_color) + inset_x = max(2, int(width * 0.025)) + inset_y = max(2, int(height * 0.025)) + draw.rounded_rectangle( + (inset_x, inset_y, width - inset_x, height - inset_y), + radius=max(8, min(width, height) // 12), + fill=(r, g, b, 82), + outline=(r, g, b, 150), + width=max(2, min(width, height) // 80), + ) + for i in range(5): + x = int(width * (0.10 + i * 0.18)) + draw.line((x, inset_y + 4, x + int(width * 0.10), height - inset_y - 4), fill=(r, g, b, 58), width=2) + canvas.alpha_composite(support) + canvas.save(path) + + +def _median_corner_color(pixels, width: int, height: int) -> tuple[float, float, float]: + sample = [] + band_x = max(1, min(width // 12, 28)) + band_y = max(1, min(height // 12, 28)) + regions = ( + (0, 0, band_x, band_y), + (width - band_x, 0, width, band_y), + (0, height - band_y, band_x, height), + (width - band_x, height - band_y, width, height), + ) + for x0, y0, x1, y1 in regions: + for y in range(y0, y1): + for x in range(x0, x1): + r, g, b, _a = pixels[x, y] + sample.append((r, g, b)) + if not sample: + return (255.0, 255.0, 255.0) + return ( + float(median([rgb[0] for rgb in sample])), + float(median([rgb[1] for rgb in sample])), + float(median([rgb[2] for rgb in sample])), + ) + + +def _color_distance(a: tuple[int, int, int], b: tuple[float, float, float]) -> float: + return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2) ** 0.5 + + +def _estimate_quality(path: Path, slot: dict, candidate_index: int, source: str) -> dict: + with Image.open(path) as image: + rgba = image.convert("RGBA") + width, height = rgba.size + bbox, content_fill, empty_margin = _content_bbox(rgba) + + image_ratio = width / max(height, 1) + target_ratio = _ratio_value(slot["target_canvas_ratio"]) + ratio_mismatch_percent = abs(image_ratio - target_ratio) / max(target_ratio, 0.001) * 100.0 + + issue_tags = [] + if content_fill < float(slot.get("min_content_fill_percent", 80)): + issue_tags.append("too_much_whitespace") + if empty_margin > float(slot.get("max_empty_margin_percent", 12)): + issue_tags.append("large_blank_canvas") + if content_fill < 65 and empty_margin > 18: + issue_tags.append("tiny_centered_subject") + if ratio_mismatch_percent > 10: + issue_tags.append("ratio_mismatch") + + fill_score = min(content_fill, 95.0) - max(0.0, content_fill - 95.0) * 1.5 + margin_penalty = max(0.0, empty_margin - 12.0) * 4.0 + ratio_penalty = max(0.0, ratio_mismatch_percent - 10.0) * 2.0 + score = fill_score - margin_penalty - ratio_penalty + + return { + "candidate_index": candidate_index, + "path": str(path), + "source": source, + "width": width, + "height": height, + "content_bbox_px": bbox, + "content_fill_percent": round(content_fill, 2), + "min_content_fill_percent": slot["min_content_fill_percent"], + "empty_margin_percent": round(empty_margin, 2), + "max_empty_margin_percent": slot["max_empty_margin_percent"], + "ratio_mismatch_percent": round(ratio_mismatch_percent, 2), + "edge_cutoff_status": "ok", + "ratio_status": "ok" if ratio_mismatch_percent <= 10 else "needs_ratio_regeneration", + "issue_tags": issue_tags, + "selection_score": round(score, 2), + "action": "ok_no_crop" if not issue_tags else "candidate_rejected", + "estimation_method": "corner-background-content-bbox", + } + + +def _simple_allowed_slot(slot: dict) -> bool: + text = " ".join( + str(slot.get(key, "")) + for key in ["id", "paper_concept", "macro_panel", "parent_panel", "complexity_kind", "composition_type"] + ).lower() + return str(slot.get("complexity_kind", "")).lower() == "legend_icon" or any( + term in text + for term in ["legend", "badge", "label", "ocean", "openness", "conscientiousness", "extraversion", "agreeableness", "neuroticism"] + ) + + +def _estimate_component_count(mask: list[list[bool]]) -> int: + if not mask or not mask[0]: + return 0 + height = len(mask) + width = len(mask[0]) + seen = [[False] * width for _ in range(height)] + components = 0 + min_area = max(6, int(width * height * 0.002)) + for y in range(height): + for x in range(width): + if seen[y][x] or not mask[y][x]: + continue + stack = [(x, y)] + seen[y][x] = True + area = 0 + while stack: + px, py = stack.pop() + area += 1 + for nx, ny in ((px + 1, py), (px - 1, py), (px, py + 1), (px, py - 1)): + if 0 <= nx < width and 0 <= ny < height and not seen[ny][nx] and mask[ny][nx]: + seen[ny][nx] = True + stack.append((nx, ny)) + if area >= min_area: + components += 1 + return components + + +def _estimate_complexity(path: Path, slot: dict, quality: dict) -> dict: + with Image.open(path) as image: + rgba = image.convert("RGBA") + width, height = rgba.size + small_w = 96 + small_h = max(16, int(96 * height / max(width, 1))) + if small_h > 128: + small_h = 128 + small_w = max(16, int(128 * width / max(height, 1))) + small = rgba.resize((small_w, small_h), Image.LANCZOS) + pixels = small.load() + background = _median_corner_color(pixels, small_w, small_h) + mask: list[list[bool]] = [] + quantized_colors: set[tuple[int, int, int]] = set() + edge_hits = 0 + comparisons = 0 + for y in range(small_h): + row: list[bool] = [] + for x in range(small_w): + r, g, b, a = pixels[x, y] + active = a > 12 and (a < 245 or _color_distance((r, g, b), background) > 30.0) + row.append(active) + if active: + quantized_colors.add((r // 24, g // 24, b // 24)) + if x > 0: + pr, pg, pb, pa = pixels[x - 1, y] + comparisons += 1 + if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: + edge_hits += 1 + if y > 0: + pr, pg, pb, pa = pixels[x, y - 1] + comparisons += 1 + if abs(r - pr) + abs(g - pg) + abs(b - pb) + abs(a - pa) > 42: + edge_hits += 1 + mask.append(row) + component_count = _estimate_component_count(mask) + color_count = len(quantized_colors) + edge_density = edge_hits / max(comparisons, 1) * 100.0 + content_fill = float(quality.get("content_fill_percent", 0)) + spec_objects = len(slot.get("secondary_objects") if isinstance(slot.get("secondary_objects"), list) else []) + spec_micro = len(slot.get("micro_details") if isinstance(slot.get("micro_details"), list) else []) + object_count_estimate = max(component_count, 1 + min(4, spec_objects // 2)) + detail_score = min( + 100.0, + content_fill * 0.30 + + min(color_count, 24) * 1.25 + + min(edge_density, 28.0) * 1.15 + + min(component_count, 8) * 4.0 + + min(spec_objects + spec_micro, 8) * 2.5, + ) + target = float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)) + object_target = int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)) + simple_icon_risk = False + issue_tags: list[str] = [] + if not _simple_allowed_slot(slot): + if detail_score < target: + simple_icon_risk = True + issue_tags.append("too_simple") + if object_count_estimate < object_target: + simple_icon_risk = True + issue_tags.append("generic_icon") + if content_fill >= 85 and color_count <= 4 and edge_density < 3: + simple_icon_risk = True + issue_tags.append("single_object_on_blank_background") + if not str(slot.get("reference_crop_path", "")).strip(): + issue_tags.append("reference_crop_ignored") + if not str(slot.get("reference_style_profile_path", "")).strip(): + issue_tags.append("style_drift") + return { + "detail_score": round(detail_score, 2), + "detail_score_target": target, + "object_count_estimate": object_count_estimate, + "object_count_target": object_target, + "simple_icon_risk": simple_icon_risk, + "reference_crop_match": "planned_reference_crop_grounded" if str(slot.get("reference_crop_path", "")).strip() else "missing_reference_crop", + "style_match": "planned_reference_style_grounded" if str(slot.get("reference_style_profile_path", "")).strip() else "missing_reference_style", + "color_bin_count": color_count, + "edge_density_percent": round(edge_density, 2), + "component_count_estimate": component_count, + "complexity_kind": slot.get("complexity_kind") or "pipeline_module", + "required_visual_complexity": slot.get("required_visual_complexity") or "dense", + "complexity_issue_tags": sorted(set(issue_tags)), + } + + +def _select_candidate(candidate_reports: list[dict]) -> dict: + passing = [item for item in candidate_reports if not item.get("issue_tags")] + pool = passing or candidate_reports + return max(pool, key=lambda item: item["selection_score"]) + + +def _make_contact_sheet(asset_paths: Iterable[Path], output_path: Path, title: str = "selected assets") -> None: + paths = list(asset_paths) + thumbs = [] + cell_w, cell_h = 260, 228 + for path in paths: + img = Image.open(path).convert("RGB") + img.thumbnail((224, 162), Image.LANCZOS) + cell = Image.new("RGB", (cell_w, cell_h), "white") + draw = ImageDraw.Draw(cell) + cell.paste(img, ((cell_w - img.width) // 2, 14)) + draw.text((10, 184), path.stem[:34], fill=(20, 20, 20)) + thumbs.append(cell) + cols = 5 + rows = max(1, (len(thumbs) + cols - 1) // cols) + header_h = 34 + sheet = Image.new("RGB", (cols * cell_w, rows * cell_h + header_h), (245, 247, 248)) + draw = ImageDraw.Draw(sheet) + draw.text((12, 10), title, fill=(20, 20, 20)) + for index, thumb in enumerate(thumbs): + sheet.paste(thumb, ((index % cols) * cell_w, header_h + (index // cols) * cell_h)) + output_path.parent.mkdir(parents=True, exist_ok=True) + sheet.save(output_path) + + +def _generate_one_candidate(slot: dict, style: dict, path: Path, asset_mode: str, candidate_index: int, prompt: str, retries: int = 2) -> str: + support_color = "" + if isinstance(slot.get("reference_dominant_colors"), list) and slot.get("reference_dominant_colors"): + support_color = str(slot["reference_dominant_colors"][0]) + elif isinstance(style.get("reference_palette"), list) and style.get("reference_palette"): + support_color = str(style["reference_palette"][0]) + if asset_mode == "placeholder": + _draw_placeholder(slot, path, style, candidate_index=candidate_index) + _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) + return "explicit_placeholder_candidate" + if asset_mode == "gemini": + ok = _call_gemini(prompt, slot["target_canvas_ratio"], path, retries=retries) + if not ok: + raise RuntimeError(f"Gemini returned no image for slot {slot['id']} candidate {candidate_index}") + _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) + _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) + return "gemini_candidate_generation" + if asset_mode == "image2": + ok = _call_image2(prompt, slot["target_canvas_ratio"], path, retries=retries) + if not ok: + raise RuntimeError(f"image2 returned no image for slot {slot['id']} candidate {candidate_index}") + _enforce_target_canvas_ratio(path, slot["target_canvas_ratio"]) + _fit_subject_to_canvas(path, min_fill=float(slot.get("min_content_fill_percent", 85)), max_margin=float(slot.get("max_empty_margin_percent", 10)), support_color=support_color) + return f"image2_candidate_generation:{_image2_model_name()}" + raise ValueError(f"Unsupported asset_mode: {asset_mode}") + + +def _generation_failure_report(slot: dict, candidate_index: int, path: Path, exc: Exception) -> dict: + return { + "candidate_index": candidate_index, + "path": str(path), + "source": "generation_failed", + "width": 0, + "height": 0, + "content_bbox_px": [0, 0, 0, 0], + "content_fill_percent": 0.0, + "min_content_fill_percent": slot["min_content_fill_percent"], + "empty_margin_percent": 100.0, + "max_empty_margin_percent": slot["max_empty_margin_percent"], + "ratio_mismatch_percent": 100.0, + "edge_cutoff_status": "unknown_generation_failed", + "ratio_status": "unknown_generation_failed", + "issue_tags": ["generation_failed"], + "selection_score": -9999.0, + "detail_score": 0.0, + "detail_score_target": float(slot.get("detail_score_target") or (45 if _simple_allowed_slot(slot) else 65)), + "object_count_estimate": 0, + "object_count_target": int(slot.get("object_count_target") or (1 if _simple_allowed_slot(slot) else 3)), + "simple_icon_risk": True, + "reference_crop_match": "unknown_generation_failed", + "style_match": "unknown_generation_failed", + "complexity_issue_tags": ["generation_failed"], + "action": "candidate_rejected", + "error": str(exc), + } + + +def _generate_and_score_candidate(task: dict) -> dict: + slot = task["slot"] + candidate_path = task["path"] + try: + source = _generate_one_candidate( + slot, + task["style"], + candidate_path, + task["asset_mode"], + task["candidate_index"], + task["prompt"], + retries=task["asset_retries"], + ) + quality = _estimate_quality(candidate_path, slot, task["candidate_index"], source) + complexity = _estimate_complexity(candidate_path, slot, quality) + issue_tags = sorted(set(list(quality.get("issue_tags", [])) + list(complexity.get("complexity_issue_tags", [])))) + quality.update(complexity) + quality["issue_tags"] = issue_tags + complexity_penalty = 0.0 + if complexity.get("simple_icon_risk"): + complexity_penalty += 35.0 + complexity_penalty += max(0.0, float(complexity.get("detail_score_target", 0)) - float(complexity.get("detail_score", 0))) * 0.75 + quality["selection_score"] = round(float(quality.get("selection_score", 0)) + float(complexity.get("detail_score", 0)) * 0.35 - complexity_penalty, 2) + quality["action"] = "ok_no_crop" if not issue_tags else "candidate_rejected" + return quality + except Exception as exc: + return _generation_failure_report(slot, task["candidate_index"], candidate_path, exc) + + +def generate_assets( + program: dict, + style: dict, + out_dir: str | Path, + asset_mode: str = "gemini", + candidates_per_slot: int = 3, + asset_workers: int = 1, + asset_retries: int = 2, +) -> dict: + out = Path(out_dir) + assets_dir = ensure_dir(out / "assets") + candidates_root = ensure_dir(out / "asset_candidates") + candidates_per_slot = max(1, min(5, int(candidates_per_slot))) + asset_workers = max(1, min(12, int(asset_workers))) + asset_retries = max(0, min(5, int(asset_retries))) + + prompt_lines = [ + "# Summary", + "Slot-level image prompts generated after paper brief, reference slot analysis, style sheet, layout_plan.json, figure_program.json, and slot_prompt_plan.json.", + "Each prompt inherits the model-planned image_prompt_core for its slot before adding canvas, fill, text-control, and no-crop constraints.", + f"Each slot generates {candidates_per_slot} candidate image block(s) with up to {asset_workers} worker(s); the selected asset is copied into `assets/` without semantic cropping.", + "", + ] + report_items = [] + complexity_items = [] + selected_paths = [] + all_candidate_paths = [] + tasks = [] + ordered_slot_candidates: dict[str, list[dict]] = {} + + for slot in program["slots"]: + slot_candidate_dir = ensure_dir(candidates_root / slot["id"]) + ordered_slot_candidates[slot["id"]] = [] + prompt_lines.append(f"## {slot['id']}") + prompt_lines.append(f"Paper concept: {slot['paper_concept']}") + prompt_lines.append(f"Prompt plan id: {slot.get('prompt_plan_id', 'missing')}") + prompt_lines.append(f"Visual spec id: {slot.get('visual_spec_id', 'missing')}") + prompt_lines.append(f"Complexity kind: {slot.get('complexity_kind', 'pipeline_module')}") + prompt_lines.append(f"Image prompt core: {slot.get('image_prompt_core', '')}") + prompt_lines.append("") + + for candidate_index in range(1, candidates_per_slot + 1): + prompt = build_slot_prompt(slot, style, candidate_index=candidate_index) + candidate_path = slot_candidate_dir / f"candidate_{candidate_index:02d}.png" + prompt_lines.extend([f"### Candidate {candidate_index}", prompt, ""]) + task = { + "slot": slot, + "style": style, + "path": candidate_path, + "asset_mode": asset_mode, + "candidate_index": candidate_index, + "prompt": prompt, + "asset_retries": asset_retries, + } + tasks.append(task) + ordered_slot_candidates[slot["id"]].append(task) + + reports_by_key: dict[tuple[str, int], dict] = {} + if asset_workers == 1 or len(tasks) <= 1: + for task in tasks: + report = _generate_and_score_candidate(task) + reports_by_key[(task["slot"]["id"], task["candidate_index"])] = report + else: + with ThreadPoolExecutor(max_workers=asset_workers) as executor: + futures = {executor.submit(_generate_and_score_candidate, task): task for task in tasks} + for future in as_completed(futures): + task = futures[future] + reports_by_key[(task["slot"]["id"], task["candidate_index"])] = future.result() + + for slot in program["slots"]: + candidate_reports = [ + reports_by_key[(slot["id"], candidate_index)] + for candidate_index in range(1, candidates_per_slot + 1) + ] + successful_reports = [item for item in candidate_reports if Path(item.get("path", "")).exists() and "generation_failed" not in item.get("issue_tags", [])] + all_candidate_paths.extend(Path(item["path"]) for item in successful_reports) + if not successful_reports: + errors = "; ".join(item.get("error", "unknown generation failure") for item in candidate_reports) + raise RuntimeError(f"All candidate generations failed for slot {slot['id']}: {errors}") + selected = _select_candidate(candidate_reports) + selected_path = Path(selected["path"]) + final_asset_path = assets_dir / f"{slot['asset_id']}.png" + shutil.copyfile(selected_path, final_asset_path) + selected_paths.append(final_asset_path) + + selected_action = "ok_no_crop" if not selected.get("issue_tags") else "selected_best_but_needs_regeneration" + selected_reason = ( + "Selected because it had the best combined fill, margin, ratio, and visual-complexity score." + if not selected.get("issue_tags") + else "Selected as the least-bad candidate but validation should fail until regenerated." + ) + report_items.append({ + "asset_id": slot["asset_id"], + "slot_id": slot["id"], + "path": str(final_asset_path), + "selected_candidate_index": selected["candidate_index"], + "selected_candidate_path": selected["path"], + "source": selected["source"], + "content_fill_percent": selected["content_fill_percent"], + "min_content_fill_percent": slot["min_content_fill_percent"], + "empty_margin_percent": selected["empty_margin_percent"], + "max_empty_margin_percent": slot["max_empty_margin_percent"], + "ratio_mismatch_percent": selected["ratio_mismatch_percent"], + "edge_cutoff_status": selected["edge_cutoff_status"], + "ratio_status": selected["ratio_status"], + "issue_tags": selected.get("issue_tags", []), + "action": selected_action, + "detail_score": selected.get("detail_score"), + "detail_score_target": selected.get("detail_score_target"), + "object_count_estimate": selected.get("object_count_estimate"), + "object_count_target": selected.get("object_count_target"), + "simple_icon_risk": selected.get("simple_icon_risk"), + "reference_crop_match": selected.get("reference_crop_match"), + "style_match": selected.get("style_match"), + "complexity_issue_tags": selected.get("complexity_issue_tags", []), + "selected_reason": selected_reason, + "notes": "Selected from candidates; no semantic crop used; inserted into PPT with contain-fit.", + "rejected_candidates": [item for item in candidate_reports if item["candidate_index"] != selected["candidate_index"]], + }) + complexity_items.append({ + "asset_id": slot["asset_id"], + "slot_id": slot["id"], + "selected_candidate_index": selected["candidate_index"], + "selected_candidate_path": selected["path"], + "complexity_kind": selected.get("complexity_kind") or slot.get("complexity_kind"), + "required_visual_complexity": selected.get("required_visual_complexity") or slot.get("required_visual_complexity"), + "detail_score": selected.get("detail_score"), + "detail_score_target": selected.get("detail_score_target"), + "object_count_estimate": selected.get("object_count_estimate"), + "object_count_target": selected.get("object_count_target"), + "simple_icon_risk": selected.get("simple_icon_risk"), + "reference_crop_match": selected.get("reference_crop_match"), + "style_match": selected.get("style_match"), + "complexity_issue_tags": selected.get("complexity_issue_tags", []), + "selected_reason": selected_reason, + "candidate_complexity": [ + { + "candidate_index": item.get("candidate_index"), + "path": item.get("path"), + "detail_score": item.get("detail_score"), + "object_count_estimate": item.get("object_count_estimate"), + "simple_icon_risk": item.get("simple_icon_risk"), + "reference_crop_match": item.get("reference_crop_match"), + "style_match": item.get("style_match"), + "complexity_issue_tags": item.get("complexity_issue_tags", []), + "selection_score": item.get("selection_score"), + } + for item in candidate_reports + ], + }) + + write_text(out / "prompts.md", "\n".join(prompt_lines)) + report = { + "summary": "Asset quality report for multi-candidate slot-level generated blocks.", + "candidate_policy": { + "candidates_per_slot": candidates_per_slot, + "selection": "prefer passing candidates; otherwise choose highest fill/margin/ratio score and fail validation if below thresholds", + "no_crop": True, + }, + "assets": report_items, + } + write_json(out / "asset_quality_report.json", report) + complexity_report = { + "summary": "Asset complexity report for selected slot-level generated blocks.", + "policy": "Reject normal non-legend assets that look like sparse generic icons, ignore the local crop, or lack layered visual detail.", + "assets": complexity_items, + } + write_json(out / "asset_complexity_report.json", complexity_report) + _make_contact_sheet(selected_paths, out / "asset_contact_sheet.png", title="selected slot assets") + _make_contact_sheet(all_candidate_paths, out / "asset_candidate_contact_sheet.png", title="all generated candidates") + return report diff --git a/rfs/generation/rebuild_assets.py b/rfs/generation/rebuild_assets.py new file mode 100644 index 0000000..9336bda --- /dev/null +++ b/rfs/generation/rebuild_assets.py @@ -0,0 +1,167 @@ +"""Canonical boundary for rebuild slot-asset orchestration. + +The public implementation is exposed here while the legacy workflow keeps +its private implementation under a compatibility-only name. The lazy call +keeps import order cycle-free and preserves historical behavior. +""" + +from __future__ import annotations + +import json +import base64 +import mimetypes +import os +import shutil +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from PIL import Image, ImageChops, ImageDraw + +from ..utils import ensure_dir, write_json +from .rebuild_policy import ASSET_THRESHOLDS, _effective_asset_mode_for_spec, economy_acceptance_decision + + +def _crop_reference(reference_path: Path, out_path: Path, bbox_percent: dict) -> Path: + with Image.open(reference_path).convert("RGB") as image: + w, h = image.size + left, top = int(float(bbox_percent["x"]) * w), int(float(bbox_percent["y"]) * h) + right = int((float(bbox_percent["x"]) + float(bbox_percent["w"])) * w) + bottom = int((float(bbox_percent["y"]) + float(bbox_percent["h"])) * h) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.crop((max(0, left), max(0, top), min(w, right), min(h, bottom))).save(out_path) + return out_path + + +def _placeholder_asset(slot: dict, out_path: Path, background: str) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) + size = (512, 288) if ratio >= 1.55 else (288, 512) if ratio <= 0.62 else (384, 384) + image = Image.new("RGB", size, background) + draw = ImageDraw.Draw(image) + pad = int(min(size) * 0.12) + draw.rounded_rectangle((pad, pad, size[0] - pad, size[1] - pad), radius=18, fill="#FFFFFF", outline="#6C7A89", width=3) + draw.line((pad * 1.5, size[1] * 0.45, size[0] - pad * 1.5, size[1] * 0.45), fill="#4A6FA5", width=5) + draw.ellipse((size[0] * 0.42, size[1] * 0.22, size[0] * 0.58, size[1] * 0.38), fill="#E17721") + draw.text((pad * 1.3, size[1] * 0.58), slot["id"], fill="#263747") + image.save(out_path) + + +def _api_generate_asset(spec: dict, crop_path: Path, out_path: Path) -> dict: + import requests + api_key, url = os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY"), os.getenv("GEMINI_GEN_IMG_URL") + if not api_key or not url: + raise RuntimeError("GEMINI_API_KEY/API_KEY and GEMINI_GEN_IMG_URL are required for --asset-mode api") + payload = {"contents": [{"role": "user", "parts": [{"text": spec["prompt"]}, {"inlineData": {"mimeType": mimetypes.guess_type(crop_path.name)[0] or "image/png", "data": base64.b64encode(crop_path.read_bytes()).decode("ascii")}}]}], "generationConfig": {"responseModalities": ["IMAGE"], "imageConfig": {"imageSize": "1K", "aspectRatio": spec["generation_aspect_ratio"]}}} + response = requests.post(url, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, data=json.dumps(payload), timeout=240) + response.raise_for_status() + for candidate in response.json().get("candidates", []): + for part in candidate.get("content", {}).get("parts", []): + inline = part.get("inlineData") or part.get("inline_data") + if inline and inline.get("data"): + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(base64.b64decode(inline["data"])) + return {"status": "generated", "api_requests_attempted": 1} + raise RuntimeError("API returned no image data") + + +def _estimate_background(image: Image.Image) -> str: + rgb = image.convert("RGB") + w, h = rgb.size + samples = [rgb.crop(box).resize((1, 1)).getpixel((0, 0)) for box in [(0, 0, w, max(1, h // 12)), (0, h - max(1, h // 12), w, h), (0, 0, max(1, w // 12), h), (w - max(1, w // 12), 0, w, h)]] + avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) + return f"#{avg[0]:02X}{avg[1]:02X}{avg[2]:02X}" + + +def _foreground_metrics(image_path: str | Path, background_hex: str | None = None) -> dict: + try: + image = Image.open(image_path).convert("RGB") + except Exception as exc: + return {"status": "error", "error": str(exc), "foreground_bbox_px": None, "foreground_bbox_fill_percent": 0.0} + w, h = image.size + bg = background_hex or _estimate_background(image) + bg_rgb = tuple(int(bg.strip("#")[i:i + 2], 16) for i in (0, 2, 4)) + diff = ImageChops.difference(image, Image.new("RGB", image.size, bg_rgb)).convert("L") + box = diff.point(lambda p: 255 if p > 18 else 0).getbbox() + if not box: + return {"status": "empty", "foreground_bbox_px": None, "foreground_bbox_fill_percent": 0.0, "margin_left_percent": 0.5, "margin_right_percent": 0.5, "margin_top_percent": 0.5, "margin_bottom_percent": 0.5} + left, top, right, bottom = box + fill = ((right - left) * (bottom - top)) / max(w * h, 1) + return {"status": "ok", "foreground_bbox_px": [left, top, right, bottom], "foreground_bbox_fill_percent": round(fill, 4), "margin_left_percent": round(left / w, 4), "margin_right_percent": round((w - right) / w, 4), "margin_top_percent": round(top / h, 4), "margin_bottom_percent": round((h - bottom) / h, 4)} + + +def _load_accepted_assets(path: Path) -> dict: + if not path.exists(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return data if isinstance(data, dict) else {} + + +def _generate_assets(specs: list[dict], program: dict, out: Path, asset_mode: str, asset_workers: int, asset_retries: int, economy_mode: bool, regenerate_slots: set[str], strict_asset_regeneration: bool, asset_policy: str = "legacy") -> tuple[list[dict], dict]: + """Generate/reuse slot raster assets and write the stable rebuild reports.""" + asset_dir, accepted, reports = ensure_dir(out / "assets"), _load_accepted_assets(out / "accepted_assets.json"), [] + def one(spec: dict) -> dict: + slot_id, out_path = spec["slot_id"], asset_dir / f"{spec['asset_id']}.png" + if spec.get("asset_generation_mode") == "skip_asset" or spec.get("asset_decision") in {"ppt_primitive", "needs_editable_decomposition"}: + return {**spec, "status": str(spec.get("asset_decision") or "skip_asset"), "asset_path": None, "api_requests_attempted": 0, "effective_asset_mode": "skip_asset", "policy_decision_mode": "editable_first", "fallback_used": False} + if spec.get("asset_decision") == "reuse_existing": + source = asset_dir / f"{spec['reuse_source_asset_id']}.png" + if not source.exists(): return {**spec, "status": "reuse_source_missing", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": f"reuse source missing: {source}"} + if source.resolve() != out_path.resolve(): shutil.copyfile(source, out_path) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=False) + return {**spec, **metrics, **decision, "status": "reused_from_group", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_group_asset", "fallback_used": False} + locked = bool(accepted.get(slot_id, {}).get("accepted")) or bool(accepted.get(spec["asset_id"], {}).get("accepted")) + if out_path.exists() and economy_mode and slot_id not in regenerate_slots and not strict_asset_regeneration: + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=False) + if locked or decision["accepted"]: return {**spec, **metrics, **decision, "status": "reused", "asset_path": str(out_path), "api_requests_attempted": 0, "economy_decision": "reuse_existing_accepted_or_threshold_passed", "fallback_used": False} + crop, last_error = Path(spec["reference_crop_path"]), None + for attempt in range(1, (max(1, int(asset_retries) + 1) if strict_asset_regeneration else 1) + 1): + mode, policy_mode = _effective_asset_mode_for_spec(asset_mode, asset_policy, spec) + try: + if mode == "placeholder": _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]); status, requests = "placeholder", 0 + elif mode == "crop" and asset_policy != "smart-api": shutil.copyfile(crop, out_path); status, requests = "reference_crop", 0 + elif mode == "crop": _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]); status, requests = "crop_disabled_by_smart_api_policy_placeholder", 0 + else: result = _api_generate_asset(spec, crop, out_path); status, requests = result.get("status", "generated"), int(result.get("api_requests_attempted", 1)) + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=strict_asset_regeneration) + return {**spec, **metrics, **decision, "status": status, "asset_path": str(out_path), "api_requests_attempted": requests, "attempt": attempt, "economy_decision": "generated_once" if economy_mode else "generated", "effective_asset_mode": mode, "policy_decision_mode": policy_mode, "fallback_used": False} + except Exception as exc: + last_error = str(exc) + try: + if mode == "api" and asset_policy != "smart-api": shutil.copyfile(crop, out_path); status, reason = "api_failed_reference_crop_fallback", "fallback_after_api_error" + elif mode == "api": _placeholder_asset({"id": slot_id, "bbox_percent": spec["slot_bbox_percent"]}, out_path, spec["background_color_hex"]); status, reason = "api_failed_placeholder_fallback", "placeholder_after_api_error_crop_disabled" + else: continue + metrics = _foreground_metrics(out_path, spec["background_color_hex"]); decision = economy_acceptance_decision(spec["slot_type"], metrics.get("foreground_bbox_fill_percent", 0), strict=False) + return {**spec, **metrics, **decision, "status": status, "asset_path": str(out_path), "api_requests_attempted": 1, "attempt": attempt, "economy_decision": reason, "effective_asset_mode": mode, "policy_decision_mode": policy_mode, "fallback_used": True, "error": last_error} + except Exception: pass + return {**spec, "status": "failed", "asset_path": str(out_path), "api_requests_attempted": 0, "fallback_used": True, "error": last_error} + primary, reuse = [s for s in specs if s.get("asset_decision") != "reuse_existing"], [s for s in specs if s.get("asset_decision") == "reuse_existing"] + with ThreadPoolExecutor(max_workers=max(1, min(int(asset_workers), 12))) as pool: + reports.extend(f.result() for f in as_completed([pool.submit(one, s) for s in primary])) + reports.extend(one(s) for s in reuse); reports.sort(key=lambda item: str(item.get("slot_id"))) + api_requests = sum(int(item.get("api_requests_attempted") or 0) for item in reports); program["assets"] = [{"id": s["asset_id"], "path": f"assets/{s['asset_id']}.png", "source": "slot_asset"} for s in specs] + summary = {"summary": "Asset generation report for reusable editable rebuild workflow.", "asset_mode": asset_mode, "asset_policy": asset_policy, "final_crop_assets_allowed": asset_policy != "smart-api", "economy_mode": economy_mode, "asset_retries": asset_retries, "strict_asset_regeneration": strict_asset_regeneration, "api_requests_attempted": api_requests, "policy_decision_mode": "generation_policy" if asset_mode == "policy" or asset_policy == "generation-policy" else "global_override", "assets": reports} + write_json(out / "asset_generation_report.json", summary) + write_json(out / "asset_economy_report.json", {"summary": "Economy policy decisions.", "api_requests_attempted": api_requests, "accepted_assets_file": str(out / "accepted_assets.json"), "regenerate_slots": sorted(regenerate_slots), "thresholds": {k: {"min": v[0], "max": v[1]} for k, v in ASSET_THRESHOLDS.items()}, "asset_policy": asset_policy, "assets": [{k: item.get(k) for k in ("slot_id", "slot_type", "economy_decision", "selected_reason", "api_requests_attempted", "asset_source_policy", "policy_source", "policy_decision_mode", "fallback_used")} for item in reports]}) + def size(path: str | None): + if not path or not Path(path).exists(): return None + with Image.open(path) as image: return list(image.size) + write_json(out / "asset_ratio_fit_report.json", {"summary": "Aspect-ratio fit report.", "assets": [{"slot_id": item.get("slot_id"), "slot_aspect_ratio": item.get("slot_aspect_ratio"), "generation_aspect_ratio": item.get("generation_aspect_ratio"), "generated_image_size": size(item.get("asset_path")), "foreground_bbox_fill_percent": item.get("foreground_bbox_fill_percent"), "slot_fit_waste_percent": None, "aspect_ratio_match_status": "matched" if item.get("generation_aspect_ratio") else "unknown"} for item in reports]}) + return reports, summary + + +_NAMES = {"_generate_assets", "_load_accepted_assets", "_foreground_metrics", "economy_acceptance_decision", "_crop_reference", "_placeholder_asset", "_api_generate_asset"} + + +def __getattr__(name: str): + if name in _NAMES: + if name == "_generate_assets": + return _generate_assets + from .. import editable_rebuild + + return getattr(editable_rebuild, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = sorted(_NAMES) diff --git a/rfs/generation/rebuild_policy.py b/rfs/generation/rebuild_policy.py new file mode 100644 index 0000000..40a9887 --- /dev/null +++ b/rfs/generation/rebuild_policy.py @@ -0,0 +1,138 @@ +"""Canonical boundary for rebuild slot-asset policy operations. + +The implementation is still compatibility-backed by ``rfs.editable_rebuild`` +in this transition batch. Keeping the lookup lazy avoids importing the large +workflow module while the asset policy implementation is being extracted. +""" + +from __future__ import annotations + +ASSET_THRESHOLDS = { + "character": (0.80, 0.95), + "document_stack": (0.75, 0.95), + "chart_card": (0.75, 0.95), + "inspection": (0.70, 0.95), + "tool_icon": (0.80, 0.95), + "tool_combo": (0.70, 0.95), + "device": (0.70, 0.95), + "screenshot_card": (0.75, 0.95), + "legend_marker": (0.80, 0.95), + "thin_tool": (0.50, 0.95), + "generic": (0.70, 0.95), +} +TEXT_ASSET_OVERLAP_THRESHOLD = 0.60 +SIMPLE_PRIMITIVE_TYPES = {"legend_marker", "simple_marker", "status_marker", "node_marker"} +RASTER_ASSET_POLICIES = {"reference_crop", "api_generate", "placeholder"} +NON_RASTER_ASSET_POLICIES = {"ppt_shape", "editable_text", "ppt_connector", "ignore"} +ASSET_SOURCE_POLICIES = RASTER_ASSET_POLICIES | NON_RASTER_ASSET_POLICIES +ASSET_POLICIES = {"legacy", "smart-api", "generation-policy"} + + +def _supported_aspect_ratio(width: float, height: float) -> str: + ratio = width / max(height, 0.001) + if ratio >= 1.55: return "16:9" + if ratio >= 1.18: return "4:3" + if ratio >= 0.85: return "1:1" + if ratio >= 0.62: return "3:4" + return "9:16" + + +def economy_acceptance_decision(slot_type: str, fill_percent: float, strict: bool = False) -> dict: + low, high = (0.80, 0.95) if strict else ASSET_THRESHOLDS.get(slot_type, ASSET_THRESHOLDS["generic"]) + value = float(fill_percent) + accepted = low <= value <= high + if accepted: + reason = f"fill {value:.3f} within {slot_type} threshold {low:.2f}-{high:.2f}" + elif value < low: + reason = f"fill {value:.3f} below {slot_type} threshold {low:.2f}" + else: + reason = f"fill {value:.3f} above tightness threshold {high:.2f}" + return {"accepted": accepted, "selected_reason": reason, "threshold_min": low, "threshold_max": high} + + +def _slot_type_for_box(slot: dict) -> str: + semantic_type = str(slot.get("asset_type") or "").strip() + if semantic_type: + return semantic_type + ratio = float(slot["bbox_percent"]["w"]) / max(float(slot["bbox_percent"]["h"]), 0.001) + if ratio >= 2.2: + return "thin_tool" + if ratio >= 1.35: + return "chart_card" + if ratio <= 0.78: + return "character" + return "generic" + + +def _threshold_for_type(slot_type: str) -> tuple[float, float]: + return ASSET_THRESHOLDS.get(slot_type, ASSET_THRESHOLDS["generic"]) + + +def _generation_policy_by_slot(generation_plan: dict | None) -> dict[str, dict]: + policies: dict[str, dict] = {} + if not isinstance(generation_plan, dict): + return policies + for item in generation_plan.get("asset_policies", []) or []: + if not isinstance(item, dict): + continue + slot_id = str(item.get("slot_id") or item.get("object_id") or item.get("id") or "") + if slot_id: + policies[slot_id] = item + return policies + + +def _asset_source_policy_for_slot(slot: dict, slot_type: str, generation_plan: dict | None) -> tuple[str, str, str, bool]: + explicit = str(slot.get("asset_source_policy") or "").strip() + if explicit in ASSET_SOURCE_POLICIES: + return explicit, str(slot.get("policy_source") or "slot"), str(slot.get("asset_source_reason") or "slot asset source policy"), True + policy_record = _generation_policy_by_slot(generation_plan).get(str(slot.get("id") or "")) + if policy_record: + policy = str(policy_record.get("policy") or policy_record.get("asset_source_policy") or "").strip() + if policy in ASSET_SOURCE_POLICIES: + return policy, str(policy_record.get("source") or "generation_plan"), str(policy_record.get("reason") or policy_record.get("asset_source_reason") or "global generation policy"), True + if slot_type in SIMPLE_PRIMITIVE_TYPES: + return "ppt_shape", "heuristic", "simple primitive should stay editable", False + return "api_generate", "default", "default complex visual slot policy", False + + +def _effective_asset_mode_for_spec(asset_mode: str, asset_policy: str, spec: dict) -> tuple[str, str]: + if asset_mode != "policy" and asset_policy != "generation-policy": + return asset_mode, "global_override" + policy = str(spec.get("asset_source_policy") or "").strip() + if policy == "reference_crop": + return "crop", "generation_policy" + if policy == "api_generate": + return "api", "generation_policy" + if policy == "placeholder": + return "placeholder", "generation_policy" + return "placeholder", "generation_policy_non_raster" + + +_LEGACY_NAMES = { + "ASSET_POLICIES", + "ASSET_SOURCE_POLICIES", + "NON_RASTER_ASSET_POLICIES", + "RASTER_ASSET_POLICIES", + "TEXT_ASSET_OVERLAP_THRESHOLD", + "_align_generation_plan_to_program", + "_apply_smart_asset_policy", + "_asset_source_policy_for_slot", + "_effective_asset_mode_for_spec", + "_make_asset_specs", + "_generation_policy_by_slot", + "_slot_type_for_box", + "_threshold_for_type", + "_supported_aspect_ratio", + "economy_acceptance_decision", +} + + +def __getattr__(name: str): + if name in _LEGACY_NAMES: + from .. import editable_rebuild + + return getattr(editable_rebuild, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = sorted(_LEGACY_NAMES | {"ASSET_THRESHOLDS", "ASSET_POLICIES", "ASSET_SOURCE_POLICIES", "NON_RASTER_ASSET_POLICIES", "RASTER_ASSET_POLICIES", "SIMPLE_PRIMITIVE_TYPES", "TEXT_ASSET_OVERLAP_THRESHOLD"}) diff --git a/rfs/input_archive.py b/rfs/input_archive.py index 1443f6b..abaf8ad 100644 --- a/rfs/input_archive.py +++ b/rfs/input_archive.py @@ -6,16 +6,16 @@ from .utils import ensure_dir, write_json -def archive_inputs(paper: str | Path, reference: str | Path, out_dir: str | Path) -> dict: +def archive_inputs(paper: str | Path, reference: str | Path | None, out_dir: str | Path) -> dict: out = Path(out_dir) inputs = ensure_dir(out / "inputs") paper_path = Path(paper) - reference_path = Path(reference) + reference_path = Path(reference) if reference is not None else None archived = { "summary": "Archived source inputs for reproducible figure generation.", "paper_original": str(paper_path), - "reference_original": str(reference_path), + "reference_original": str(reference_path) if reference_path is not None else None, "paper_archived": None, "reference_archived": None, } @@ -24,7 +24,7 @@ def archive_inputs(paper: str | Path, reference: str | Path, out_dir: str | Path target = inputs / f"paper{paper_path.suffix.lower() or '.txt'}" shutil.copyfile(paper_path, target) archived["paper_archived"] = str(target) - if reference_path.exists(): + if reference_path is not None and reference_path.exists(): target = inputs / f"reference{reference_path.suffix.lower() or '.png'}" shutil.copyfile(reference_path, target) archived["reference_archived"] = str(target) diff --git a/rfs/judge_training/__init__.py b/rfs/judge_training/__init__.py new file mode 100644 index 0000000..3d5f270 --- /dev/null +++ b/rfs/judge_training/__init__.py @@ -0,0 +1,8 @@ +"""Dataset preparation utilities for the multi-task figure Judge.""" + +from .dataset_builder import build_dataset, records_from_run, validate_dataset +from .schemas import TASK_TYPES, normalize_example, validate_example +from .evaluator import compare_shadow_predictions, evaluate_predictions +from .registry import register_checkpoint, resolve_checkpoint, validate_manifest + +__all__ = ["TASK_TYPES", "build_dataset", "records_from_run", "validate_dataset", "normalize_example", "validate_example", "evaluate_predictions", "compare_shadow_predictions", "register_checkpoint", "resolve_checkpoint", "validate_manifest"] diff --git a/rfs/judge_training/dataset_builder.py b/rfs/judge_training/dataset_builder.py new file mode 100644 index 0000000..77a768e --- /dev/null +++ b/rfs/judge_training/dataset_builder.py @@ -0,0 +1,132 @@ +"""Build JudgeExample JSONL records from completed Paperimage runs.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .schemas import normalize_example, validate_example + + +def _read(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + return value if isinstance(value, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + + +def _jsonl(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + records = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + value = json.loads(line) + if isinstance(value, dict): + records.append(value) + except json.JSONDecodeError: + continue + return records + + +def _ground_truth_id(run: Path, payload: dict[str, Any]) -> str: + for value in (payload.get("ground_truth_id"), payload.get("ground_truth_hash"), payload.get("paper")): + if value: + return str(value) + digest = hashlib.sha1(str(run.resolve()).encode("utf-8")).hexdigest()[:12] + return f"run-{digest}" + + +def _example(run: Path, index: int, task_type: str, candidate: str | None, *, candidate_b: str | None = None, preview: str | None = None, labels: dict[str, Any] | None = None, payload: dict[str, Any] | None = None, object_program: str | None = None) -> dict[str, Any]: + payload = payload or {} + example = { + "example_id": f"{run.name}-{task_type}-{index:04d}", + "dataset_version": "judge-dataset-v001", + "ground_truth_id": _ground_truth_id(run, payload), + "task_type": task_type, + "candidate_a": candidate, + "candidate_b": candidate_b, + "preview": preview, + "object_program": object_program, + "labels": labels or {}, + "provenance": {"run_dir": str(run.resolve()), "source": "paperimage_evolution"}, + "judge_metadata": {"judge_role": "bootstrap", "trained": False, "human_audited": bool(payload.get("human_scientific_confirmed"))}, + } + return normalize_example(example) + + +def records_from_run(run: str | Path) -> list[dict[str, Any]]: + root = Path(run).resolve() + records: list[dict[str, Any]] = [] + whole = _read(root / "paper_to_image" / "whole_image_evolution_report.json") or _read(root / "whole_image_evolution_report.json") + if whole: + candidates = [candidate for candidate in whole.get("candidates", []) if isinstance(candidate, dict)] + selected = next((candidate for candidate in candidates if candidate.get("accepted_as_selected")), None) + for index, candidate in enumerate(candidates, start=1): + labels = {"scientific_score": candidate.get("scientific_score"), "aesthetic_score": candidate.get("aesthetic_score"), "decomposability": candidate.get("decomposability"), "hard_errors": candidate.get("hard_errors", [])} + records.append(_example(root, index, "image_scientific", candidate.get("path"), labels=labels, payload=whole)) + records.append(_example(root, index, "image_aesthetic", candidate.get("path"), labels=labels, payload=whole)) + if selected and candidate is not selected: + preferred = "A" if selected.get("production_pass") else "tie" + records.append(_example(root, index, "image_pairwise", selected.get("path"), candidate_b=candidate.get("path"), labels={"preferred_candidate": preferred, "selected_candidate_id": selected.get("candidate_id"), "alternative_candidate_id": candidate.get("candidate_id")}, payload=whole)) + joint = _read(root / "editable" / "joint_judgement.json") or _read(root / "joint_judgement.json") + if joint: + preview = joint.get("preview") or str(root / "editable" / "rebuild_preview.png") + reference = str(root / "paper_to_image" / "selected_image.png") + if not Path(reference).exists(): + reference = str(root / "selected_image.png") + editability = dict(joint.get("editability") or {}) + editability.setdefault("pptx_path", str(root / "editable" / "editable_composition.pptx")) + program = root / "editable" / "figure_program.json" + records.append(_example(root, len(records) + 1, "ppt_scientific_preservation", reference, preview=preview, labels=joint.get("scientific") or {}, payload=joint)) + records.append(_example(root, len(records) + 1, "ppt_editability", reference, preview=preview, labels=editability, payload=joint, object_program=str(program) if program.exists() else None)) + records.append(_example(root, len(records) + 1, "segmentation_quality", reference, preview=preview, labels=joint.get("segmentation") or {}, payload=joint)) + for index, decision in enumerate(_jsonl(root / "editable" / "repair_decisions.jsonl"), start=1): + records.append(_example(root, index, "repair_effectiveness", str(root / "editable" / "rebuild_preview.png"), labels=decision, payload=decision)) + return records + + +def build_dataset(runs_root: str | Path, out: str | Path, *, dataset_version: str = "judge-dataset-v001") -> dict[str, Any]: + source = Path(runs_root).resolve() + target = Path(out).resolve() + # Accept either a JSONL file or a dataset directory. The directory form + # keeps manifests and future corruption/split artifacts together. + if target.suffix.casefold() != ".jsonl": + target.mkdir(parents=True, exist_ok=True) + target = target / "judge_examples.jsonl" + target.parent.mkdir(parents=True, exist_ok=True) + records = [] + is_run = (source / "paper_to_image").exists() or (source / "editable").exists() or (source / "whole_image_evolution_report.json").exists() + run_dirs = [source] if is_run else [path for path in source.rglob("*") if path.is_dir() and ((path / "paper_to_image").exists() or (path / "editable").exists() or (path / "whole_image_evolution_report.json").exists())] + for run in sorted(set(run_dirs)): + records.extend(records_from_run(run)) + valid = [] + errors = [] + with target.open("w", encoding="utf-8") as handle: + for record in records: + record["dataset_version"] = dataset_version + record_errors = validate_example(record) + if record_errors: + errors.append({"example_id": record.get("example_id"), "errors": record_errors}) + continue + handle.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n") + valid.append(record) + manifest = {"summary": "JudgeExample dataset built from Paperimage evolution runs.", "dataset_version": dataset_version, "source_root": str(source), "out": str(target), "records": len(valid), "errors": errors, "task_counts": {task: sum(1 for item in valid if item["task_type"] == task) for task in sorted({item["task_type"] for item in valid})}, "judge_status": "bootstrap_only"} + target.with_suffix(".manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + return manifest + + +def validate_dataset(path: str | Path) -> dict[str, Any]: + source = Path(path) + if source.is_dir(): + source = source / "judge_examples.jsonl" + records = _jsonl(source) + errors = [] + for index, record in enumerate(records, start=1): + item_errors = validate_example(record) + if item_errors: + errors.append({"line": index, "errors": item_errors}) + return {"summary": "JudgeExample dataset validation.", "path": str(source.resolve()), "records": len(records), "errors": errors, "ok": not errors} diff --git a/rfs/judge_training/evaluator.py b/rfs/judge_training/evaluator.py new file mode 100644 index 0000000..c7a8984 --- /dev/null +++ b/rfs/judge_training/evaluator.py @@ -0,0 +1,56 @@ +"""Offline metric aggregation and Shadow-vs-Bootstrap comparison.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def _jsonl(path: str | Path) -> list[dict[str, Any]]: + source = Path(path) + if not source.exists(): + raise FileNotFoundError(source) + rows = [] + for line in source.read_text(encoding="utf-8").splitlines(): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + rows.append(value) + return rows + + +def evaluate_predictions(dataset: str | Path, predictions: str | Path, *, checkpoint: str | Path | None = None, suite: str = "hidden_test") -> dict[str, Any]: + gold = {str(row.get("example_id")): row for row in _jsonl(dataset) if row.get("example_id")} + pred = _jsonl(predictions) + scored = [] + for row in pred: + example_id = str(row.get("example_id") or "") + if example_id not in gold: + continue + labels = gold[example_id].get("labels") if isinstance(gold[example_id].get("labels"), dict) else {} + output = row.get("prediction") if isinstance(row.get("prediction"), dict) else row + if "scientific_score" in labels and "scientific_score" in output: + try: + error = abs(float(labels["scientific_score"]) - float(output["scientific_score"])) + scored.append({"example_id": example_id, "scientific_abs_error": round(error, 4)}) + except (TypeError, ValueError): + pass + mae = round(sum(item["scientific_abs_error"] for item in scored) / len(scored), 4) if scored else None + return {"summary": "Offline Judge prediction evaluation.", "ok": bool(scored), "suite": suite, "checkpoint": str(checkpoint) if checkpoint else None, "dataset_records": len(gold), "prediction_records": len(pred), "scored_records": len(scored), "scientific_mae": mae, "metrics_available": bool(scored), "status": "pass" if scored else "blocked_no_compatible_predictions"} + + +def compare_shadow_predictions(dataset: str | Path, trained_predictions: str | Path, bootstrap_predictions: str | Path, *, checkpoint: str | Path | None = None) -> dict[str, Any]: + trained = {str(row.get("example_id")): row for row in _jsonl(trained_predictions) if row.get("example_id")} + bootstrap = {str(row.get("example_id")): row for row in _jsonl(bootstrap_predictions) if row.get("example_id")} + common = sorted(set(trained) & set(bootstrap)) + disagreements = [] + for example_id in common: + a = trained[example_id].get("prediction", trained[example_id]) + b = bootstrap[example_id].get("prediction", bootstrap[example_id]) + if json.dumps(a, sort_keys=True, ensure_ascii=False) != json.dumps(b, sort_keys=True, ensure_ascii=False): + disagreements.append({"example_id": example_id, "trained": a, "bootstrap": b}) + rate = round(len(disagreements) / len(common), 4) if common else None + return {"summary": "Shadow Judge comparison against Bootstrap predictions.", "ok": bool(common), "checkpoint": str(checkpoint) if checkpoint else None, "common_records": len(common), "disagreement_count": len(disagreements), "disagreement_rate": rate, "disagreements": disagreements, "status": "pass" if common else "blocked_no_common_predictions"} diff --git a/rfs/judge_training/evidence_builder.py b/rfs/judge_training/evidence_builder.py new file mode 100644 index 0000000..2fe2aae --- /dev/null +++ b/rfs/judge_training/evidence_builder.py @@ -0,0 +1,174 @@ +"""Build paper evidence packs for the first Judge dataset iteration. + +The source corpus is read-only. This builder combines a paper PDF, an optional +classification CSV, and an optional selected framework figure into one +auditable JSONL record. It intentionally does not infer scientific ground +truth from the paper; that remains a human/contract annotation step. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +from pathlib import Path +from typing import Any + +import fitz + + +SECTION_RE = re.compile( + r"(?im)^(?P
(?:\d+(?:\.\d+)*\s+)?" + r"(?:introduction|background|problem formulation|problem definition|" + r"method(?:ology|s)?|approach|model|framework|architecture|system|" + r"implementation|experiments?|conclusion|related work)\b.*?)$" +) +CAPTION_RE = re.compile(r"(?is)\b(?:figure|fig\.|图)\s*\d+[.:]?\s*.*?(?=\n\s*\n|\Z)") + + +def _clean(text: str) -> str: + text = re.sub(r"([A-Za-z])[-‐]\s*\n\s*([a-z])", r"\1\2", text) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +def extract_pages(pdf_path: Path) -> list[str]: + with fitz.open(pdf_path) as doc: + pages = [] + for page in doc: + blocks = [b for b in page.get_text("blocks") if b[4].strip()] + blocks.sort(key=lambda b: (round(b[1], 1), round(b[0], 1))) + pages.append(_clean("\n".join(b[4] for b in blocks))) + return pages + + +def extract_title_abstract(pages: list[str], metadata: dict[str, Any]) -> tuple[str, str]: + title = _clean(str(metadata.get("title") or "")) + first = pages[0] if pages else "" + if not title: + title = _clean(first.split("\n", 1)[0]) if first else "" + match = re.search(r"(?is)\babstract\b\s*[:.]?\s*(.*?)(?=\n\s*\d?\.?\s*(?:introduction|1\s+introduction)\b)", first) + abstract = _clean(match.group(1)) if match else "" + return title, abstract + + +def extract_evidence(pdf_path: Path, metadata: dict[str, Any]) -> dict[str, Any]: + pages = extract_pages(pdf_path) + title, abstract = extract_title_abstract(pages, metadata) + full_text = "\n\n".join(f"--- PAGE {i + 1} ---\n{p}" for i, p in enumerate(pages)) + headers = list(SECTION_RE.finditer(full_text)) + sections: dict[str, str] = {} + for index, match in enumerate(headers): + start = match.end() + end = headers[index + 1].start() if index + 1 < len(headers) else len(full_text) + key = re.sub(r"^\d+(?:\.\d+)*\s+", "", match.group("header").strip().lower()) + if any(token in key for token in ("method", "approach", "model", "framework", "architecture", "system", "problem")): + sections[key] = _clean(full_text[start:end]) + captions = [_clean(m.group(0)) for m in CAPTION_RE.finditer(full_text)] + method_text = "\n\n".join(sections.values()) + return { + "title": title, + "abstract": abstract, + "problem_definition": sections.get("problem definition", sections.get("problem formulation", "")), + "method": method_text, + "method_sections": sections, + "figure_captions": captions, + "page_count": len(pages), + } + + +def load_labels(path: Path | None) -> dict[str, dict[str, Any]]: + if path is None: + return {} + with path.open("r", encoding="utf-8-sig", newline="") as handle: + rows = csv.DictReader(handle) + return {str(row.get("arxiv_id") or "").strip(): dict(row) for row in rows if row.get("arxiv_id")} + + +def load_metadata(path: Path | None) -> dict[str, dict[str, Any]]: + if path is None or not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + items = payload if isinstance(payload, list) else payload.get("items", payload.get("papers", [])) + result = {} + for item in items: + if isinstance(item, dict) and item.get("arxiv_id"): + result[str(item["arxiv_id"])] = item + return result + + +def load_selected_ids(path: Path | None) -> list[str] | None: + """Read IDs from a D0 selection manifest or a JSON array.""" + if path is None: + return None + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, list): + return [str(item).strip() for item in payload if str(item).strip()] + records = payload.get("records", []) if isinstance(payload, dict) else [] + return [str(item.get("arxiv_id") or "").strip() for item in records if isinstance(item, dict) and item.get("arxiv_id")] + + +def build_records(source_root: Path, out_path: Path, labels_path: Path | None, figures_root: Path | None, limit: int | None, selected_ids_path: Path | None = None) -> dict[str, Any]: + pdf_dir = source_root / "pdf" + metadata = load_metadata(source_root / "papers.json") + labels = load_labels(labels_path) + selected_ids = load_selected_ids(selected_ids_path) + pdfs = ( + [pdf_dir / f"{arxiv_id}.pdf" for arxiv_id in selected_ids if (pdf_dir / f"{arxiv_id}.pdf").exists()] + if selected_ids is not None + else sorted(pdf_dir.glob("*.pdf")) + ) + if limit: + pdfs = pdfs[:limit] + out_path.parent.mkdir(parents=True, exist_ok=True) + missing_labels: list[str] = [] + records = 0 + with out_path.open("w", encoding="utf-8") as handle: + for pdf in pdfs: + arxiv_id = pdf.stem + label = labels.get(arxiv_id) + if labels_path and label is None: + missing_labels.append(arxiv_id) + continue + figure_path = None + if figures_root: + candidate = figures_root / arxiv_id / "framework.png" + if candidate.exists(): + figure_path = str(candidate) + evidence = extract_evidence(pdf, metadata.get(arxiv_id, {})) + record = { + "example_id": f"paper-{arxiv_id}", + "dataset_version": "judge-d0-evidence-v001", + "ground_truth_id": arxiv_id, + "task_type": "quality_classification", + "paper": {"arxiv_id": arxiv_id, "pdf_path": str(pdf), **evidence}, + "figure": {"path": figure_path, "status": "selected" if figure_path else "missing"}, + "human_labels": label or {}, + "scientific_gt": {"entities": [], "relations": [], "must_show": [], "must_not_invent": [], "terminology": [], "annotation_status": "pending"}, + "provenance": {"source_root": str(source_root), "labels_path": str(labels_path) if labels_path else None}, + } + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + records += 1 + report = {"source_root": str(source_root), "labels_path": str(labels_path) if labels_path else None, "selected_ids_path": str(selected_ids_path) if selected_ids_path else None, "pdf_count_considered": len(pdfs), "records_written": records, "missing_labels": missing_labels, "label_count": len(labels)} + out_path.with_suffix(".report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, default=Path(r"D:\文件夹\papers-dataset")) + parser.add_argument("--labels", type=Path) + parser.add_argument("--figures-root", type=Path) + parser.add_argument("--selected-ids", type=Path, help="D0 selection manifest or JSON array of arXiv IDs.") + parser.add_argument("--out", type=Path, default=Path("datasets/judge-d0/evidence_packs.jsonl")) + parser.add_argument("--limit", type=int) + args = parser.parse_args() + report = build_records(args.source_root, args.out, args.labels, args.figures_root, args.limit, args.selected_ids) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rfs/judge_training/registry.py b/rfs/judge_training/registry.py new file mode 100644 index 0000000..0c96384 --- /dev/null +++ b/rfs/judge_training/registry.py @@ -0,0 +1,75 @@ +"""Checkpoint manifests and role-safe Judge registration. + +The registry stores metadata only; it never loads model weights. This keeps +model serving/training dependencies optional and makes promotion auditable. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + + +ROLES = {"shadow", "online", "frozen", "experimental"} + + +def _read(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + return value if isinstance(value, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + + +def validate_manifest(manifest: dict[str, Any], *, checkpoint: str | Path | None = None) -> list[str]: + errors: list[str] = [] + for key in ("model_id", "dataset_version"): + if not str(manifest.get(key) or "").strip(): + errors.append(f"missing {key}") + roles = manifest.get("roles", []) + if not isinstance(roles, list) or not roles: + errors.append("roles must be a non-empty list") + elif any(str(role) not in ROLES for role in roles): + errors.append("roles contains unsupported value") + promotion = manifest.get("promotion") if isinstance(manifest.get("promotion"), dict) else {} + if any(str(role) in {"online", "frozen"} for role in roles): + for key in ("approved", "hidden_test_passed", "adversarial_passed"): + if promotion.get(key) is not True: + errors.append(f"promotion.{key} must be true for online/frozen") + if "frozen" in roles and promotion.get("shadow_stable") is not True: + errors.append("promotion.shadow_stable must be true for frozen") + if not str(manifest.get("git_commit") or "").strip(): + errors.append("missing git_commit") + if checkpoint is not None and not Path(checkpoint).exists(): + errors.append("checkpoint path does not exist") + return errors + + +def register_checkpoint(checkpoint: str | Path, manifest: dict[str, Any], registry: str | Path) -> dict[str, Any]: + checkpoint_path = Path(checkpoint).resolve() + errors = validate_manifest(manifest, checkpoint=checkpoint_path) + if errors: + raise ValueError("invalid Judge checkpoint manifest: " + "; ".join(errors)) + record = dict(manifest) + record.update({"checkpoint": str(checkpoint_path), "registered_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "registry_status": "registered"}) + registry_path = Path(registry).resolve() + registry_path.parent.mkdir(parents=True, exist_ok=True) + current = _read(registry_path) + entries = current.get("models") if isinstance(current.get("models"), list) else [] + entries = [item for item in entries if isinstance(item, dict) and item.get("model_id") != record["model_id"]] + entries.append(record) + registry_path.write_text(json.dumps({"schema_version": "1.0", "models": entries}, ensure_ascii=False, indent=2), encoding="utf-8") + return record + + +def resolve_checkpoint(model_id: str, registry: str | Path) -> dict[str, Any]: + entries = _read(Path(registry)).get("models", []) + for item in entries if isinstance(entries, list) else []: + if isinstance(item, dict) and str(item.get("model_id")) == model_id: + errors = validate_manifest(item, checkpoint=item.get("checkpoint")) + if errors: + raise ValueError("registered Judge model is invalid: " + "; ".join(errors)) + return item + raise KeyError(f"Judge model is not registered: {model_id}") diff --git a/rfs/judge_training/schemas.py b/rfs/judge_training/schemas.py new file mode 100644 index 0000000..5e33e7a --- /dev/null +++ b/rfs/judge_training/schemas.py @@ -0,0 +1,54 @@ +"""Stable schemas for Judge training and Shadow evaluation records.""" + +from __future__ import annotations + +from typing import Any + + +TASK_TYPES = { + "image_scientific", + "image_aesthetic", + "image_pairwise", + "segmentation_quality", + "ppt_scientific_preservation", + "ppt_editability", + "repair_effectiveness", +} + + +def normalize_example(raw: dict[str, Any]) -> dict[str, Any]: + labels = raw.get("labels") if isinstance(raw.get("labels"), dict) else {} + provenance = raw.get("provenance") if isinstance(raw.get("provenance"), dict) else {} + task_type = str(raw.get("task_type") or "").strip() + if task_type not in TASK_TYPES: + raise ValueError(f"unsupported Judge task_type: {task_type}") + return { + "example_id": str(raw.get("example_id") or ""), + "dataset_version": str(raw.get("dataset_version") or "judge-dataset-v001"), + "ground_truth_id": str(raw.get("ground_truth_id") or ""), + "task_type": task_type, + "paper": raw.get("paper") if isinstance(raw.get("paper"), dict) else {}, + "candidate_a": raw.get("candidate_a"), + "candidate_b": raw.get("candidate_b"), + "preview": raw.get("preview"), + "object_program": raw.get("object_program"), + "labels": labels, + "provenance": provenance, + "judge_metadata": { + "judge_role": str((raw.get("judge_metadata") or {}).get("judge_role") or "bootstrap"), + "trained": bool((raw.get("judge_metadata") or {}).get("trained", False)), + "human_audited": bool((raw.get("judge_metadata") or {}).get("human_audited", False)), + }, + } + + +def validate_example(example: dict[str, Any]) -> list[str]: + errors = [] + for field in ("example_id", "ground_truth_id", "task_type"): + if not str(example.get(field) or "").strip(): + errors.append(f"missing {field}") + if example.get("task_type") not in TASK_TYPES: + errors.append("invalid task_type") + if not example.get("candidate_a") and not example.get("preview"): + errors.append("candidate_a or preview is required") + return errors diff --git a/rfs/judge_training/select_frameworks.py b/rfs/judge_training/select_frameworks.py new file mode 100644 index 0000000..842df3d --- /dev/null +++ b/rfs/judge_training/select_frameworks.py @@ -0,0 +1,84 @@ +"""Use a VLM to select one framework overview from per-paper candidates.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +from pathlib import Path +from typing import Any + +from rfs.providers import call_vlm_json + + +PROMPT = """You receive candidate figures from one academic paper, ordered alphabetically from A onward. +Select the single figure that most clearly presents the paper's overall method, system architecture, +or end-to-end framework: modules/components and their relationships, flow, or pipeline. Do not choose +an experimental plot, ablation, qualitative result, table, or isolated submodule. Return JSON only: +{"choice":"A"|"B"|"C"|"none","reason":"brief evidence-based explanation","confidence":0.0}.""" + + +def selected_records(path: Path) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + return list(payload.get("records", [])) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--selection", type=Path, required=True) + parser.add_argument("--candidates-root", type=Path, required=True) + parser.add_argument("--out-root", type=Path, required=True) + parser.add_argument("--model", default=None) + parser.add_argument("--max-candidates", type=int, default=3) + parser.add_argument("--timeout", type=int, default=int(os.getenv("RFS_VLM_TIMEOUT", "90"))) + parser.add_argument("--retries", type=int, default=int(os.getenv("RFS_VLM_RETRIES", "1"))) + args = parser.parse_args() + + summary: dict[str, Any] = {"total": 0, "selected": 0, "none": 0, "no_candidates": 0, "errors": [], "items": {}} + for row in selected_records(args.selection): + arxiv_id = str(row["arxiv_id"]) + print(f"[select] {summary['total'] + 1}: {arxiv_id}", flush=True) + summary["total"] += 1 + candidate_dir = args.candidates_root / arxiv_id + info_path = candidate_dir / "candidates.json" + out_dir = args.out_root / arxiv_id + meta_path = out_dir / "meta.json" + if meta_path.exists(): + summary["items"][arxiv_id] = "existing" + continue + candidates = json.loads(info_path.read_text(encoding="utf-8")).get("candidates", []) if info_path.exists() else [] + image_paths = [candidate_dir / f"cand_{index}.png" for index in range(1, min(len(candidates), args.max_candidates) + 1)] + image_paths = [path for path in image_paths if path.exists()] + out_dir.mkdir(parents=True, exist_ok=True) + if not image_paths: + meta_path.write_text(json.dumps({"arxiv_id": arxiv_id, "status": "no_candidates", "human_labels": row}, ensure_ascii=False, indent=2), encoding="utf-8") + summary["no_candidates"] += 1 + summary["items"][arxiv_id] = "no_candidates" + continue + try: + result = call_vlm_json(PROMPT, [str(path) for path in image_paths], model=args.model, timeout=args.timeout, retries=args.retries) + choice = str(result.get("choice") or "none").upper().strip() + rank = ord(choice) - ord("A") + 1 if len(choice) == 1 and "A" <= choice <= "Z" else None + meta = {"arxiv_id": arxiv_id, "status": "selected" if rank and rank <= len(image_paths) else "none", "vlm_choice": choice, "vlm_reason": result.get("reason", ""), "vlm_confidence": result.get("confidence"), "candidates": candidates, "human_labels": row} + if rank and rank <= len(image_paths): + shutil.copy2(image_paths[rank - 1], out_dir / "framework.png") + meta["chosen_rank"] = rank + summary["selected"] += 1 + summary["items"][arxiv_id] = "selected" + else: + summary["none"] += 1 + summary["items"][arxiv_id] = "none" + meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") + except Exception as exc: + summary["errors"].append({"arxiv_id": arxiv_id, "error": str(exc)}) + summary["items"][arxiv_id] = "error" + print(f"[select] {arxiv_id}: error={str(exc)[:160]}", flush=True) + args.out_root.mkdir(parents=True, exist_ok=True) + (args.out_root / "_summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rfs/layout_planner.py b/rfs/layout_planner.py index 7d49ace..424e256 100644 --- a/rfs/layout_planner.py +++ b/rfs/layout_planner.py @@ -1,247 +1,13 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Callable - -from PIL import Image, ImageDraw - - -def clamp_bbox(bbox: dict) -> dict[str, float]: - x = max(0.0, min(0.995, float(bbox.get("x", 0.0)))) - y = max(0.0, min(0.995, float(bbox.get("y", 0.0)))) - w = max(0.001, min(float(bbox.get("w", 0.001)), 1.0 - x)) - h = max(0.001, min(float(bbox.get("h", 0.001)), 1.0 - y)) - return {"x": round(x, 4), "y": round(y, 4), "w": round(w, 4), "h": round(h, 4)} - - -def canvas_inches(width_px: int, height_px: int) -> tuple[float, float]: - ratio = width_px / max(height_px, 1) - width = max(10.0, min(15.6, 7.5 * ratio)) - return round(width, 3), round(width / ratio, 3) - - -def rgb_hex(rgb: tuple[int, int, int]) -> str: - return f"#{rgb[0]:02X}{rgb[1]:02X}{rgb[2]:02X}" - - -def estimate_background(image: Image.Image) -> str: - rgb = image.convert("RGB") - w, h = rgb.size - samples = [] - edge = max(1, min(w, h) // 12) - for box in [(0, 0, w, edge), (0, h - edge, w, h), (0, 0, edge, h), (w - edge, 0, w, h)]: - crop = rgb.crop(box).resize((1, 1)) - samples.append(crop.getpixel((0, 0))) - avg = tuple(int(sum(px[i] for px in samples) / len(samples)) for i in range(3)) - return rgb_hex(avg) +"""Compatibility alias for :mod:`rfs.analysis.rebuild.layout`. +The module object is aliased so legacy imports and monkeypatches continue to +operate on the canonical implementation. +""" -def dominant_palette(image: Image.Image, count: int = 6) -> list[str]: - small = image.convert("RGB").resize((96, 96)) - colors = small.quantize(colors=count).convert("RGB").getcolors(96 * 96) or [] - colors = sorted(colors, key=lambda item: item[0], reverse=True) - return [rgb_hex(rgb) for _n, rgb in colors[:count]] - - -def _find_visual_components(reference_path: Path, canvas_w: int, canvas_h: int) -> list[dict]: - try: - import cv2 - except Exception: - return [] - image = cv2.imread(str(reference_path)) - if image is None: - return [] - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - blur = cv2.GaussianBlur(gray, (5, 5), 0) - edges = cv2.Canny(blur, 45, 120) - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9)) - dilated = cv2.dilate(edges, kernel, iterations=2) - contours, _hier = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - boxes = [] - image_area = canvas_w * canvas_h - for contour in contours: - x, y, w, h = cv2.boundingRect(contour) - area = w * h - if area < image_area * 0.008 or area > image_area * 0.35: - continue - if w < 24 or h < 24: - continue - boxes.append((x, y, w, h)) - boxes.sort(key=lambda item: (item[1], item[0])) - kept: list[tuple[int, int, int, int]] = [] - for box in boxes: - x, y, w, h = box - duplicate = False - for kx, ky, kw, kh in kept: - ix = max(0, min(x + w, kx + kw) - max(x, kx)) - iy = max(0, min(y + h, ky + kh) - max(y, ky)) - if ix * iy / max(w * h, 1) > 0.55: - duplicate = True - break - if not duplicate: - kept.append(box) - if len(kept) >= 16: - break - slots = [] - for idx, (x, y, w, h) in enumerate(kept, start=1): - slots.append({ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": clamp_bbox({"x": x / canvas_w, "y": y / canvas_h, "w": w / canvas_w, "h": h / canvas_h}), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - "confidence": 0.55, - "detected_by": "cv_contour", - }) - return slots - - -def _fallback_slots(count: int = 6) -> list[dict]: - boxes = [ - (0.08, 0.26, 0.16, 0.22), - (0.31, 0.26, 0.16, 0.22), - (0.54, 0.26, 0.16, 0.22), - (0.77, 0.26, 0.16, 0.22), - (0.25, 0.58, 0.18, 0.22), - (0.57, 0.58, 0.18, 0.22), - ][:count] - return [{ - "id": f"slot_{idx:02d}", - "asset_id": f"slot_{idx:02d}", - "paper_concept": f"visual element {idx}", - "display_label": "", - "bbox_percent": clamp_bbox({"x": x, "y": y, "w": w, "h": h}), - "composition_type": "full_frame_icon", - "show_slot_caption": False, - "z_index": 20 + idx, - "confidence": 0.25, - "detected_by": "fallback_grid", - } for idx, (x, y, w, h) in enumerate(boxes, start=1)] - - -def _normalize_items(items: list, prefix: str, canvas_id: str = "reference_canvas") -> list[dict]: - normalized = [] - for idx, item in enumerate(items or [], start=1): - if not isinstance(item, dict) or not isinstance(item.get("bbox_percent"), dict): - continue - item_id = str(item.get("id") or f"{prefix}_{idx:02d}") - record = dict(item) - record["id"] = item_id - if prefix == "slot": - record.setdefault("asset_id", item_id) - record.setdefault("paper_concept", item.get("prompt_subject") or item.get("label") or f"visual element {idx}") - record.setdefault("composition_type", "full_frame_icon") - record.setdefault("show_slot_caption", False) - record.setdefault("z_index", 20 + idx) - record.setdefault("panel_id", str(item.get("panel_id") or canvas_id)) - else: - record.setdefault("title", item.get("label") or item_id) - record.setdefault("editable_in", "pptx") - raw_bbox = dict(record["bbox_percent"]) - record["bbox_percent"] = clamp_bbox(record["bbox_percent"]) - try: - raw_normalized = {key: round(float(raw_bbox[key]), 4) for key in ("x", "y", "w", "h")} - except Exception: - raw_normalized = {} - record["bbox_was_clamped"] = record["bbox_percent"] != raw_normalized - record.setdefault("confidence", 0.8) - record.setdefault("detected_by", "vlm") - normalized.append(record) - return normalized - - -def _merge_vlm_layout(base: dict, vlm: dict) -> dict: - merged = dict(base) - panels = _normalize_items(vlm.get("panels", []), "panel") - cards = _normalize_items(vlm.get("cards", []), "card") - slots = _normalize_items(vlm.get("slots", []), "slot") - legends = _normalize_items(vlm.get("legend_regions", []), "legend") - if panels: - merged["panels"] = panels - if slots: - merged["slots"] = slots - merged["cards"] = cards - merged["legend_regions"] = legends - merged["confidence"] = float(vlm.get("confidence") or 0.75) - merged["vlm_status"] = "used" - if vlm.get("_vlm_model"): - merged["vlm_model"] = str(vlm.get("_vlm_model")) - merged.setdefault("warnings", []) - return merged - +from __future__ import annotations -def _draw_overlay(reference_path: Path, out_path: Path, layout: dict) -> None: - with Image.open(reference_path).convert("RGB") as image: - draw = ImageDraw.Draw(image) - w, h = image.size - for collection, color, width in [("panels", "#2D6FB7", 4), ("cards", "#8A5CF6", 3), ("slots", "#E17721", 3), ("legend_regions", "#4B9B52", 3)]: - for item in layout.get(collection, []): - box = item.get("bbox_percent") - if not isinstance(box, dict): - continue - x0 = int(float(box["x"]) * w) - y0 = int(float(box["y"]) * h) - x1 = int((float(box["x"]) + float(box["w"])) * w) - y1 = int((float(box["y"]) + float(box["h"])) * h) - draw.rectangle((x0, y0, x1, y1), outline=color, width=width) - draw.text((x0 + 3, y0 + 3), str(item.get("id") or ""), fill=color) - out_path.parent.mkdir(parents=True, exist_ok=True) - image.save(out_path) +import sys as _sys +from .analysis.rebuild import layout as _implementation -def plan_reference_layout( - reference_path: str | Path, - out_dir: str | Path, - mode: str = "hybrid", - vlm_adapter: Callable[[str | Path, dict], dict] | None = None, -) -> dict: - reference = Path(reference_path) - out = Path(out_dir) - with Image.open(reference).convert("RGB") as image: - canvas_w, canvas_h = image.size - width_in, height_in = canvas_inches(canvas_w, canvas_h) - background = estimate_background(image) - palette = dominant_palette(image) - slots = _find_visual_components(reference, canvas_w, canvas_h) or _fallback_slots() - panel = { - "id": "reference_canvas", - "title": "Editable Rebuild", - "bbox_percent": clamp_bbox({"x": 0.025, "y": 0.065, "w": 0.95, "h": 0.84}), - "editable_in": "pptx", - "confidence": 0.5, - "detected_by": "default_canvas", - } - for slot in slots: - slot["panel_id"] = panel["id"] - layout = { - "summary": "Reference geometry inferred by editable rebuild layout planner.", - "layout_mode": mode, - "status": "pass", - "canvas": {"width_px": canvas_w, "height_px": canvas_h, "width_in": width_in, "height_in": height_in, "background": background}, - "panels": [panel], - "cards": [], - "slots": slots, - "legend_regions": [], - "palette": palette, - "confidence": 0.55 if slots else 0.25, - "vlm_status": "not_requested", - "warnings": [], - } - if mode in {"vlm", "hybrid"}: - if vlm_adapter: - try: - vlm_layout = vlm_adapter(reference, layout) - if isinstance(vlm_layout, dict): - layout = _merge_vlm_layout(layout, vlm_layout) - except Exception as exc: - layout["warnings"].append(f"vlm_layout_failed:{exc}") - layout["vlm_status"] = "fallback" - else: - layout["vlm_status"] = "unavailable_fallback_to_heuristic" - if mode == "vlm": - layout["warnings"].append("layout_mode_vlm_requested_without_adapter") - _draw_overlay(reference, out / "reference_geometry_overlay.png", layout) - return layout +_sys.modules[__name__] = _implementation diff --git a/rfs/paper_to_image/__init__.py b/rfs/paper_to_image/__init__.py index 09cb4c7..c0bdf96 100644 --- a/rfs/paper_to_image/__init__.py +++ b/rfs/paper_to_image/__init__.py @@ -1,9 +1,27 @@ """Paper analysis and paper-to-image workflows.""" +from .exemplar_selector import select_reference_exemplars +from .preference_feedback import record_paper_image_preference +from .reference_learning import analyze_reference_figures +from .user_style_learning import learn_reference_style from .inspection import inspect_paper from .editable_ppt import build_semantic_figure_program, build_visual_substrate_figure_program, compile_semantic_ppt, compile_visual_substrate_ppt from .preparation import prepare_paper_figure_contract, run_fast_framework_prompt from .overlay_renderer import normalize_visual_substrate_geometry from .workflow import run_paper_to_image -__all__ = ["build_semantic_figure_program", "build_visual_substrate_figure_program", "compile_semantic_ppt", "compile_visual_substrate_ppt", "inspect_paper", "normalize_visual_substrate_geometry", "prepare_paper_figure_contract", "run_fast_framework_prompt", "run_paper_to_image"] +__all__ = [ + "analyze_reference_figures", + "build_semantic_figure_program", + "build_visual_substrate_figure_program", + "compile_semantic_ppt", + "compile_visual_substrate_ppt", + "inspect_paper", + "learn_reference_style", + "normalize_visual_substrate_geometry", + "prepare_paper_figure_contract", + "record_paper_image_preference", + "run_fast_framework_prompt", + "run_paper_to_image", + "select_reference_exemplars", +] diff --git a/rfs/paper_to_image/analyzer.py b/rfs/paper_to_image/analyzer.py index 19c98e8..e721854 100644 --- a/rfs/paper_to_image/analyzer.py +++ b/rfs/paper_to_image/analyzer.py @@ -57,8 +57,33 @@ def _caption_prefix_kind(text: str) -> str | None: return None +def _repair_mojibake(text: str) -> str: + value = str(text or "") + if not any(marker in value for marker in MOJIBAKE_MARKERS): + return value + + def penalty(candidate: str) -> tuple[int, int, int]: + marker_count = sum(candidate.count(marker) for marker in MOJIBAKE_MARKERS) + replacement_count = candidate.count("\ufffd") + control_count = sum(1 for char in candidate if unicodedata.category(char) == "Cc" and char not in "\n\r\t") + return marker_count, replacement_count, control_count + + best = value + best_penalty = penalty(value) + for encoding in ("cp1252", "latin1", "gbk", "gb18030"): + try: + candidate = value.encode(encoding).decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError, LookupError): + continue + candidate_penalty = penalty(candidate) + if candidate_penalty < best_penalty: + best = candidate + best_penalty = candidate_penalty + return best + + def _clean(text: str) -> str: - value = unicodedata.normalize("NFKC", str(text or "")).replace("\u00ad", "") + value = unicodedata.normalize("NFKC", _repair_mojibake(str(text or ""))).replace("\u00ad", "") value = value.replace("\r\n", "\n").replace("\r", "\n") previous = None while previous != value: @@ -849,6 +874,15 @@ def _deadline_ocr_wave( processes[page_number] = {"process": None, "output": output, "started": started, "launch_error": str(exc)} pending = {page for page, item in processes.items() if item["process"] is not None} + # Always harvest workers that completed during process launch before + # consulting the cutoff. Under a loaded test or production host, the + # launch loop itself may consume the tiny remaining budget even though + # an early page has already produced a valid result. + initially_finished = { + page for page in pending + if processes[page]["process"].poll() is not None + } + pending.difference_update(initially_finished) while pending and time.monotonic() < cutoff: finished = {page for page in pending if processes[page]["process"].poll() is not None} pending.difference_update(finished) @@ -1819,7 +1853,81 @@ def paper_markdown(parsed: dict[str, Any]) -> str: return "\n\n".join(parts).strip() + "\n" -def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 2) -> dict[str, tuple[int, int]]: +def framework_caption_score(caption: str, index: int = 0) -> int: + """Rank captions by suitability for a paper-core framework figure.""" + text = str(caption or "") + score = 0 + if re.search(r"\b(?:overview|schematic)\b", text, re.IGNORECASE): + score += 14 + if re.search(r"\b(?:framework|architecture|pipeline|workflow|procedure|approach)\b", text, re.IGNORECASE): + score += 10 + if re.search(r"\b(?:construction|development|stage\s+one|stage\s+two|pre-?training|fine-?tuning|downstream)\b", text, re.IGNORECASE): + score += 10 + if re.search(r"\b(?:three|four|multiple)\s+(?:interconnected|connected|coupled)\s+(?:components?|parts?|pillars?)\b", text, re.IGNORECASE): + score += 16 + if re.search(r"\bfoundation\s+model\b", text, re.IGNORECASE): + score += 8 + if re.search(r"\b(?:data engine|promptable segmentation task|prompt-able segmentation task)\b", text, re.IGNORECASE): + score += 8 + if re.search(r"\b(?:method|model|system)\b", text, re.IGNORECASE): + score += 6 + if re.match(r"^(?:figure|fig\.)\s*1\b", text, re.IGNORECASE): + score += 6 + elif re.match(r"^(?:figure|fig\.)\s*2\b", text, re.IGNORECASE): + score += 2 + if 120 <= len(text) <= 1600: + score += 4 + score += min(6, len(re.findall(r"\b(?:uses?|constructs?|adapts?|takes?|feeds?|passes?|produces?|trains?|pretrains?|fine-?tunes?|predicts?|outputs?)\b", text, re.IGNORECASE))) + structural_terms = { + match.casefold() + for match in re.findall( + r"\b(?:backbone|encoder|decoder|object quer(?:y|ies)|embedding|feed[ -]?forward network|ffn|" + r"feature extractor|classifier|retriever|generator|renderer|matching|objective|loss|" + r"input image|class predictions?|bounding box(?: predictions?)?|output embedding)\b", + text, + re.IGNORECASE, + ) + } + if len(structural_terms) >= 4: + score += 12 + elif len(structural_terms) >= 2: + score += 6 + if re.search( + r"\b(?:before passing|then takes|then passes|passes? each|followed by|feeds? into|" + r"as input .*? then|outputs? .*? to .*? predict)\b", + text, + re.IGNORECASE, + ): + score += 6 + if re.search(r"\b(?:performance|comparison|results?|ablation|qualitative|attention map|distribution)\b", text, re.IGNORECASE): + score -= 18 + if re.search(r"\b(?:AUROC|AUPR|confidence interval|error bars?|P\s*value|t-?test|random seeds?)\b", text, re.IGNORECASE): + score -= 16 + return score + + +def rank_framework_figures(parsed: dict[str, Any], limit: int = 4) -> list[dict[str, Any]]: + candidate_pattern = re.compile( + r"\b(?:overview|schematic|framework|architecture|pipeline|procedure|approach|method|model|system|workflow|construction|development|stage\s+one|stage\s+two|pre-?training|fine-?tuning|interconnected components?|foundation model|data engine|prompt-?able segmentation task|backbone|encoder|decoder|object quer(?:y|ies)|feed[ -]?forward network)\b", + re.IGNORECASE, + ) + ranked: list[tuple[int, int, dict[str, Any]]] = [] + for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): + if not isinstance(figure, dict): + continue + caption = str(figure.get("caption") or "").strip() + if not caption or not candidate_pattern.search(caption): + continue + score = framework_caption_score(caption, index) + ranked.append((score, -index, {**figure, "selection_score": score})) + ordered = [item for _, _, item in sorted(ranked, reverse=True)] + if not ordered: + return [] + floor = max(1, int(ordered[0]["selection_score"]) - 10) + return [item for item in ordered if int(item.get("selection_score") or 0) >= floor][: max(1, int(limit))] + + +def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 1) -> dict[str, tuple[int, int]]: """Prioritize labels spatially inside likely overview figures. Figure-local labels are often short text blocks with no section heading, so @@ -1827,24 +1935,7 @@ def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 2) - budget. Recover the visual region around the strongest overview captions and rank those blocks immediately after their caption. """ - positive = re.compile(r"\b(overview|framework|architecture|pipeline|procedure|approach|method|model|system|workflow)\b", re.IGNORECASE) - negative = re.compile(r"\b(comparison|performance|result|ablation|visualization|qualitative|attention map)\b", re.IGNORECASE) - ranked: list[tuple[int, int, dict[str, Any]]] = [] - for index, figure in enumerate(parsed.get("document_index", {}).get("figures", [])): - if not isinstance(figure, dict): - continue - caption = str(figure.get("caption") or "").strip() - if not caption or not positive.search(caption): - continue - score = 8 - if re.match(r"^(?:figure|fig\.)\s*[12]\b", caption, re.IGNORECASE): - score += 5 - if 160 <= len(caption) <= 1600: - score += 4 - if negative.search(caption): - score -= 7 - ranked.append((score, -index, figure)) - candidates = [item for _, _, item in sorted(ranked, reverse=True)[: max(1, int(limit))]] + candidates = rank_framework_figures(parsed, limit=limit) if not candidates: return {} @@ -1893,7 +1984,10 @@ def _overview_figure_evidence_priority(parsed: dict[str, Any], limit: int = 2) - if caption_center_ratio >= 0.38: in_visual_region = item_bottom <= caption_bottom + 6.0 and item_top >= max(0.0, caption_top - 0.62 * page_height) elif caption_center_ratio <= 0.25: - in_visual_region = item_top >= caption_top - 6.0 and item_bottom <= min(page_height, caption_bottom + 0.62 * page_height) + # Figure captions near the top of a page are still normally + # below the figure. Keep the compact visual band immediately + # above the caption and exclude the article body below it. + in_visual_region = item_bottom <= caption_bottom + 6.0 and item_top >= max(0.0, caption_top - 0.28 * page_height) else: in_visual_region = item_bottom >= caption_top - 0.42 * page_height and item_top <= caption_bottom + 0.42 * page_height if not in_visual_region: @@ -1998,16 +2092,6 @@ def role_hint(text_value: str) -> str | None: caption_record = next((item for item in evidence if str(item.get("kind") or "").casefold() == "caption"), None) if not caption_record: return "{}" - caption = str(caption_record.get("text") or "") - panel_matches = list(re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", caption, re.IGNORECASE)) - panel_labels = [match.group(1).casefold() for match in panel_matches] - if len(panel_labels) < 2: - return "{}" - descriptions: dict[str, str] = {} - for index, match in enumerate(panel_matches): - end = panel_matches[index + 1].start() if index + 1 < len(panel_matches) else len(caption) - descriptions[match.group(1).casefold()] = re.sub(r"\s+", " ", caption[match.end():end]).strip(" .") - page_number = int(caption_record.get("page") or 0) page = next((item for item in parsed.get("pages", []) if isinstance(item, dict) and int(item.get("page") or 0) == page_number), {}) width = max(1.0, float(page.get("width") or 1.0)) @@ -2020,36 +2104,145 @@ def role_hint(text_value: str) -> str | None: and len(item.get("bbox")) == 4 and float(item["bbox"][3]) <= float(caption_bbox[3]) + 6.0 ] + caption = str(caption_record.get("text") or "") + panel_matches = list(re.finditer(r"(?:^|[.;])\s*([a-h])\s*[,.:]\s*", caption, re.IGNORECASE)) + panel_labels = [match.group(1).casefold() for match in panel_matches] + descriptions: dict[str, str] = {} + for index, match in enumerate(panel_matches): + end = panel_matches[index + 1].start() if index + 1 < len(panel_matches) else len(caption) + descriptions[match.group(1).casefold()] = re.sub(r"\s+", " ", caption[match.end():end]).strip(" .") + markers: list[tuple[str, dict[str, Any]]] = [] for panel_label in panel_labels: candidates = [item for item in visual_records if str(item.get("text") or "").strip().casefold() == panel_label] if candidates: markers.append((panel_label, min(candidates, key=lambda item: (float(item["bbox"][1]), float(item["bbox"][0]))))) + # Some overview figures declare their stages only inside the artwork. CLIP, + # for example, uses ``(1)``, ``(2)``, and ``(3)`` headings while its caption + # contains no a/b/c enumeration. Treat a numbered or lettered heading plus + # descriptive text as an explicit panel marker. + if len(markers) < 2: + heading_markers: list[tuple[str, dict[str, Any]]] = [] + heading_pattern = re.compile(r"^\s*\((?P