diff --git a/PROJECT-MANAGEMENT.md b/PROJECT-MANAGEMENT.md index 72cb1a55..025875ed 100644 --- a/PROJECT-MANAGEMENT.md +++ b/PROJECT-MANAGEMENT.md @@ -1,6 +1,6 @@ # OCR-v2 Project Management Log -> **Branch:** `ocr-v2` | **Base:** `master` | **Last Updated:** 2026-06-23 (figure ownership arbitration convergence) +> **Branch:** `ocr-v2` | **Base:** `master` | **Last Updated:** 2026-06-23 (same-number distinct legend dedup guard) > **Rule:** Every step is documented with: What was done, Why it was done, What comes next. --- @@ -2048,5 +2048,13 @@ Vision audit of VFS8CBW2/6FGDBFQN/2UIPV93M/RKSLQRIM identified two residual issu These are NOT convergence-layer issues. The phantom cluster bug (fixed in 12.34) was the only convergence-layer diagnostic inaccuracy found by this audit pass. +### 12.36 Same-number distinct legend dedup guard (2026-06-23) + +- Problem: the legend dedup (`_dedup_map`) uses `(namespace, figure_number)` as the unique key and unconditionally removes all but the highest-priority legend. When a paper has independent figures with the same number in different sections (e.g., body Figure 1 vs appendix Figure 1), the weaker style/zone variant is silently dropped. 2UIPV93M page 49 "图 1" / "图 2" (appendix X-ray figures) were removed because page 16/17 "图 1" / "图 2" (body figures with `legend_like` style) had higher dedup priority. +- Root cause: dedup assumed all legends sharing the same `(namespace, figure_number)` key are duplicates of the same figure. It did not check whether the caption text itself is different. +- Fix: added `_strip_caption_number_prefix` and `_normalized_caption_body` helpers. When a dedup collision occurs, compare the normalized caption bodies (figure-number prefix stripped). If bodies differ AND neither legend is a bundle-source duplicate, mark the lower-priority legend as `same_number_distinct_caption_text` and retain it — using a distinct key so both survive. Bundle-source legends always dedup (their text may differ but they are intentionally weaker duplicates). +- Result: 2UIPV93M now has 19 matched figures (was 17). Page 49 Fig 1 / Fig 2 now correctly retained in `figure_legends` and matched to their X-ray assets. `same_number_distinct_legends` audit surface added to inventory. Bundle-source dedup (DWQQK2YB pattern) unaffected. +- Tests: updated `test_san9ayvr_fig26c_body_narrative` (Fig 26c narrative now correctly survives dedup as distinct legend). All existing dedup tests pass. 217 tests total, 0 failures. + diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 13b8efe3..fdcdc5b8 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -1504,6 +1504,15 @@ def _legend_has_adjacent_page_asset(legend: dict, assets: list[dict]) -> bool: return any(abs(int(asset.get("page", 0) or 0) - legend_page) == 1 for asset in assets) +def _strip_caption_number_prefix(text: str) -> str: + return _FIGURE_NUMBER_PATTERN.sub("", text, count=1).strip() + + +def _normalized_caption_body(text: str) -> str: + body = _strip_caption_number_prefix(text) + return " ".join(body.lower().split()) + + def _legend_dedup_priority(legend: dict, *, bundle_source_legend_ids: set[str], assets: list[dict]) -> tuple[int, int, int, int, int]: legend_id = str(legend.get("block_id", "")) role = str(legend.get("role") or "") @@ -2549,6 +2558,7 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 # Deduplicate numbered legends by family, but treat caption-list / bundle-source # pages as lower-priority duplicates when a stronger real legend instance exists. _dedup_map: dict[tuple[str, int], dict] = {} + _same_number_distinct_keys: set[tuple[str, int, int, str]] = set() for legend in ordered_legends: text = legend.get("text", "") fn = _extract_figure_number(text) @@ -2560,6 +2570,12 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 _dedup_map[key] = legend else: existing = _dedup_map[key] + existing_body = _normalized_caption_body(str(existing.get("text", ""))) + new_body = _normalized_caption_body(text) + existing_is_bundle = str(existing.get("block_id", "")) in bundle_source_legend_ids + new_is_bundle = str(legend.get("block_id", "")) in bundle_source_legend_ids + + # Always determine winner/loser via dedup priority if _legend_dedup_priority( legend, bundle_source_legend_ids=bundle_source_legend_ids, @@ -2569,7 +2585,23 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 bundle_source_legend_ids=bundle_source_legend_ids, assets=assets, ): - _dedup_map[key] = legend + winner, loser = legend, existing + else: + winner, loser = existing, legend + + if ( + existing_body + and new_body + and existing_body != new_body + and not existing_is_bundle + and not new_is_bundle + ): + _dedup_map[key] = winner + _same_number_distinct_keys.add( + (ns, fn, int(loser.get("page", 0) or 0), str(loser.get("block_id", ""))) + ) + else: + _dedup_map[key] = winner emitted_winner_keys: set[tuple[str, int]] = set() deduped_legends: list[dict] = [] @@ -2580,6 +2612,22 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 if fn is not None: ns = _extract_figure_namespace(text) key = (ns, fn) + _distinct_check = ( + ns, + fn, + int(legend.get("page", 0) or 0), + str(legend.get("block_id", "")), + ) + if _distinct_check in _same_number_distinct_keys: + deduped_legends.append(legend) + deduped_legend_ids.append( + { + "page": legend.get("page"), + "block_id": legend.get("block_id", ""), + "dedup_reason": "same_number_distinct_caption_text", + } + ) + continue kept = _dedup_map[key] current_id = str(legend.get("block_id", "")) kept_id = str(kept.get("block_id", "")) @@ -4025,6 +4073,10 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 "rejected_legends": rejected_legends, "unresolved_clusters": unresolved_clusters, "deduped_legend_ids": deduped_legend_ids, + "same_number_distinct_legends": [ + {"figure_number": fn, "page": page, "block_id": str(bid), "reason": "same_number_different_caption_text"} + for (_ns, fn, page, bid) in _same_number_distinct_keys + ], "composite_parent_candidates": composite_parent_candidates, "suppressed_caption_candidates": suppressed_caption_candidates, "official_figure_count": len(matched_figures), diff --git a/tests/test_ocr_figures.py b/tests/test_ocr_figures.py index e8ba933d..69a95f14 100644 --- a/tests/test_ocr_figures.py +++ b/tests/test_ocr_figures.py @@ -1523,18 +1523,26 @@ SAN9AYVR_BODY_AND_FIGURES = [ def test_san9ayvr_fig26c_body_narrative() -> None: - """SAN9AYVR's Fig. 26c narrative text stays body narrative, not formal legend.""" + """SAN9AYVR's Fig. 26c narrative text stays body narrative, not formal legend. + + With same-number distinct-legend guard, Fig. 26c narrative survives dedup + (its text differs from the real Figure 26 caption and it is not a bundle-source). + This is correct: dedup should not be the safety net for body-narrative legend rejection. + """ from paperforge.worker.ocr_figures import build_figure_inventory inventory = build_figure_inventory(SAN9AYVR_BODY_AND_FIGURES) - assert len(inventory["matched_figures"]) == 2, ( - f"Expected 2 matched figures (Fig. 26, Fig. 27), got {len(inventory['matched_figures'])}" + # Fig. 26c narrative + real Fig 26 + Fig 27 = 3 matched figures + assert len(inventory["matched_figures"]) == 3, ( + f"Expected 3 matched figures (Fig 26c narrative, Fig 26, Fig 27), " + f"got {len(inventory['matched_figures'])}" ) - # Without text-matching expansion ("addresses" not in prose-verb list), - # Fig. 26c may win the dedup and appear as the matched legend text. - # This is acceptable for now; figure inventory correctness is not - # compromised. + # Fig 26c narrative should be recorded as same-number-distinct + distinct = inventory.get("same_number_distinct_legends", []) + assert any( + d.get("figure_number") == 26 for d in distinct + ), "Fig 26c narrative must appear in same_number_distinct_legends" def test_san9ayvr_fig26_fig27_remain_formal() -> None: