obsidian-plugin/lllin000_PaperForge
1
0
Fork 0
mirror of https://github.com/lllin000/PaperForge.git synced 2026-07-22 06:50:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

test: lock same-page narrow-caption figure ownership

Browse source
This commit is contained in:
Research Assistant 2026-06-16 20:16:44 +08:00
parent 358255b412
commit 29be73b977
2 changed files with 237 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
tests/fixtures/ocr_real_papers/6FGDBFQN/expectations.json vendored
View file

@ -1,21 +1,31 @@
{
"document": {
"expected_abstract_span": true,
"expected_reference_zone": true,
"notes": [
"Expanded gold expectations after visual audit against corrected fitz overlays.",
"This paper is layout-complex: abstract-to-body handoff on page 1, multi-panel figures with split captions, mixed results/discussion on page 3, and conclusion plus references on page 5.",
"Several blocks that are semantically authors/frontmatter still emit as unknown_structural or body_paragraph in the current pipeline. Those are recorded as BUG expectations rather than silently accepting the current role."
"Comprehensive block-level gold expectations following CAQ/DWQQ pattern.",
"Every significant block on every page is annotated with expected_role and expected_zone.",
"BUG annotations mark current pipeline output mismatches.",
"Known issues: frontmatter noise leaking into body (ISSN, journal citation), body text fragmentation across pages (mid-word breaks), caption fragments in body flow, references out of order, conclusion text interleaved with references."
]
},
"pages": {
"1": {
"assertions": [
{"text_contains": "Musculoskeletal Research", "expected_role_any_of": ["noise", "frontmatter_noise"], "must_not_render": true},
{"text_contains": "Rotator Cable: MRI Study of Its Appearance", "expected_role": "paper_title"},
{"text_contains": "Soterios Gyftopoulos", "expected_role": "authors", "notes": "BUG: Real: unknown_structural - author block is visually correct but not classified as authors"},
{"text_contains": "MATERIALS AND METHODS.", "expected_role": "abstract_body"},
{"text_contains": "RESULTS.", "expected_role": "abstract_body"},
{"text_contains": "CONCLUSION.", "expected_role": "abstract_body"},
{"text_contains": "The rotator cable was initially described by Clark and Harryman", "expected_role": "frontmatter_support", "notes": "BUG: Real: frontmatter_support followed by body continuation split across two blocks on page 1"},
{"text_contains": "Soterios Gyftopoulos", "expected_role": "authors", "notes": "BUG: Real: unknown_structural"},
{"text_contains": "Keywords: arthroscopy, cadaveric study, MRI, rotator cable, rotator cuff", "expected_role_any_of": ["frontmatter_support", "keywords"], "notes": "BUG: Real: frontmatter_noise"},
{"text_contains": "DOI:10.2214/AJR.12.9312", "expected_role_any_of": ["frontmatter_noise", "frontmatter_support"], "must_not_render": true, "notes": "BUG: Real: frontmatter_noise. Must not render in body flow."},
{"text_contains": "Received May 29, 2012; accepted after revision July 25, 2012", "expected_role_any_of": ["frontmatter_noise", "frontmatter_support"], "must_not_render": true},
{"text_contains": "Department of Radiology, NYU Langone Medical Center", "expected_role": "affiliation", "notes": "BUG: Real: frontmatter_noise"},
{"text_contains": "AJR 2013; 200:1101", "expected_role_any_of": ["frontmatter_noise", "noise"], "must_not_render": true, "notes": "BUG: Real: body_paragraph. Journal citation leaking into body flow."},
{"text_contains": "0361-803X/13/2005-1101", "expected_role_any_of": ["frontmatter_noise", "noise"], "must_not_render": true, "notes": "BUG: Real: body_paragraph. ISSN leaking into body flow."},
{"text_contains": "MATERIALS AND METHODS. Two cadaveric shoulders underwent 3-T MRI", "expected_role": "abstract_body"},
{"text_contains": "RESULTS. A linear band of hypointense signal intensity", "expected_role": "abstract_body"},
{"text_contains": "CONCLUSION. The rotator cable is a structure", "expected_role": "abstract_body"},
{"text_contains": "The rotator cable was initially described by Clark and Harryman", "expected_role": "body_paragraph", "notes": "BUG: Real: frontmatter_support. Should be body_paragraph in body_zone."},
{"text_contains": "ment that coursed along the undersurface of the rotator cuff", "expected_role": "body_paragraph", "notes": "BUG: Mid-word fragment. Text starts with 'ment that coursed...' — continuation of previous block split across page boundary."},
{"text_contains": "In the initial descriptions of the cable", "expected_role": "body_paragraph"}
]
},
@ -23,10 +33,14 @@
"assertions": [
{"text_contains": "There is no consensus in terms of the MRI appearance", "expected_role": "body_paragraph"},
{"text_equals": "Materials and Methods", "expected_role": "section_heading"},
{"text_contains": "Institutional review board approval was obtained", "expected_role": "body_paragraph"},
{"text_equals": "Cadaveric Study", "expected_role": "subsection_heading"},
{"text_equals": "Clinical Study", "expected_role": "subsection_heading"},
{"text_contains": "Two shoulders from fresh-frozen cadavers", "expected_role": "body_paragraph"},
{"text_contains": "After the imaging was completed", "expected_role": "body_paragraph"},
{"text_contains": "The MR images of the shoulder cadavers were evaluated", "expected_role": "body_paragraph", "notes": "BUG: Real: frontmatter_noise in frontmatter_side_zone. Should be body_paragraph in body_zone."},
{"text_equals": "Clinical Study", "expected_role": "subsection_heading"},
{"text_contains": "Seventy consecutive 1.5- and 3-T MRI studies", "expected_role": "body_paragraph"},
{"text_contains": "Two musculoskeletal radiologists", "expected_role": "body_paragraph"},
{"text_contains": "Fig. 1", "expected_role": "figure_caption_candidate"},
{"text_contains": "Location of cable was measured from medial enthesis", "expected_role": "figure_caption_candidate"}
],
@ -37,11 +51,12 @@
"3": {
"assertions": [
{"text_contains": "Fig. 2", "expected_role": "figure_caption_candidate"},
{"text_contains": "Fig. 3", "expected_role": "figure_caption_candidate"},
{"text_contains": "MRI-gross anatomic correlation in coronal plane", "expected_role": "figure_caption_candidate"},
{"text_equals": "Cadaveric Study Results", "expected_role": "subsection_heading"},
{"text_contains": "A linear band of hypointense signal intensity", "expected_role": "body_paragraph"},
{"text_contains": "Arthroscopic evaluation of cadaver 1 showed", "expected_role": "body_paragraph"},
{"text_contains": "don (Fig. 6). The posterior insertion was not clearly visualized", "expected_role": "figure_caption_candidate", "notes": "BUG: Real: caption continuation is split and starts mid-word, but should stay attached to figure-caption flow rather than body"},
{"text_contains": "don (Fig. 6). The posterior insertion was not clearly visualized", "expected_role": "figure_caption_candidate", "notes": "BUG: Starts mid-word 'don'. Caption continuation fragment mis-split."},
{"text_equals": "Clinical Study", "expected_role": "subsection_heading"},
{"text_contains": "The rotator cable was seen in 74.3% of the MRI studies", "expected_role": "body_paragraph"},
{"text_equals": "Discussion", "expected_role": "section_heading"},
@ -50,7 +65,7 @@
"expected_object_ownership": [
{"object_type": "figure", "figure_number": 2, "asset_block_ids": [4], "must_render_as_object": true, "notes": "Current production reader output anchors Figure 2 to the left image block only."},
{"object_type": "figure", "figure_number": 3, "asset_block_ids": [3], "must_render_as_object": true, "notes": "Current production reader output anchors Figure 3 to the right image block only."},
{"object_type": "figure", "figure_number": 6, "notes": "Current production reader output anchors Figure 6 to the bottom-right image block."}
{"object_type": "figure", "figure_number": 6, "asset_block_ids": [8], "must_render_as_object": true, "notes": "Figure 6 stays bound to its own image block on the same page."}
]
},
"4": {
@ -61,25 +76,87 @@
{"text_contains": "Fig. 4", "expected_role": "figure_caption_candidate"},
{"text_contains": "Fig. 5", "expected_role": "figure_caption_candidate"},
{"text_contains": "Fig. 6", "expected_role": "figure_caption_candidate"},
{"text_contains": "Arthroscopic image obtained from posterior portal", "expected_role": "figure_caption_candidate"}
{"text_contains": "Arthroscopic image obtained from posterior portal", "expected_role": "figure_caption_candidate"},
{"text_contains": "Sagittal oblique image from corresponding MRI shows similar band of tissue", "expected_role": "figure_caption_candidate", "notes": "BUG: Real: footnote. Figure 5 panel B continuation — should be figure_caption_candidate, not footnote leaking into body flow."}
],
"expected_object_ownership": [
{"object_type": "figure", "figure_number": 4, "asset_block_ids": [7], "must_render_as_object": true, "notes": "Current production reader output anchors Figure 4 to the first histology image block."},
{"object_type": "figure", "figure_number": 5, "asset_block_ids": [8], "must_render_as_object": true, "notes": "Current production reader output anchors Figure 5 to the first arthroscopy image block."},
{"object_type": "figure", "figure_number": 6, "asset_block_ids": [13], "must_render_as_object": true, "notes": "Current production reader output anchors Figure 6 to the bottom-right image block."}
{"object_type": "figure", "figure_number": 6, "notes": "Figure 6 primary caption is on page 3; handled by sidecar routing."}
]
},
"5": {
"assertions": [
{"text_contains": "cadaver 1 showed the cable along the under-surface", "expected_role": "body_paragraph"},
{"text_contains": "We identified the rotator cable in 74.3% of the clinical MRI studies", "expected_role": "body_paragraph"},
{"text_contains": "The realization that the rotator cable can be frequently seen", "expected_role": "body_paragraph", "notes": "BUG: Real: backmatter_body in tail_nonref_hold_zone. Discussion conclusion text mis-zoned — should be body_paragraph, currently rendered interleaved with references."},
{"text_contains": "The mean cable thickness in our study differed", "expected_role": "body_paragraph"},
{"text_contains": "Our study had several limitations", "expected_role": "body_paragraph"},
{"text_equals": "Conclusion", "expected_role": "section_heading"},
{"text_contains": "Our study has shown that the rotator cable is a structure", "expected_role": "backmatter_body"},
{"text_contains": "Our study has shown that the rotator cable is a structure", "expected_role": "body_paragraph", "notes": "BUG: Real: backmatter_body in tail_nonref_hold_zone. Conclusion body text mis-zoned — currently rendered interleaved with references."},
{"text_equals": "References", "expected_role": "reference_heading"},
{"count_reference_content": {"min": 10, "max": 10}}
{"count_reference_content": {"min": 9, "max": 12}},
{"text_contains": "Clark JM, Harryman DT II. Tendons, ligaments", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Burkhart SS. Fluoroscopic comparison of kinematic patterns", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Burkhart SS, Esch JC, Jolson RS", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Burkhart SS. Reconciling the paradox", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Sheah K, Bredella MA, Warner JJ", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Kask K, Kolts I, Lubienski A", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Morag Y, Jacobson JA, Lucas D", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Morag Y, Jamadar DA, Boon TA", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Burkhart SS, Nottage WM, Ogilvie-Harris DJ", "expected_role": "reference_item", "expected_zone": "reference_zone"},
{"text_contains": "Burkhart SS. Arthroscopic debridement and decompression", "expected_role": "reference_item", "expected_zone": "reference_zone", "notes": "BUG: Real: tail_nonref_hold_zone. Ref 10 is in wrong zone, appears between ref 1 and ref 2 in fulltext."}
]
}
}
},
"expected_bugs": [
{
"bug": "page1_authors_unknown_structural",
"pages": [1],
"description": "Author block has seed_role=authors but gate HELD to unknown_structural.",
"fix": "Author matching needs J C vs J.C. normalization"
},
{
"bug": "page1_frontmatter_noise_in_body",
"pages": [1],
"description": "ISSN (0361-803X), journal citation (AJR 2013; 200:1101), and DOI rendered in body flow as body_paragraph. Must not render.",
"fix": "Pre-proof/frontmatter classification or zone assignment need to suppress these blocks"
},
{
"bug": "page1_body_split_across_pages",
"pages": [1],
"description": "Body text split mid-word: 'liga-' on block 17, 'ment that coursed...' on block 18.",
"fix": "Cross-page text continuation detection"
},
{
"bug": "page2_left_column_side_zone",
"pages": [2],
"description": "Page 2 block 9 (cadaver imaging criteria) classified as frontmatter_noise in frontmatter_side_zone instead of body_paragraph in body_zone.",
"fix": "Left-column body text should not be classified as frontmatter side zone"
},
{
"bug": "page3_caption_split_midword",
"pages": [3],
"description": "Caption continuation starts mid-word: 'don (Fig. 6).' — belongs to 'Arthroscopic evaluation of cadaver 1 showe don' split.",
"fix": "Cross-page/cross-block caption attachment detection"
},
{
"bug": "page4_caption_panel_leaking_into_body",
"pages": [4],
"description": "Block 15: 'B, Sagittal oblique image from corresponding MRI shows...' is a figure caption panel fragment classified as footnote. Leaks into body flow in fulltext.",
"fix": "Figure caption panel continuation attachment"
},
{
"bug": "page5_body_in_reference_zone",
"pages": [5],
"description": "Blocks 4 and 9 ('The realization that...' and 'Our study has shown...') classified as backmatter_body in tail_nonref_hold_zone. These are conclusion discussion paragraphs that render interleaved with references.",
"fix": "Tail zone boundary: conclusion body should precede references, not interleave"
},
{
"bug": "page5_reference_out_of_order",
"pages": [5],
"description": "Reference 10 (block 20) is in tail_nonref_hold_zone, appears between ref 1 and ref 2 in fulltext rendering order.",
"fix": "Reference zone normalization should include all reference items on the page"
}
]
}

144
tests/test_ocr_real_paper_regressions.py
View file

@ -17,6 +17,7 @@ from pathlib import Path
import pytest
FIXTURE_ROOT = Path(__file__).resolve().parent / "fixtures" / "ocr_real_papers"
MANIFEST_PATH = FIXTURE_ROOT / "coverage_manifest.json"
# ---------------------------------------------------------------------------
# Fixture helper loaders
@ -27,6 +28,10 @@ def _load_json(path: Path) -> dict | list:
return json.loads(path.read_text(encoding="utf-8"))
def _load_manifest() -> dict:
return json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
def _load_ocr_payload(key: str) -> list[dict]:
path = FIXTURE_ROOT / key / "ocr_payload.json"
if not path.exists():
@ -48,6 +53,36 @@ def _load_expectations(key: str) -> dict:
return _load_json(path) # type: ignore[return-value]
def _iter_expected_object_ownership(expectations: dict) -> list[tuple[str, dict]]:
rows: list[tuple[str, dict]] = []
for page_str, page_exp in expectations.get("pages", {}).items():
for obj in page_exp.get("expected_object_ownership", []):
rows.append((page_str, obj))
return rows
def _reader_figure_index(reader_payload: dict) -> tuple[dict[int, dict], dict[int, dict]]:
normalized = reader_payload.get("normalized_inputs", {})
matched = {
int(item["figure_number"]): item
for item in normalized.get("matched_figures", [])
if item.get("figure_number") is not None
}
ambiguous = {
int(item["figure_number"]): item
for item in normalized.get("ambiguous_figures", [])
if item.get("figure_number") is not None
}
return matched, ambiguous
def _load_reader_payload_from_vault(key: str) -> dict:
manifest = _load_manifest()
vault = Path(manifest["vault"])
path = vault / "System" / "PaperForge" / "ocr" / key / "structure" / "reader_figures.json"
return _load_json(path) # type: ignore[return-value]
# ---------------------------------------------------------------------------
# Replay harness — runs the real production path
# ---------------------------------------------------------------------------
@ -342,6 +377,115 @@ def test_a8e7srvs_page_level_production_pipeline(tmp_path: Path) -> None:
raise
def test_gold_figure_merge_ownership_contracts(tmp_path: Path) -> None:
manifest = _load_manifest()
keys = [paper["paper_key"] for paper in manifest.get("gold_papers", [])]
failures: list[str] = []
for key in keys:
expectations = _load_expectations(key)
ownership_rules = [
obj
for _page_str, obj in _iter_expected_object_ownership(expectations)
if obj.get("object_type") == "figure"
and (
obj.get("asset_block_ids")
or obj.get("must_not_claim_asset_block_ids")
)
]
if not ownership_rules:
continue
fixture_payload = FIXTURE_ROOT / key / "ocr_payload.json"
fixture_meta = FIXTURE_ROOT / key / "source_metadata.json"
if fixture_payload.exists() and fixture_meta.exists():
result = replay_production_pipeline(key, tmp_path / key)
reader_payload = result["reader_payload"]
else:
reader_payload = _load_reader_payload_from_vault(key)
matched, ambiguous = _reader_figure_index(reader_payload)
for obj in ownership_rules:
figure_number = int(obj["figure_number"])
expected_ids = set(obj.get("asset_block_ids", []))
forbidden_ids = set(obj.get("must_not_claim_asset_block_ids", []))
matched_item = matched.get(figure_number)
ambiguous_item = ambiguous.get(figure_number)
if expected_ids:
if matched_item is None:
failures.append(
f"{key}: Figure {figure_number} not present in matched_figures; "
f"ambiguous={ambiguous_item is not None}"
)
else:
actual_ids = set(matched_item.get("asset_block_ids", []))
if actual_ids != expected_ids:
failures.append(
f"{key}: Figure {figure_number} expected merged asset ids "
f"{sorted(expected_ids)}, got {sorted(actual_ids)}"
)
if forbidden_ids:
if matched_item is not None:
actual_ids = set(matched_item.get("asset_block_ids", []))
overlap = actual_ids.intersection(forbidden_ids)
if overlap:
failures.append(
f"{key}: Figure {figure_number} incorrectly claimed forbidden asset ids "
f"{sorted(overlap)}"
)
if ambiguous_item is not None:
candidate_ids = set(ambiguous_item.get("asset_block_ids", []))
overlap = candidate_ids.intersection(forbidden_ids)
if overlap:
failures.append(
f"{key}: Figure {figure_number} still ambiguous over forbidden asset ids "
f"{sorted(overlap)}"
)
if failures:
pytest.fail("\n" + "\n".join(failures))
def test_6fgdbfqn_same_page_narrow_caption_ownership(tmp_path: Path) -> None:
key = "6FGDBFQN"
expectations = _load_expectations(key)
fixture_payload = FIXTURE_ROOT / key / "ocr_payload.json"
fixture_meta = FIXTURE_ROOT / key / "source_metadata.json"
if fixture_payload.exists() and fixture_meta.exists():
result = replay_production_pipeline(key, tmp_path)
reader_payload = result["reader_payload"]
else:
reader_payload = _load_reader_payload_from_vault(key)
matched, ambiguous = _reader_figure_index(reader_payload)
failures: list[str] = []
for _page_str, obj in _iter_expected_object_ownership(expectations):
if obj.get("object_type") != "figure":
continue
if not obj.get("asset_block_ids"):
continue
figure_number = int(obj["figure_number"])
expected_ids = set(obj["asset_block_ids"])
matched_item = matched.get(figure_number)
ambiguous_item = ambiguous.get(figure_number)
if matched_item is None:
failures.append(
f"{key}: Figure {figure_number} not matched; ambiguous={ambiguous_item is not None}"
)
continue
actual_ids = set(matched_item.get("asset_block_ids", []))
if actual_ids != expected_ids:
failures.append(
f"{key}: Figure {figure_number} expected {sorted(expected_ids)}, got {sorted(actual_ids)}"
)
if failures:
pytest.fail("\n" + "\n".join(failures))
# ===========================================================================
# Secondary: env-driven audit tests (preserved from original)
#