""" Frame library v2: candidate revised suggest_frames per RULE_AUDIT.md. This module is used by 05_rerun_v2_detector.py to produce v2 detector labels for the validation study corpus, so v1 vs v2 F1 can be compared empirically. Changes vs `frame_library.suggest_frames` (repo root): - FVS-001 Frame Amplification: REMOVED (retired pending new signal) - FVS-002 Fluency-Quality Illusion: KEEP unchanged - FVS-007 Failure Framing: ADD voice filter (promotional or advisory) - FVS-008 Growth Frame: REMOVED (retired pending growth-vocabulary signal) - FVS-009 Risk Frame: REMOVED uncertainty-coverage requirement - FVS-010 Completeness Illusion: KEEP unchanged - FVS-011 Stakeholder Frame: TIGHTENED density threshold 5 -> 10 - FVS-012 Uncertainty Frame: LOWERED threshold 3 -> 2 (regex expansion deferred) - FVS-014 Temporal Anchoring: LOWERED thresholds (70/60) -> (35/35) - FVS-015 Efficiency Frame: REMOVED (retired pending efficiency-vocabulary signal) - FVS-016 Authority by Citation: LOWERED threshold 50 -> 20 (interim; proper fix requires new named-author-citation signal) All changes are diagnosed in RULE_AUDIT.md §2 with specific evidence from the validation study corpus. This file implements only the rule-level changes feasible without touching framing.py. """ from typing import Any # Import definitions/names from the live library so the v2 rules still # produce the correct labels on output. import sys from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[2] if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) from frame_library import FVS_NAMES, _DEFINITIONS # noqa: E402 def suggest_frames_v2( coverage: dict[str, Any], voice: dict[str, Any], temporal: dict[str, Any], epistemic: dict[str, Any], ) -> list[dict[str, str]]: """Candidate v2 suggest_frames per RULE_AUDIT.md. Signature identical to v1 for drop-in replacement in harness. """ suggestions: list[dict[str, str]] = [] cats = coverage.get("categories", {}) covered = set(coverage.get("covered", [])) missing = set(coverage.get("missing", [])) voice_type = voice.get("voice", "") sourced_pct = epistemic.get("sourced_pct", 100) total_sentences = voice.get("total_sentences", 0) # Preserve the short-doc guard. if total_sentences < 5: return suggestions def _add(fvs_id, name, signal, question): suggestions.append({ "fvs_id": fvs_id, "name": name, "signal": signal, "question": question, "definition": _DEFINITIONS.get(fvs_id, ""), "url": f"/corpus/library/{fvs_id}.html", }) # FVS-001 Frame Amplification: RETIRED (per RULE_AUDIT §2.1) # The rule's density+missing signal does not track sophistication # or frame lock. Retired until a within-session or frame-repetition # signal is available. # FVS-002 Fluency-Quality Illusion: KEEP unchanged. if voice_type == "promotional" and sourced_pct < 30: _add( "FVS-002", "Fluency-Quality Illusion", "Promotional voice with few sourced claims", "If this were written in rough notes instead of polished prose, would you still accept the claims?", ) # FVS-010 Completeness Illusion: KEEP unchanged. if coverage.get("coverage_count", 0) >= 4: densities = [ cats[c]["density_per_1kw"] for c in covered if cats.get(c, {}).get("density_per_1kw", 0) > 0 ] if len(densities) >= 2: max_d = max(densities) min_d = min(d for d in densities if d > 0) if max_d / min_d > 3: _add( "FVS-010", "Completeness Illusion", f"Covers {coverage['coverage_count']} dimensions but density is {max_d / min_d:.0f}x skewed", "Does this document MENTION all perspectives or ANALYZE all perspectives?", ) # FVS-007 Failure Framing: NARROWED with voice filter. # v1 fired on risks missing + uncertainty missing + sourced_pct < 40. # v2 additionally requires voice in (promotional, advisory) to # exclude analytical encyclopedic text that lacks risk/uncertainty # for genre reasons (Wikipedia, descriptive prose). if ( "risks" in missing and "uncertainty" in missing and voice_type in ("promotional", "advisory") ): unhedged_pct = 100 - sourced_pct if unhedged_pct > 60: _add( "FVS-007", "Failure Framing (absent)", "No risk or uncertainty coverage, high unsupported assertion rate, in promotional/advisory voice", "What would have to be true for this analysis to be wrong?", ) # FVS-008 Growth Frame: RETIRED (per RULE_AUDIT §2.4) # Requires a dedicated growth-vocabulary signal that does not exist # in framing.py today. Current rule fires on genres lacking risks # coverage for unrelated reasons and misses the canonical case # (Altman) because risks-vocabulary was dismissively mentioned. # FVS-015 Efficiency Frame: RETIRED (per RULE_AUDIT §2.10) # Requires a dedicated efficiency-vocabulary signal. # FVS-009 Risk Frame (active): LOOSENED. # v1 required risks covered + risks_density > 5 + uncertainty # covered + voice analytical. v2 drops the uncertainty-coverage # requirement. Many documents carry Risk Frame analytically # without explicit uncertainty vocabulary; curator labels # confirm this (b01 has risks 13/kw but uncertainty only 2.2/kw). risks_density = cats.get("risks", {}).get("density_per_1kw", 0) if ( "risks" in covered and risks_density > 5 and voice_type == "analytical" ): _add( "FVS-009", "Risk Frame (active)", f"Substantive risk analysis ({risks_density}/1Kw) in analytical voice", "Is the risk analysis proportionate, or is it overweighting worst-case scenarios?", ) # FVS-011 Stakeholder Frame (active): TIGHTENED density threshold. # v1 threshold: 5/kw. v2: 10/kw. Evidence: the 5/kw threshold fires # on documents that merely list stakeholder groups (b01 nvidia 6.5; # b03 social-media 10.8; c02 EU AI Act 20.0). Raising to 10/kw # may lose b05 (8.8) at the boundary; comparison in REPORT_V2. stakeholder_density = cats.get("stakeholders", {}).get("density_per_1kw", 0) if ( "stakeholders" in covered and stakeholder_density > 10 and voice_type in ("analytical", "advisory") ): _add( "FVS-011", "Stakeholder Frame (active)", f"Stakeholder coverage present ({stakeholder_density}/1Kw) in analytical voice", "Is the stakeholder analysis substantive (specific impacts per group, tradeoffs named) or performative (stakeholder groups mentioned without analyzing who wins and who loses)?", ) # FVS-012 Uncertainty Frame (active): LOWERED threshold 3 -> 2. # Regex expansion deferred (requires framing.py change). This rule # change alone is expected to catch b01 (2.2/kw) and b03 (2.2/kw) # as curator flagged. b06 (0/kw) will still be missed because the # underlying regex doesn't catch its hedge constructions; that is # a signal-level gap the audit names. uncertainty_density = cats.get("uncertainty", {}).get("density_per_1kw", 0) if "uncertainty" in covered and uncertainty_density > 2: _add( "FVS-012", "Uncertainty Frame (active)", f"Uncertainty coverage present ({uncertainty_density}/1Kw)", "Is the uncertainty named substantively (ranges, assumptions, expert disagreement) or merely hedged (language softened without specifics)?", ) # FVS-014 Temporal Anchoring: LOWERED thresholds 70/60 -> 35/35. # v1 thresholds were structurally inert on this corpus (0 fires # across 12 docs; max observed past = 38%, max future = 40%). # v2 uses symmetric 35% threshold. Risk of FP on encyclopedic # history text; will be measured in v2. past_pct = temporal.get("past_pct", 0) future_pct = temporal.get("future_pct", 0) if past_pct >= 35: _add( "FVS-014", "Temporal Anchoring (past)", f"Past-oriented language in {past_pct}% of sentences", "What has changed since this data was current? Is the document presenting historical data as if it were the present?", ) elif future_pct >= 35: _add( "FVS-014", "Temporal Anchoring (future)", f"Future-oriented language in {future_pct}% of sentences", "What is the historical base rate for projections like these? How accurate have similar forecasts been?", ) # FVS-016 Authority by Citation: LOWERED threshold 50 -> 20. # v1 threshold 50% was structurally unreachable (0 fires across # 12 docs, max observed sourced_pct = 26%). v2 interim threshold # 20% is a proxy; a proper fix requires a named-author-citation # signal in framing.py (deferred). if sourced_pct >= 20: _add( "FVS-016", "Authority by Citation", f"High citation density ({sourced_pct}% of claims source-attributed)", "Can you look up the specific sources cited? Would the argument survive if the citations were removed?", ) return suggestions # ──────────────────────────────────────────────────────────────────── # Summary of retired vs kept rules, for comparison tables. # ──────────────────────────────────────────────────────────────────── V2_STATUS = { "FVS-001": "retired", "FVS-002": "kept_unchanged", "FVS-007": "narrowed_voice_filter", "FVS-008": "retired", "FVS-009": "loosened_drop_uncertainty", "FVS-010": "kept_unchanged", "FVS-011": "tightened_density_5_to_10", "FVS-012": "loosened_threshold_3_to_2", "FVS-014": "lowered_thresholds_70_60_to_35_35", "FVS-015": "retired", "FVS-016": "lowered_threshold_50_to_20", }