Baseline Comparison Audit

REVIEWER_MODEL        = gpt-5.6-sol                  # different family from executor (Claude)
REVIEWER_REASONING    = xhigh                    # always; effort never lowers reviewer quality
REVIEWER_SANDBOX      = read-only                # detect-only; never mutate the paper
REVIEWER_CWD          = <paper-dir>              # so it can read claims.json + sources directly
THREAD_POLICY         = fresh mcp__codex__codex per DIMENSION (and per entry on fan-out);
                        NEVER mcp__codex__codex-reply across dimensions/entries
TAXONOMY_VERSION      = 0.5                      # references/hack-pattern-taxonomy.md
PROFILE_VERSION       = 0.1                      # the per-domain baseline profile above (advisory)
PATTERNS_OWNED        = HP-MISSING-BASELINE, HP-WEAK-BASELINE, HP-SIG-OVERLAP,
                        HP-DELTA-ERROR (cross-row comparison form only — see Step 4),
                        HP-RESOURCE-IDENTITY-MISMATCH (named dataset/model/benchmark vs its
                        public record — HF card / Papers-with-Code; gather-facts-then-judge,
                        observability_level_required 0; FP-suppress subset/variant/version)
FINDINGS_FILE         = baseline-comparison-audit.findings.json
FINDING_ID_NAMESPACE  = BC###                    # distinct from F###/NUM###/HL### (consistency), EF### (experiment)
TRACE_POLICY          = forensic (never silently dropped)
TRACE_DIR             = .aris/traces/baseline-comparison-audit/<YYYY-MM-DD>_run<NN>/

ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
# $ARGUMENTS is a paper-dir OR a claims.json path:
LEDGER="$ARGUMENTS"; [ -d "$LEDGER" ] && LEDGER="$LEDGER/claims.json"
# Only the NO-ARGUMENT case defaults to the CWD ledger. An EXPLICIT argument that
# resolves to a missing claims.json must NOT silently fall back to $(pwd) — that
# could audit the wrong paper; let the NO_LEDGER check below fire instead.
[ -z "$ARGUMENTS" ] && LEDGER="$(pwd)/claims.json"
python3 - "$LEDGER" <<'PY'
import json, sys, os, collections
p = sys.argv[1]
if not os.path.isfile(p):
    sys.exit("NO_LEDGER: claims.json not found. Run /evidence-ledger FIRST "
             "(it writes artifact_manifest.json + claims.json).")
d = json.load(open(p, encoding="utf-8"))
claims = d.get("claims", [])
by = collections.Counter(c.get("type") for c in claims)
print("LEDGER       =", os.path.abspath(p))
print("PAPER_DIR    =", os.path.dirname(os.path.abspath(p)) or ".")
print("PAPER_ID     =", d.get("paper_id", "?"))
print("RUN_LEVEL_L  =", d.get("observability_level", 0))
print("CLAIMS       =", len(claims), dict(by))
# applicability signal — comparison / scope / baseline / table_cell are this skill's inputs:
rel = sum(by.get(t, 0) for t in ("comparison", "scope", "baseline", "table_cell"))
print("APPLICABLE   =", "yes" if rel else "low (no comparison/scope/baseline/table claims)")
PY

ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
LEDGER="<abs path to claims.json from Step 0>"
python3 - "$LEDGER" <<'PY'
import json, re, sys, collections
d = json.load(open(sys.argv[1], encoding="utf-8"))
claims = d.get("claims", [])
COMPARE = re.compile(r"\b(state[- ]of[- ]the[- ]art|SOTA|outperform\w*|best|"
                     r"surpass\w*|beats?|superior|first to|consistently|"
                     r"compared?\s+(?:to|with)|baseline|prior\s+(?:work|art))\b", re.I)
anchors, baselines = [], []
for c in claims:
    t, span = c.get("type"), c.get("text_span", "")
    # the SOTA / outperforms assertion (the anchor for completeness + fairness):
    if t == "comparison" or (t == "scope" and COMPARE.search(span)):
        anchors.append((c["claim_id"], t, c.get("location", {}).get("section", "?"), span[:150]))
    # the baseline SET the paper reports (mechanical; the reviewer decides completeness):
    if t == "baseline":
        baselines.append((c["claim_id"], c.get("location", {}).get("section", "?"), span[:150]))
print(f"ANCHOR (comparison/SOTA) claims: {len(anchors)}   baseline-list claims: {len(baselines)}")
for cid, t, sec, sp in anchors[:40]:
    print(f"  [anchor:{t}] {cid} [{sec}] {sp!r}")
for cid, sec, sp in baselines[:20]:
    print(f"  [baseline] {cid} [{sec}] {sp!r}")
nums = collections.Counter(c.get("type") for c in claims if c.get("type") in ("number", "table_cell"))
mets = collections.Counter((c.get("value") or {}).get("metric") for c in claims
                           if (c.get("value") or {}).get("metric"))
print("VALUE CLAIMS =", dict(nums), "  METRICS (seed the profile row) =", dict(mets))
print("APPLICABLE   =", "yes" if (anchors or baselines) else "no -> write [] and stop")
PY

LEDGER="<abs path to claims.json from Step 0>"
grep -rInE '\\begin\{tabular|\\caption|baseline|w\.r\.t|vs\.?|\bours?\b' \
    "$(dirname "$LEDGER")" --include='*.tex' 2>/dev/null | head -60

   WebSearch: "<benchmark> state-of-the-art <paper/current year> leaderboard"
   WebSearch: "<benchmark> papers with code"
   WebFetch:  <the Papers-with-Code / leaderboard URL>   # top 5–8 systems + their dates

DATE=$(date +%Y-%m-%d); N=1
while [ -d ".aris/traces/baseline-comparison-audit/${DATE}_run$(printf %02d $N)" ]; do N=$((N+1)); done
RUNDIR=".aris/traces/baseline-comparison-audit/${DATE}_run$(printf %02d $N)"; mkdir -p "$RUNDIR"
echo "RUNDIR = $RUNDIR"   # carry this exact path forward (shell state does not persist)

{
  "task": "GSM8K grade-school math (LLM reasoning)",
  "profile_version": "0.1",
  "profile_hit": true,
  "leaderboard_source": "https://paperswithcode.com/sota/arithmetic-reasoning-on-gsm8k  (read <UTC date>)",
  "expected_baselines": [
    {"name": "Self-Consistency CoT", "year": 2023, "same_benchmark": true, "published_before_paper": true, "source": "<url>"},
    {"name": "<recent frontier model> few-shot CoT", "year": 2024, "same_benchmark": true, "published_before_paper": true, "source": "<url>"}
  ],
  "notes": "off-profile or inconclusive search -> completeness becomes needs_external_check, not a guess"
}

mcp__codex__codex:
  model: gpt-5.6-sol
  config: {"model_reasoning_effort": "xhigh"}
  sandbox: read-only
  cwd: <absolute PAPER_DIR from Step 0>
  prompt: |
    You are a baseline-COMPLETENESS forensics reviewer. You judge ONE thing: given
    what this paper claims ("state-of-the-art / best / first / outperforms prior
    work") on a benchmark, is an OBVIOUS, RECENT, RELEVANT baseline absent from the
    comparison? You do NOT judge whether numbers are real and you do NOT grade the
    paper. Describe a discrepancy to CHECK, never an accusation; hand off what you
    cannot ground.

    INPUTS (in your working directory — read them directly):
      - claims.json — the evidence ledger. The SOTA/comparison LANGUAGE lives in
        type:"comparison" and type:"scope" claims; the PRESENT baseline set in
        type:"baseline" claims; reported values + table rows in type:"number" /
        type:"table_cell"; figure/table labels in type:"caption". This is the ONLY
        structure you reason over; each claim = {claim_id, type, text_span (VERBATIM),
        location, value?}. You MAY re-open a source file to confirm a span is real,
        but you may NOT introduce a claim that is not in the ledger.
    REFERENCE (external facts gathered by the executor — cite the source in your
    finding; treat as GIVEN data, NOT a verdict, do not second-guess):
      - Profile row (PROFILE_VERSION 0.1): expected baseline FAMILIES = [...];
        matched-budget axis = [...].
      - Live leaderboard (<URL>, accessed <DATE>): top current systems + their
        dates/venues = [...].
      - Candidate expected set (name · same_benchmark? · published_before_paper? ·
        source): [paste expected_baseline_set.json from Step 2, or "NO_PROFILE /
        search unavailable"].
    ANCHOR TARGETS (the SOTA/comparison claims — claim_id + verbatim):
      [paste the anchor claims from Step 1]
    BASELINES THE PAPER REPORTS (mechanical extraction — may be incomplete):
      [paste the baseline-list claims + the grep names from Step 1]
    RUN OBSERVABILITY LEVEL L = <L from Step 0>.

    HARD RULES (a finding that breaks any of these is worthless):
    1. ANCHOR. Every finding above "info" MUST carry >=1 evidence {claim_id, span}
       where claim_id EXISTS in claims.json and span is a VERBATIM whitespace-
       normalized SUBSTRING of THAT claim's text_span (no paraphrase). The primary
       anchor is the SOTA/outperforms claim (a comparison/scope claim); the named
       missing baseline goes in `description`, never as the anchor. ALWAYS anchor —
       even a needs_external_check finding — so it stays navigable.
    2. DISCREPANCY, NOT ACCUSATION. Never "reject", "fabricated", "the authors hid X".
    3. OBSERVABILITY. A missing-baseline-as-STATED is decidable from the manuscript
       => observability_level_required = 0.
    4. HAND OFF WHAT YOU CANNOT SETTLE (the core rule of this step). Emit
       HP-MISSING-BASELINE above info ONLY when ALL hold: (a) a SOTA/best/outperforms
       claim is anchored; (b) a SPECIFIC, NAMED baseline is absent from the present
       set; (c) per the supplied sources that baseline is an ESTABLISHED, publicly-
       available, PRE-DATING standard for THIS exact benchmark (not concurrent, not
       post-dating, not unavailable, not justified-as-omitted in the paper). If ANY of
       (b)/(c) is uncertain — concurrent/post-dating per the dates, no code, niche
       benchmark, NO_PROFILE, the paper justifies the omission — DO NOT flag: set
       verdict_local "needs_external_check", requires_external_check true, severity
       "info", false_positive_risk "high", and name what a human should verify.
    5. HONEST FP. Concurrency/post-dating, unavailability, and a stated justification
       are the common false positives here — say so. If the expected set is empty/
       inconclusive, do NOT manufacture a missing baseline.
    6. pattern_id MUST be HP-MISSING-BASELINE.

    SEVERITY DECISION (HP-MISSING-BASELINE):
      - unambiguous, sourced, same-benchmark, clearly PRE-DATING omission AND the
        paper's HEADLINE is the SOTA/best claim -> "critical", FP "low",
        requires_external_check false.
      - clearly relevant + likely prior but contestable, OR a skipped strong FLOOR
        (BM25/GBDT/linear) while only weak baselines are beaten -> "major", FP
        "medium", requires_external_check true.
      - uncertain / NO_PROFILE / cannot confirm recency or relevance -> "info",
        verdict_local "needs_external_check", requires_external_check true.

    OUTPUT: a single JSON array and NOTHING ELSE (no prose, no code fence). Each
    element conforms to schemas/finding.schema.json:
      {"finding_id":"BC001","skill":"baseline-comparison-audit",
       "pattern_id":"HP-MISSING-BASELINE","title":"short, neutral",
       "description":"which expected baseline is absent + the SOTA claim it undermines + the source",
       "severity":"critical|major|minor|info","observability_level_required":0,
       "evidence":[{"claim_id":"C0xx","span":"verbatim substring",
                    "location":{"file":"...","section":"..."}}],
       "verdict_local":"fail|warn|clean|needs_external_check",
       "requires_external_check":true|false,"false_positive_risk":"low|medium|high",
       "alternative_explanation_checked": "<criticals ONLY — the benign readings you explicitly ruled out: rounding/display precision, unit or metric convention, statistical reporting convention, scope difference. Anchoring proves the text exists, NOT that your interpretation is right; a critical without this field demotes to major>",
       "recommended_reviewer_action":"what to CHECK or ASK — never 'reject'"}
    An empty array [] is a valid, honest result (the expected baselines are all present).

ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
PAPER_DIR="<abs PAPER_DIR from Step 0>"
L="<L from Step 0>"
# (L2 ONLY) per-method budget / tuning / seed asymmetry FACTS — paths + raw hits.
# Guarded: skip entirely at L0/L1 (no configs to read — the reviewer block gets "L<2: ...").
if [ "$L" = "2" ]; then
  grep -rInE 'epochs?|max_steps|learning_rate|\blr\b|batch_?size|seed|n_seeds|num_runs|backbone|warm.?up|sweep|tune|budget' \
      "$PAPER_DIR" --include='*.yaml' --include='*.yml' --include='*.json' --include='*.toml' 2>/dev/null | head -80
  find "$PAPER_DIR" -type f \( -name '*config*' -o -name '*args*' -o -name 'hparams*' \) 2>/dev/null | sort | head -40
  # space-safe and tolerant of zero matches (a bare `shasum $(...)` would hang on no args):
  find "$PAPER_DIR" -maxdepth 3 \( -name '*config*' -o -path '*results*.json' \) 2>/dev/null \
      | head -n 20 | while IFS= read -r ff; do shasum -a 256 "$ff" 2>/dev/null; done
fi

mcp__codex__codex:
  model: gpt-5.6-sol
  config: {"model_reasoning_effort": "xhigh"}
  sandbox: read-only
  cwd: <absolute PAPER_DIR from Step 0>
  prompt: |
    You are a baseline-FAIRNESS-and-SIGNIFICANCE forensics reviewer. For each head-
    to-head comparison the paper draws ("we outperform / are better than / achieve X
    vs baseline Y"), check whether it is FAIR (matched budget/tuning/config),
    SIGNIFICANT (the gap exceeds reported noise), and whether the stated cross-row
    improvement matches its operands. PROPOSE findings only — do NOT grade the paper;
    describe a discrepancy to CHECK, never an accusation. You do NOT judge whether the
    numbers are real (that needs the code) — only what the paper's OWN comparison shows.

    INPUTS (read directly in your working directory):
      - claims.json — comparison LANGUAGE in type:"comparison" / type:"scope"; the
        baseline set in type:"baseline"; values + table rows in type:"number" /
        type:"table_cell"; labels in type:"caption". The ONLY structure you navigate
        by; each claim = {claim_id, type, text_span (VERBATIM), location, value?}. You
        MAY re-open a source file (and, at L2, the config/result files) to confirm a span.
    RUN OBSERVABILITY LEVEL L = <L from Step 0>.
    HEAD-TO-HEAD comparison claims (anchor targets — claim_id + verbatim text_span):
      [paste the comparison/scope anchor claims + the relevant number/table_cell claims]
    REPORTED VARIANCE / SEEDS (if any, with claim_id):
      [paste any "+/-", "std", "over N seeds", "n=" spans, or write "NONE REPORTED"]
    L2 CONFIG FACTS (raw grep/hash hits + config paths — uninterpreted; empty if L<2):
      [paste the grep/find/shasum output above, or "L<2: no repo/configs available"]

    HARD RULES:
    1. ANCHOR every finding above "info" to a real claim_id + a VERBATIM substring of
       that claim. Anchor to the comparison claim (and the row's number/table_cell as
       extra evidence). A config file:line is forensic detail for the description —
       NOT a valid anchor. No verbatim span ⇒ keep at "info".
    2. DISCREPANCY, NOT ACCUSATION. Say what to CHECK/ASK. Never "reject"/"faked".
    3. OBSERVABILITY — set observability_level_required to the LOWEST tier at which the
       discrepancy is DECIDABLE: 0 when the asymmetry/variance/delta is VISIBLE IN THE
       PAPER TEXT/TABLES (e.g. "we train ours 300 epochs" vs a cited 90-epoch baseline
       number); 2 when CONFIRMING it needs the repo's config/result files (it auto-
       demotes on an L0/L1 run — that is correct, not a loss).
    4. HONEST FP. A documented identical budget, standard reference numbers cited from
       a baseline's own paper, a large gap, a reported significance test, and
       genuinely deterministic metrics are COMMON false positives — say so.
    5. pattern_id MUST be one of: HP-WEAK-BASELINE, HP-SIG-OVERLAP, HP-DELTA-ERROR,
       HP-RESOURCE-IDENTITY-MISMATCH.

    CHECKLIST (one finding per concrete discrepancy):
     1. FAIRNESS / WEAK BASELINE [HP-WEAK-BASELINE] — the proposed method gets more
        compute / tuning / data, or runs at more favorable settings, than the baseline;
        the compared rows use non-matching configs (backbone / data / split /
        decoding); a baseline is left at defaults while the method is tuned; a baseline
        number is copied from an old paper at a different budget. The strongest single
        check: is the method's OWN backbone-without-the-new-component (the ablation-as-
        baseline) reported at an IDENTICAL budget? severity major. observability 0 if
        the asymmetry is STATED in text; 2 if only the configs reveal it. FP: identical
        budget documented; a standard reference number quoted from the baseline's own
        paper (citing a published number is legitimate — note any config delta, do not
        allege).
     2. SIGNIFICANCE / OVERLAP [HP-SIG-OVERLAP] — "outperforms / better / consistently"
        claimed where reported error bars OVERLAP, or where NO variance / seed count is
        reported for a SMALL gap (within the field's typical noise for this metric).
        severity: major if error bars are reported AND overlap; minor if merely absent
        variance for a small gap (rise to major only if the headline rests on it).
        observability 0. FP (high → say so): a large gap; a significance test reported;
        a genuinely deterministic metric (exact match on a fixed test set). Two further
        thin-evidence signals — each a recurring real-review tell — to flag EXPLICITLY
        (SAME pattern_id HP-SIG-OVERLAP, SAME anchor = the comparison/SOTA claim):
        (a) NO VARIANCE / SEEDS REPORTED — the comparison rests on bare point estimates
            with NO ±/std/CI and NO seed/run count reported AT ALL (one number per cell,
            no "over N seeds"), so the gap cannot be told from run-to-run noise. severity
            minor for a small gap; major when the headline rests on it OR the profile's
            "Variance norm" column expects >=N seeds for this domain (e.g. RL >=5 seeds,
            retrieval per-query CI) and none are reported. observability 0. FP (high →
            say so): a large clearly-separated gap; a reported significance test; a
            genuinely deterministic / single-pass metric where seeds are moot.
        (b) SINGLE-DATASET-ONLY — a "consistently / robustly / across-the-board /
            general" comparison claim that rests on ONE dataset/benchmark (or a single
            split/domain), too thin for the breadth the wording asserts. severity minor;
            major only when that breadth IS the headline. observability 0. FP (high →
            say so): the claim is explicitly SCOPED to that one benchmark ("on GSM8K we
            …"); the dataset is the field-standard SOLE benchmark for the task; or broad
            scope genuinely exists elsewhere in the paper. LANE: this fires ONLY when
            anchored to a comparison/SOTA claim — generic scope-language inflation with
            NO comparison claim ("a comprehensive study") is consistency-audit's
            HP-SCOPE-INFLATE, not this; do NOT double-emit.
     3. CROSS-ROW DELTA ARITHMETIC [HP-DELTA-ERROR] — recompute a stated "improves over
        <baseline> by X%" as (proposed-baseline)/baseline AND absolute points; flag if
        X disagrees beyond rounding, or relative/absolute are conflated to inflate.
        severity major; critical if the corrected delta deflates a "large/significant"
        framing. observability 0. FP: abs-vs-rel stated explicitly; rounding. DELTA
        SCOPE (critical to avoid double-counting): flag ONLY when the two operands are
        a BASELINE row value and the PROPOSED row value living in DIFFERENT
        sentences/cells — the cross-row case a single-sentence regex cannot pair. Do
        NOT re-flag a delta whose operands AND stated value sit in ONE sentence — the
        deterministic consistency pass already owns those.
     4. RESOURCE IDENTITY [HP-RESOURCE-IDENTITY-MISMATCH] — a named dataset / benchmark /
        model is described with a checkable PUBLIC-RECORD property its registry contradicts
        (ImageNet-1k stated with the wrong #classes/size; a model's parameter count off from
        its card; a "SOTA 91.2 on <benchmark>" disagreeing with that benchmark's public
        leaderboard). RESOLVE each named resource against its HuggingFace dataset/model card
        or Papers-with-Code record (WebFetch/WebSearch — FACTS only; put the URL + access
        date in `description`); the reviewer judges the discrepancy. Anchor to the paper
        claim NAMING the resource (never the registry URL). severity major; critical if the
        mis-described resource IS the headline (the SOTA number that is the contribution).
        observability 0 (public-record contradiction) / 2 (the repo loads a different
        resource than named). FP (→ needs_external_check, never a guessed "wrong"): a
        declared subset/variant (ImageNet-100, a 10% split, a distilled/quantized model); a
        version difference (-21k vs -1k, v1 vs v2); an explicit redefinition; a stale /
        ambiguous registry or a leaderboard updated after submission. LANE: method
        described ≠ method evaluated is HP-METHOD-DRIFT (consistency-audit); a fabricated
        citation identity is HP-CITE-HALLUC — this is the named RESOURCE's identity vs its
        public record.

    OUTPUT: a single JSON array and NOTHING ELSE (schemas/finding.schema.json), same
    shape as the completeness prompt; set finding_id "BC0xx",
    skill "baseline-comparison-audit". An empty array [] is valid and honest. Set
    requires_external_check only when you genuinely cannot settle a point at this level.

ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
LEDGER="<abs path to claims.json>"
OUT="$(dirname "$LEDGER")/baseline-comparison-audit.findings.json"
# args: LEDGER OUT then each saved raw reviewer response file from Steps 3–4:
python3 - "$LEDGER" "$OUT" "<resp_completeness.md>" "<resp_fairness.md>" <<'PY'
import json, re, sys, os
ledger_path, out_path = sys.argv[1], sys.argv[2]
resp_paths = [p for p in sys.argv[3:] if p and os.path.isfile(p)]

def nw(s):                                   # mirror adjudicator _norm_ws (whitespace only)
    return " ".join((s or "").split())

OWNED = {"HP-MISSING-BASELINE", "HP-WEAK-BASELINE", "HP-SIG-OVERLAP", "HP-DELTA-ERROR",
         "HP-RESOURCE-IDENTITY-MISMATCH"}
ABOVE = {"critical", "major", "minor"}

ledger = json.load(open(ledger_path, encoding="utf-8"))
base = {c["claim_id"]: nw(c.get("text_span", "")) for c in ledger.get("claims", [])
        if c.get("claim_id")}

# best-effort dedup target: claim_ids the DETERMINISTIC consistency delta pass already
# flagged (consistency-audit.deterministic.findings.json, if it exists in this dir).
det_path = os.path.join(os.path.dirname(os.path.abspath(ledger_path)),
                        "consistency-audit.deterministic.findings.json")
det_delta = set()
if os.path.isfile(det_path):
    try:
        for f in json.load(open(det_path, encoding="utf-8")):
            if f.get("pattern_id") == "HP-DELTA-ERROR":
                for ev in f.get("evidence") or []:
                    if ev.get("claim_id"): det_delta.add(ev["claim_id"])
    except Exception:
        pass

proposed = []
for p in resp_paths:
    raw = open(p, encoding="utf-8").read()
    m = re.search(r"\[.*\]", raw, re.S)      # tolerate prose / code-fence wrapping
    try:
        chunk = json.loads(m.group(0) if m else raw)
        proposed += chunk.get("findings", []) if isinstance(chunk, dict) else chunk
    except Exception as e:
        print(f"WARN: could not parse {p}: {e}", file=sys.stderr)

kept, demoted, deduped, not_owned = [], 0, 0, 0
for f in proposed:
    if not isinstance(f, dict):
        continue
    f["skill"] = "baseline-comparison-audit"
    # ANCHOR: keep only evidence whose span is a verbatim ws-normalized substring of its claim
    anchored = [ev for ev in (f.get("evidence") or [])
                if ev.get("claim_id") in base and nw(ev.get("span", "")) and
                nw(ev["span"]) in base[ev["claim_id"]]]                 # span IN claim, not claim IN span
    f["evidence"] = anchored
    # owned-pattern gate: a non-owned pattern cannot rise above info here
    if f.get("pattern_id") and f["pattern_id"] not in OWNED and f.get("severity") in ABOVE:
        f["severity"] = "info"; f.setdefault("_demotions", []).append("pattern-not-owned"); not_owned += 1
    # best-effort delta dedup vs the deterministic single-sentence pass
    if f.get("pattern_id") == "HP-DELTA-ERROR" and f.get("severity") in ABOVE \
       and any(ev.get("claim_id") in det_delta for ev in anchored):
        f["severity"] = "info"; f.setdefault("_demotions", []).append("dup-of-deterministic-delta"); deduped += 1
    # ANCHOR gate: above-info needs >=1 anchored span
    if f.get("severity") in ABOVE and not anchored:
        f["severity"] = "info"; f.setdefault("_demotions", []).append("unanchored"); demoted += 1
    # OBSERVABILITY hygiene — MIRROR the adjudicator, fail-closed: an above-info finding whose
    # observability_level_required is missing/invalid demotes to info. NEVER silently default to 0
    # (that would let a forgotten level-2 config-only asymmetry survive an L0 run). type() not
    # isinstance() so JSON booleans (True==1) are rejected, exactly as adjudicate_findings.py does.
    olr = f.get("observability_level_required")
    if f.get("severity") in ABOVE and (type(olr) is not int or not (0 <= olr <= 3)):
        f["severity"] = "info"; f.setdefault("_demotions", []).append("undeclared-observability"); demoted += 1
    # FP-RISK hygiene — false_positive_risk is the REVIEWER's self-assessment (it drives the
    # adjudicator's cap); the executor never guesses it. Missing/invalid demotes to info, never a default.
    if f.get("severity") in ABOVE and f.get("false_positive_risk") not in ("low", "medium", "high"):
        f["severity"] = "info"; f.setdefault("_demotions", []).append("undeclared-fp-risk"); demoted += 1
    import os as _aris_os
    RESOLVED_MODEL = _aris_os.environ["ARIS_RESOLVED_MODEL"]          # exported by the executor from the call that ACTUALLY ran
    RESOLVED_REASONING = _aris_os.environ["ARIS_RESOLVED_REASONING"]  # (fail LOUD if unset — never stamp a target default)
    f["reviewer"] = {"model": RESOLVED_MODEL, "reasoning": RESOLVED_REASONING, "deterministic": False}
    # honest hand-off: needs_external_check carries no severity weight (adjudicate_findings.py has
    # no such gate, so the validator makes the claim true) — pin it to info, never drop it.
    if f.get("verdict_local") == "needs_external_check":
        f["requires_external_check"] = True
        if f.get("severity") in ABOVE:
            f["severity"] = "info"; f.setdefault("_demotions", []).append("needs-external-check-no-weight")
    kept.append(f)

for k, f in enumerate(kept, 1):                                        # one namespace, sequential
    f["finding_id"] = f"BC{k:03d}"

json.dump(kept, open(out_path, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
above = sum(1 for x in kept if x.get("severity") in ABOVE)
print(f"validated {len(kept)} baseline findings (above_info={above}; {demoted} demoted->info "
      f"(unanchored/undeclared); {not_owned} not-owned->info; {deduped} delta deduped) -> {out_path}")
PY

{
  "finding_id": "BC001",
  "skill": "baseline-comparison-audit",
  "pattern_id": "HP-MISSING-BASELINE",
  "title": "Headline SOTA claim omits an expected, pre-dating same-benchmark baseline",
  "description": "Claim C007 asserts 'state-of-the-art on GSM8K'. The expected-set facts list Self-Consistency CoT (2023; same benchmark; predates this submission per paperswithcode <url>, accessed <date>) as a standard strong baseline, but it appears in no comparison/baseline claim in the ledger. Discrepancy to verify: a 'state-of-the-art' headline that omits an obvious pre-dating baseline — confirm the claim survives once it is included, or whether the omission is justified.",
  "severity": "critical",
  "observability_level_required": 0,
  "evidence": [
    {"claim_id": "C007", "span": "achieves state-of-the-art accuracy on GSM8K",
     "location": {"file": "main.tex", "section": "abstract"}}
  ],
  "verdict_local": "fail",
  "requires_external_check": false,
  "false_positive_risk": "low",
  "recommended_reviewer_action": "Ask the authors to add Self-Consistency CoT (and any other pre-dating leaderboard baseline) or to justify its omission as concurrent/unavailable; confirm the SOTA claim survives the comparison."
}

{
  "finding_id": "BC002",
  "skill": "baseline-comparison-audit",
  "pattern_id": "HP-MISSING-BASELINE",
  "title": "Baseline completeness not settleable internally (off-profile benchmark)",
  "description": "Claim C011 claims to 'outperform all prior methods' on a niche benchmark not covered by the baseline profile, and the leaderboard search was inconclusive. Completeness cannot be decided from the available inputs — NOT an allegation of an omission, only a hand-off for a domain expert to confirm the expected baseline set.",
  "severity": "info",
  "observability_level_required": 0,
  "evidence": [
    {"claim_id": "C011", "span": "outperform all prior methods",
     "location": {"file": "main.tex", "section": "experiments"}}
  ],
  "verdict_local": "needs_external_check",
  "requires_external_check": true,
  "false_positive_risk": "high",
  "recommended_reviewer_action": "Have a domain expert enumerate the expected baseline set for this benchmark and check it against the paper's comparison table."
}

{
  "finding_id": "BC003",
  "skill": "baseline-comparison-audit",
  "pattern_id": "HP-WEAK-BASELINE",
  "title": "Compared rows use an unequal training budget",
  "description": "Claim C019 reports the proposed method (78.0) 'outperforms the baseline' (73.1) in Table 2. configs/ours.yaml sets epochs=100, n_seeds=5 (sha256 a1b2c3…) while configs/baseline.yaml sets epochs=10, n_seeds=1 — a 10x budget asymmetry in the compared rows, with no matched-budget / equal-budget ablation-as-baseline run reported. Discrepancy to verify: the 4.9-point gap may reflect compute, not method.",
  "severity": "major",
  "observability_level_required": 2,
  "evidence": [
    {"claim_id": "C019", "span": "our method (78.0) outperforms the baseline (73.1)",
     "location": {"file": "main.tex", "section": "table:2"}}
  ],
  "verdict_local": "warn",
  "reviewer": {"model": "gpt-5.6-sol", "reasoning": "xhigh", "thread_id": "<codex thread>", "deterministic": false},
  "requires_external_check": false,
  "false_positive_risk": "low",
  "recommended_reviewer_action": "Ask the authors for the baseline under the same epochs/seeds/backbone budget as the proposed method (the config delta is at configs/{ours,baseline}.yaml), or to document why the budgets differ."
}

RUNDIR="<the RUNDIR printed in Step 2>"   # e.g. .aris/traces/baseline-comparison-audit/<date>_run01
mkdir -p "$RUNDIR"                          # idempotent — the dir already exists from Step 2

.aris/traces/baseline-comparison-audit/<date>_run<NN>/
  run.meta.json                       # {skill, paper_id, run_level_L, profile_version:"0.1", domain, ledger_sha?, generated_at, not_applicable?}
  expected_baseline_set.json          # Step 2 external facts (leaderboard URL + dates + candidate set)
  001-completeness.request.json       # the EXACT Step-3 prompt sent (paths + ledger + sourced facts + checklist)
  001-completeness.response.md        # the FULL raw reviewer response (input to Step 5)
  001-completeness.meta.json          # {model:"gpt-5.6-sol", reasoning:"xhigh", thread_id, sandbox:"read-only"}
  002-fairness-significance.request.json
  002-fairness-significance.response.md
  002-fairness-significance.meta.json
  # 00N-comparison-<entry>.* for each per-entry fan-out call, if used

ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
LEDGER="<abs path to claims.json>"; D="$(dirname "$LEDGER")"
python3 "$ROOT/tools/adjudicate_findings.py" \
    --findings "$D/baseline-comparison-audit.findings.json" \
    --ledger "$LEDGER" \
    --paper-id "<PAPER_ID>" --observability-level <L> --taxonomy-version 0.5 \
    --out "$D/baseline.report.json" --md "$D/baseline.REPORT.md"
# prints e.g.: verdict=SOFT_FLAGS crit=0 maj=1 min=2 -> baseline.report.json, baseline.REPORT.md