blackbox_negative_suite.py
#!/usr/bin/env python3
============================================================================
FIX7 Recheck-9 Packet V3 — BLACK-BOX CLI NEGATIVE SUITE (R9-B6.2).
SUT = the actual canonicalizer CLI file (and, for verifier-lane cases,
the actual manifest_tool CLI file) executed as main with real
argv via the runpy harness — observed SystemExit code + stdout.
ORACLE = manifest_tool.CLI_ORACLE: STATIC spec pin (SSOT contract text).
Never generated from SUT output.
VERIFIER = this file: for every case it prints command / expected exit /
OBSERVED exit / stdout-marker checks / artifact+digest-suppression
checks, and exits 0 ONLY iff every observation matches the oracle.
An inferred "cli_exit_contract" is NOT admissible evidence anywhere in this
suite: every exit code printed below was OBSERVED from a real CLI run.
SAFE/OFFLINE: stdlib only; reads packet files; writes only OS temp dirs.
============================================================================
import os, sys, json, shutil, tempfile
sys.dont_write_bytecode = True ROOT = os.path.dirname(os.path.abspath(file)) sys.path.insert(0, ROOT) import manifest_tool as MT
CLI = os.path.join(ROOT, MT.CANON_CLI_REL) SSOT = os.path.join(ROOT, "evidence/canonicalizer-fix7-canon-v1-ssot.md") ORACLE = MT.CLI_ORACLE
RESULTS=[] def report(label, ok, expected, observed, detail=""): RESULTS.append(ok) print(f" [{'PASS' if ok else 'FAIL'}] {label}") print(f" expected_exit={expected} observed_exit={observed}{(' — '+detail) if detail else ''}")
def corpus_case(label, tmpdocs): """Run the REAL canonicalizer CLI on a corrupted corpus; assert the full fail-closed contract: exit 4, suppression marker, corpus_ok False, membership_frozen_ok False, zero aggregate-digest leak, no artifact.""" before=MT.snapshot_tree(tmpdocs) rc,out=MT.run_cli(CLI,["--produce",tmpdocs,SSOT]) after=MT.snapshot_tree(tmpdocs) checks={ "exit==4(oracle)": rc==ORACLE["produce_corpus_error_exit"], "stdout corpus_ok: False": "corpus_ok: False" in out, "stdout membership_frozen_ok: False": "membership_frozen_ok: False" in out, "suppression marker": ORACLE["suppression_marker"] in out, "no aggregate digest leak": not MT.agg_digest_leak(out), "no output artifact created": before==after, } ok=all(checks.values()) report(f"{label} [cmd: python3 {MT.CANON_CLI_REL} --produce <case-docs> <ssot.md>]", ok, ORACLE["produce_corpus_error_exit"], rc, "; ".join(f"{k}={'OK' if v else 'VIOLATED'}" for k,v in checks.items())) return rc,out
def edit_manifest(root, fn): p=os.path.join(root,"manifest.json") m=json.load(open(p)); fn(m) json.dump(m,open(p,"w"),indent=2); open(p,"a").write("\n")
def main(): print("BLACKBOX NEGATIVE SUITE V3 — every exit below is OBSERVED from a real CLI run") print(f"oracle (static spec pin): produce_ok={ORACLE['produce_ok_exit']} " f"corpus_error={ORACLE['produce_corpus_error_exit']} selftest_pass={ORACLE['selftest_pass_exit']}") base=tempfile.mkdtemp(prefix="fix7bbox-") try: docs=os.path.join(ROOT,"docs")
# ---- P1 positive control: pinned corpus -> CLI exit 0, digests present
rc,out=MT.run_cli(CLI,["--produce",docs,SSOT])
report("P1 positive control: pinned 10-doc corpus -> produce CLI exit 0",
rc==ORACLE["produce_ok_exit"] and "corpus_ok: True" in out
and ORACLE["suppression_marker"] not in out and ORACLE["suppressed_token"] not in out,
ORACLE["produce_ok_exit"], rc)
# ---- P2 positive control: selftest CLI exit 0
rc,out=MT.run_cli(CLI,[])
report("P2 positive control: selftest CLI -> exit 0, ALL PASS: True",
rc==ORACLE["selftest_pass_exit"] and "ALL PASS: True" in out,
ORACLE["selftest_pass_exit"], rc)
# ---- N1 missing active doc 05 (the decisive Codex V2 case)
t=os.path.join(base,"missing"); shutil.copytree(docs,t)
os.remove(os.path.join(t,"05-rollback-blueprint.md"))
corpus_case("N1 missing active doc 05", t)
# ---- N2 extra active doc
t=os.path.join(base,"extra"); shutil.copytree(docs,t)
open(os.path.join(t,"99-extra-doc.md"),"w").write("<!-- DOC_STATUS: ACTIVE_AUTHORITY -->\nx\n")
corpus_case("N2 extra active doc 99", t)
# ---- N3 invalid / extract-error doc
t=os.path.join(base,"invalid"); shutil.copytree(docs,t)
open(os.path.join(t,"03-gap-classification.md"),"w").write(
"<!-- DOC_STATUS: ACTIVE_AUTHORITY -->\n<!-- SUPERSEDED_NON_AUTHORITY BEGIN -->\nbroken\n")
corpus_case("N3 invalid active doc 03 (unbalanced fence)", t)
# ---- N4 absent docs dir
corpus_case("N4 absent docs directory", os.path.join(base,"nonexistent"))
# ---- N5 duplicate active doc: N/A on case-insensitive FS + adjacent equivalent
print(" [N/A ] N5 duplicate active doc ON DISK — case-insensitive filesystem cannot host")
print(" two casefold-equal filenames; adjacent equivalent executed instead:")
canon=MT.load_canon(ROOT)
dup=canon.validate_corpus_listing(list(canon.DOCS)+["05-Rollback-Blueprint.md"])
report("N5' adjacent equivalent: duplicate-listing validator rejects casefold duplicate",
(not dup["ok"]) and len(dup["duplicate"])==1, "ok=False", f"ok={dup['ok']}")
# ---- N6 forbidden-operation marker in seal-path code -> manifest_tool CLI --scan exit 1
t=os.path.join(base,"scan"); shutil.copytree(ROOT,t,ignore=shutil.ignore_patterns(
"rerun-out","__pycache__","*.pyc",".DS_Store"))
open(os.path.join(t,"evidence/materialize_canonicalizer.py"),"a").write(
"\nimport sub"+"process # FORBIDDEN TEST VECTOR\n")
rc,out=MT.run_cli(os.path.join(t,"manifest_tool.py"),["--scan"])
report("N6 forbidden marker in seal-path code -> manifest_tool CLI --scan exit 1 (observed)",
rc==1, 1, rc)
# ---- N7 manifest tamper (authority literal) -> manifest_tool CLI --verify exit 1
t=os.path.join(base,"tamper"); shutil.copytree(ROOT,t,ignore=shutil.ignore_patterns(
"rerun-out","__pycache__","*.pyc",".DS_Store"))
edit_manifest(t, lambda m: m["authority"]["forbidden_scope"].__setitem__("forbidden_operations_found",999))
rc,out=MT.run_cli(os.path.join(t,"manifest_tool.py"),["--verify"])
report("N7 manifest authority tamper -> manifest_tool CLI --verify exit 1 (observed)",
rc==1, 1, rc)
# ---- N8 candidate/rehearsal class claimed sealed -> manifest_tool CLI --verify exit 1
t=os.path.join(base,"sealed"); shutil.copytree(ROOT,t,ignore=shutil.ignore_patterns(
"rerun-out","__pycache__","*.pyc",".DS_Store"))
edit_manifest(t, lambda m: m["authority"]["digest_classes"].__setitem__("canonicalizer_sha256","CODEX_SEALED"))
rc,out=MT.run_cli(os.path.join(t,"manifest_tool.py"),["--verify"])
report("N8 candidate digest class claimed CODEX_SEALED -> manifest_tool CLI --verify exit 1 (observed)",
rc==1, 1, rc)
finally:
shutil.rmtree(base, ignore_errors=True)
total=len(RESULTS); good=sum(RESULTS)
print(f"BLACKBOX_NEGATIVE_SUITE: {'PASS' if good==total else 'FAIL'} ({good}/{total} observed-behavior checks; "
f"every exit code above was OBSERVED from a real CLI execution, none inferred)")
return 0 if good==total else 1
if name=="main": sys.exit(main())