KB-2AFA

blackbox_negative_suite.py

8 min read Revision 1

#!/usr/bin/env python3

============================================================================

FIX7 Recheck-9 Packet V3 — BLACK-BOX CLI NEGATIVE SUITE (R9-B6.2).

SUT = the actual canonicalizer CLI file (and, for verifier-lane cases,

the actual manifest_tool CLI file) executed as main with real

argv via the runpy harness — observed SystemExit code + stdout.

ORACLE = manifest_tool.CLI_ORACLE: STATIC spec pin (SSOT contract text).

Never generated from SUT output.

VERIFIER = this file: for every case it prints command / expected exit /

OBSERVED exit / stdout-marker checks / artifact+digest-suppression

checks, and exits 0 ONLY iff every observation matches the oracle.

An inferred "cli_exit_contract" is NOT admissible evidence anywhere in this

suite: every exit code printed below was OBSERVED from a real CLI run.

SAFE/OFFLINE: stdlib only; reads packet files; writes only OS temp dirs.

============================================================================

import os, sys, json, shutil, tempfile

sys.dont_write_bytecode = True ROOT = os.path.dirname(os.path.abspath(file)) sys.path.insert(0, ROOT) import manifest_tool as MT

CLI = os.path.join(ROOT, MT.CANON_CLI_REL) SSOT = os.path.join(ROOT, "evidence/canonicalizer-fix7-canon-v1-ssot.md") ORACLE = MT.CLI_ORACLE

RESULTS=[] def report(label, ok, expected, observed, detail=""): RESULTS.append(ok) print(f" [{'PASS' if ok else 'FAIL'}] {label}") print(f" expected_exit={expected} observed_exit={observed}{(' — '+detail) if detail else ''}")

def corpus_case(label, tmpdocs): """Run the REAL canonicalizer CLI on a corrupted corpus; assert the full fail-closed contract: exit 4, suppression marker, corpus_ok False, membership_frozen_ok False, zero aggregate-digest leak, no artifact.""" before=MT.snapshot_tree(tmpdocs) rc,out=MT.run_cli(CLI,["--produce",tmpdocs,SSOT]) after=MT.snapshot_tree(tmpdocs) checks={ "exit==4(oracle)": rc==ORACLE["produce_corpus_error_exit"], "stdout corpus_ok: False": "corpus_ok: False" in out, "stdout membership_frozen_ok: False": "membership_frozen_ok: False" in out, "suppression marker": ORACLE["suppression_marker"] in out, "no aggregate digest leak": not MT.agg_digest_leak(out), "no output artifact created": before==after, } ok=all(checks.values()) report(f"{label} [cmd: python3 {MT.CANON_CLI_REL} --produce <case-docs> <ssot.md>]", ok, ORACLE["produce_corpus_error_exit"], rc, "; ".join(f"{k}={'OK' if v else 'VIOLATED'}" for k,v in checks.items())) return rc,out

def edit_manifest(root, fn): p=os.path.join(root,"manifest.json") m=json.load(open(p)); fn(m) json.dump(m,open(p,"w"),indent=2); open(p,"a").write("\n")

def main(): print("BLACKBOX NEGATIVE SUITE V3 — every exit below is OBSERVED from a real CLI run") print(f"oracle (static spec pin): produce_ok={ORACLE['produce_ok_exit']} " f"corpus_error={ORACLE['produce_corpus_error_exit']} selftest_pass={ORACLE['selftest_pass_exit']}") base=tempfile.mkdtemp(prefix="fix7bbox-") try: docs=os.path.join(ROOT,"docs")

    # ---- P1 positive control: pinned corpus -> CLI exit 0, digests present
    rc,out=MT.run_cli(CLI,["--produce",docs,SSOT])
    report("P1 positive control: pinned 10-doc corpus -> produce CLI exit 0",
           rc==ORACLE["produce_ok_exit"] and "corpus_ok: True" in out
           and ORACLE["suppression_marker"] not in out and ORACLE["suppressed_token"] not in out,
           ORACLE["produce_ok_exit"], rc)

    # ---- P2 positive control: selftest CLI exit 0
    rc,out=MT.run_cli(CLI,[])
    report("P2 positive control: selftest CLI -> exit 0, ALL PASS: True",
           rc==ORACLE["selftest_pass_exit"] and "ALL PASS: True" in out,
           ORACLE["selftest_pass_exit"], rc)

    # ---- N1 missing active doc 05 (the decisive Codex V2 case)
    t=os.path.join(base,"missing"); shutil.copytree(docs,t)
    os.remove(os.path.join(t,"05-rollback-blueprint.md"))
    corpus_case("N1 missing active doc 05", t)

    # ---- N2 extra active doc
    t=os.path.join(base,"extra"); shutil.copytree(docs,t)
    open(os.path.join(t,"99-extra-doc.md"),"w").write("<!-- DOC_STATUS: ACTIVE_AUTHORITY -->\nx\n")
    corpus_case("N2 extra active doc 99", t)

    # ---- N3 invalid / extract-error doc
    t=os.path.join(base,"invalid"); shutil.copytree(docs,t)
    open(os.path.join(t,"03-gap-classification.md"),"w").write(
        "<!-- DOC_STATUS: ACTIVE_AUTHORITY -->\n<!-- SUPERSEDED_NON_AUTHORITY BEGIN -->\nbroken\n")
    corpus_case("N3 invalid active doc 03 (unbalanced fence)", t)

    # ---- N4 absent docs dir
    corpus_case("N4 absent docs directory", os.path.join(base,"nonexistent"))

    # ---- N5 duplicate active doc: N/A on case-insensitive FS + adjacent equivalent
    print("  [N/A ] N5 duplicate active doc ON DISK — case-insensitive filesystem cannot host")
    print("         two casefold-equal filenames; adjacent equivalent executed instead:")
    canon=MT.load_canon(ROOT)
    dup=canon.validate_corpus_listing(list(canon.DOCS)+["05-Rollback-Blueprint.md"])
    report("N5' adjacent equivalent: duplicate-listing validator rejects casefold duplicate",
           (not dup["ok"]) and len(dup["duplicate"])==1, "ok=False", f"ok={dup['ok']}")

    # ---- N6 forbidden-operation marker in seal-path code -> manifest_tool CLI --scan exit 1
    t=os.path.join(base,"scan"); shutil.copytree(ROOT,t,ignore=shutil.ignore_patterns(
        "rerun-out","__pycache__","*.pyc",".DS_Store"))
    open(os.path.join(t,"evidence/materialize_canonicalizer.py"),"a").write(
        "\nimport sub"+"process  # FORBIDDEN TEST VECTOR\n")
    rc,out=MT.run_cli(os.path.join(t,"manifest_tool.py"),["--scan"])
    report("N6 forbidden marker in seal-path code -> manifest_tool CLI --scan exit 1 (observed)",
           rc==1, 1, rc)

    # ---- N7 manifest tamper (authority literal) -> manifest_tool CLI --verify exit 1
    t=os.path.join(base,"tamper"); shutil.copytree(ROOT,t,ignore=shutil.ignore_patterns(
        "rerun-out","__pycache__","*.pyc",".DS_Store"))
    edit_manifest(t, lambda m: m["authority"]["forbidden_scope"].__setitem__("forbidden_operations_found",999))
    rc,out=MT.run_cli(os.path.join(t,"manifest_tool.py"),["--verify"])
    report("N7 manifest authority tamper -> manifest_tool CLI --verify exit 1 (observed)",
           rc==1, 1, rc)

    # ---- N8 candidate/rehearsal class claimed sealed -> manifest_tool CLI --verify exit 1
    t=os.path.join(base,"sealed"); shutil.copytree(ROOT,t,ignore=shutil.ignore_patterns(
        "rerun-out","__pycache__","*.pyc",".DS_Store"))
    edit_manifest(t, lambda m: m["authority"]["digest_classes"].__setitem__("canonicalizer_sha256","CODEX_SEALED"))
    rc,out=MT.run_cli(os.path.join(t,"manifest_tool.py"),["--verify"])
    report("N8 candidate digest class claimed CODEX_SEALED -> manifest_tool CLI --verify exit 1 (observed)",
           rc==1, 1, rc)
finally:
    shutil.rmtree(base, ignore_errors=True)

total=len(RESULTS); good=sum(RESULTS)
print(f"BLACKBOX_NEGATIVE_SUITE: {'PASS' if good==total else 'FAIL'} ({good}/{total} observed-behavior checks; "
      f"every exit code above was OBSERVED from a real CLI execution, none inferred)")
return 0 if good==total else 1

if name=="main": sys.exit(main())

Back to Knowledge Hub knowledge/dev/laws/tool-kiem-thu/packets/fix7-codex-recheck-9-2026-06-10/blackbox_negative_suite.py