Problem class: false-green-gate-unmeasured-metric
I reproduced the checker, the failure modes, and the fix locally, then wrote the deliverable to ~/solution-false-green-gate-unmeasured-metric.md. The code in the report was diff-checked against the tested files and its test suite passes.
Below is the solution.
check_docs_drift.py certified test counts it never measuredProblem class: false-green-gate-unmeasured-metric
Repo: totalwindupflightsystems/gitreins @ a1c4fc5
Files: scripts/check_docs_drift.py, .github/workflows/ci.yml, README.md, CONTRIBUTING.md, tests/test_docs_drift.py
The gate printed:
docs drift check OK: version 0.14.0 matches README banner; README test counts consistent (1842 tests / 52 test files)
exit 0 — while python -m pytest --collect-only -q --override-ini=addopts= collected 1864 tests / 53 files. It never ran pytest.
Two defects combined into a false green:
N tests pass == N tests across). Internal consistency was reported under the vocabulary of external truth. The word consistent was unreachable from any measurement.A hidden second failure: CONTRIBUTING.md:35 reads Currently **1864 tests across 53\ntest files**. A literal-space regex silently skips that wrapped claim.
Fix principle: the checker runs the authority itself, parses it exactly as CI does, compares every documented claim against the measurement, and treats unmeasurable as FAIL. One implementation for local and CI.
scripts/check_docs_drift.py#!/usr/bin/env python3
"""Check documentation drift: test counts (LIVE) and version banner.
The test-count claim is checked against a live pytest collection performed by
THIS script, using exactly the command and parse the CI job uses. There is no
static path that can print "consistent": if the collection cannot be measured
the gate fails closed (exit 1).
"""
from __future__ import annotations
import argparse
import re
import subprocess
import sys
from pathlib import Path
# --- the authority: same command CI runs, pytest never imported -------------- #
COLLECT_CMD = [
sys.executable,
"-m",
"pytest",
"--collect-only",
"-q",
"--override-ini=addopts=",
]
TEST_PATH_RE = re.compile(r"^tests/[a-zA-Z0-9_./-]+\.py")
TOTAL_RE = re.compile(r"^(\d+)\b")
DOCS = ("README.md", "CONTRIBUTING.md")
# whitespace-tolerant (\s+) so a claim wrapped across a newline is still found
COUNT_CLAIMS = (
re.compile(r"(\d+)\s+tests?\s+pass\b"),
re.compile(r"(\d+)\s+tests?\s+across\b"),
)
FILE_CLAIMS = (re.compile(r"(\d+)\s+test\s+files?\b"),)
VERSION_RE = re.compile(r"__version__\s*=\s*[\"']([^\"']+)[\"']")
README_VERSION_RE = re.compile(r"version\s+(\d+\.\d+\.\d+)")
class MeasurementError(RuntimeError):
"""Raised when live collection cannot be measured - always a FAIL."""
def _line_of(text: str, offset: int) -> int:
return text.count("\n", 0, offset) + 1
def collect_live_counts(repo_root: Path) -> tuple[int, int]:
"""Run the CI collection command; return (test_count, test_file_count)."""
try:
proc = subprocess.run(
COLLECT_CMD,
cwd=str(repo_root),
capture_output=True,
text=True,
)
except OSError as exc: # pragma: no cover - only on broken envs
raise MeasurementError(f"live test collection could not be measured: {exc}")
if proc.returncode != 0:
detail = [ln for ln in (proc.stderr or proc.stdout).splitlines() if ln.strip()]
reason = detail[-1] if detail else "no output"
raise MeasurementError(
f"live test collection could not be measured "
f"(pytest exit {proc.returncode}): {reason}"
)
lines = [ln for ln in proc.stdout.splitlines() if ln.strip()]
if not lines:
raise MeasurementError("live test collection could not be measured: no output")
m = TOTAL_RE.match(lines[-1].strip())
if not m:
raise MeasurementError(
f"live test collection could not be measured: unparseable total "
f"{lines[-1]!r}"
)
count = int(m.group(1))
files = {
pm.group(0)
for ln in proc.stdout.splitlines()
if (pm := TEST_PATH_RE.match(ln)) is not None
}
return count, len(files)
def documented_claims(repo_root: Path):
"""Yield (file, line, kind, documented_value) for every count claim."""
for name in DOCS:
path = repo_root / name
if not path.exists():
continue
text = path.read_text(encoding="utf-8")
for pattern in COUNT_CLAIMS:
for match in pattern.finditer(text):
yield name, _line_of(text, match.start()), "tests", int(match.group(1))
for pattern in FILE_CLAIMS:
for match in pattern.finditer(text):
yield name, _line_of(text, match.start()), "files", int(match.group(1))
def read_package_version(repo_root: Path):
init = repo_root / "gitreins" / "__init__.py"
if not init.exists():
return None
m = VERSION_RE.search(init.read_text(encoding="utf-8"))
return m.group(1) if m else None
def check_version(repo_root: Path):
version = read_package_version(repo_root)
if version is None:
return None
readme = repo_root / "README.md"
if not readme.exists():
return f"FAIL: README.md missing; cannot verify version {version}."
if not README_VERSION_RE.search(readme.read_text(encoding="utf-8")):
return (
f"FAIL: README.md version banner drift - package is {version} "
f"but README.md has no matching version banner."
)
return None
def check_docs_drift(repo_root: Path, static_only: bool = False) -> int:
version_msg = check_version(repo_root)
if version_msg:
print(version_msg)
return 1
if static_only:
print(
"docs drift check (static): version banner OK; live test collection "
"NOT compared - documented test counts are UNVERIFIED."
)
return 0
try:
count, files = collect_live_counts(repo_root)
except MeasurementError as exc:
print(f"FAIL: {exc}")
return 1
failed = False
for name, line, kind, documented in documented_claims(repo_root):
observed = count if kind == "tests" else files
if documented == observed:
continue
failed = True
noun = "tests" if kind == "tests" else "test files"
print(
f"FAIL: {name}:{line} test-count drift - documents {documented} "
f"{noun} but pytest collects {observed} {noun} in {files} files."
)
if failed:
return 1
print(
f"docs drift check OK: version banner matches; live README/CONTRIBUTING "
f"test counts consistent ({count} tests / {files} test files)."
)
return 0
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--static", action="store_true", help="skip live collection")
parser.add_argument("repo_root", nargs="?", default=".")
args = parser.parse_args(argv)
return check_docs_drift(Path(args.repo_root).resolve(), static_only=args.static)
if __name__ == "__main__":
sys.exit(main())
Key points:
- collect_live_counts() runs [sys.executable, "-m", "pytest", "--collect-only", "-q", "--override-ini=addopts=] with cwd=repo_root; no pytest import.
- Same parse as CI: COUNT = leading integer of the last non-empty line (^(\d+)\b); FILES = unique ^tests/[a-zA-Z0-9_./-]+\.py paths.
- \s+ whole-text claim regexes cover the 53\ntest files wrap; line numbers from match offsets yield FILE:LINE.
- Fail closed: missing interpreter, non-zero exit (pytest exits 5 on empty collection), no output, or unparseable total -> exit 1 live test collection could not be measured. consistent is unreachable without a measurement.
- --static / static_only=True says NOT compared / UNVERIFIED.
- Failures name both numbers: FAIL: README.md:14 test-count drift - documents 1842 tests but pytest collects 1872 tests in 53 files.
# BEFORE: a second implementation beside the script
- name: Docs drift
run: |
python scripts/check_docs_drift.py
python -m pytest --collect-only -q --override-ini=addopts= > /tmp/collect.txt
COUNT=$(tail -n1 /tmp/collect.txt | grep -oE '^[0-9]+')
FILES=$(grep -oE '^tests/[a-zA-Z0-9_./-]+\.py' /tmp/collect.txt | sort -u | wc -l)
# ...compare with README claims...
# AFTER: the script is the single implementation, locally and in CI
- name: Docs drift
run: python scripts/check_docs_drift.py
tests/test_docs_drift.py)Each test builds its own fixture tree with a real tests/ directory, so the suite never reads the surrounding repo's live counts.
"""Hermetic tests for scripts/check_docs_drift.py.
Every test builds its own fixture tree with a REAL tests/ directory, so the
suite never reads the surrounding repository's live counts. Each test proves a
fail path on a tree it owns; the real-tree prove-and-restore is a CI/manual step.
"""
import importlib.util
import io
from contextlib import redirect_stdout
from pathlib import Path
MOD = Path(__file__).resolve().parents[1] / "scripts" / "check_docs_drift.py"
def load_checker():
spec = importlib.util.spec_from_file_location("check_docs_drift", MOD)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def build_repo(root: Path, *, tests: int = 5, files: int = 2,
readme_tests: int = 5, contrib_tests: int = 5,
contrib_newline: bool = False) -> Path:
(root / "gitreins").mkdir(parents=True, exist_ok=True)
(root / "gitreins" / "__init__.py").write_text('__version__ = "0.14.0"\n')
(root / "README.md").write_text(
"# docs\n\nversion 0.14.0\n\nEverything works: "
f"{readme_tests} tests pass.\n"
)
wrapped = "\n" if contrib_newline else " "
(root / "CONTRIBUTING.md").write_text(
"# Contributing\n\nCurrently "
f"**{contrib_tests} tests across {files}{wrapped}test files**.\n"
)
tests_dir = root / "tests"
tests_dir.mkdir(exist_ok=True)
per_file = max(1, tests // files)
remaining = tests
for i in range(files):
n = min(per_file if i < files - 1 else remaining, remaining)
body = "\n".join(f"def test_{i}_{j}(): assert True" for j in range(n))
(tests_dir / f"test_{i}.py").write_text(body + "\n")
remaining -= n
return root
def run(module, root, **kwargs):
buf = io.StringIO()
with redirect_stdout(buf):
code = module.check_docs_drift(root, **kwargs)
return code, buf.getvalue()
def test_ok(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path))
assert code == 0
assert "consistent" in out
assert "5 tests / 2 test files" in out
def test_readme_drift_names_file_line_and_both_numbers(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path, readme_tests=1842))
assert code == 1
assert "README.md:5" in out
assert "1842" in out and "5" in out
def test_wrapped_contributing_claim_is_covered(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path, contrib_tests=1864,
contrib_newline=True))
assert code == 1
assert "CONTRIBUTING.md:3" in out
assert "1864" in out
def test_unmeasurable_fails_closed(tmp_path):
root = build_repo(tmp_path)
(root / "tests" / "test_0.py").unlink()
(root / "tests" / "test_1.py").unlink()
code, out = run(load_checker(), root)
assert code == 1
assert "could not be measured" in out
assert "consistent" not in out
def test_static_never_says_consistent(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path), static_only=True)
assert code == 0
assert "UNVERIFIED" in out
assert "consistent" not in out
Verified against a faithful fixture repo. Real-repo values from the fix record (1872 tests / 53 files) behave identically.
$ python3 scripts/check_docs_drift.py .
docs drift check OK: version banner matches; live README/CONTRIBUTING test counts consistent (10 tests / 3 test files).
EXIT=0
# force README drift, then restore
$ sed -i 's/10 tests pass/1842 tests pass/' README.md
$ python3 scripts/check_docs_drift.py .
FAIL: README.md:7 test-count drift - documents 1842 tests but pytest collects 10 tests in 3 files.
EXIT=1
$ sed -i 's/1842 tests pass/10 tests pass/' README.md
... consistent (10 tests / 3 test files). EXIT=0
# newline-split CONTRIBUTING claim is covered
$ sed -i 's/\*\*10 tests across 3/**1864 tests across 53/' CONTRIBUTING.md
$ python3 scripts/check_docs_drift.py .
FAIL: CONTRIBUTING.md:4 test-count drift - documents 1864 tests but pytest collects 10 tests in 3 files.
FAIL: CONTRIBUTING.md:4 test-count drift - documents 53 test files but pytest collects 3 test files in 3 files.
EXIT=1
# unmeasurable -> FAIL, never "consistent"
$ python3 scripts/check_docs_drift.py empty
FAIL: live test collection could not be measured (pytest exit 5): no tests collected in 0.00s
EXIT=1
# static wording is honest
$ python3 scripts/check_docs_drift.py --static empty
docs drift check (static): version banner OK; live test collection NOT compared - documented test counts are UNVERIFIED.
EXIT=0
# hermetic tests stay green while the surrounding repo README is broken
$ python3 -m pytest tests/test_docs_drift.py -q
5 passed
$ python3 -m pytest -q
10 passed
On the real repo the reviewing foreman observed: live collect 1872 tests / 53 files; exit 0 naming those numbers; README.md:14 forced stale -> exit 1 naming README.md:14 with 1842 and 1872; restore -> exit 0; CONTRIBUTING.md:35 forced stale -> exit 1 naming CONTRIBUTING.md:35; fixture with no tests/ -> exit 1 live test collection could not be measured; --static -> exit 0 with not-compared wording.
For any gate: - (a) The checker must run the authority command itself and parse it the same way CI does. - (b) One implementation shared by local and CI — never a bash block beside a script. - (c) Unmeasurable = FAIL, never a pass. - (d) The success message must be disable-able in static mode with honest wording; the strong word is unreachable without a measurement. - (e) Prove the fail path on the real tree and restore it — a fixture-only fail test can pass while the real path is vacuous.
Full human-readable copy: ~/solution-false-green-gate-unmeasured-metric.md. Working reproduction and passing suite: ~/repro/repo (python3 -m pytest -q -> 10 passed).
# Evidence - Problem class: false-green-gate-unmeasured-metric - Model: openrouter/deepseek/deepseek-v4.1-flash - Solved: 2026-09-19T09:46:16.697Z - Verification: solution produced by pi in sandbox; see signatures.json
{"description": "PROBLEM (measured 2026-09-19, gitreins-poc tick 308/309): a local gate script (scripts/check_docs_drift.py) printed 'README test counts consistent (1842 tests / 52 test files)' and exited 0 while the ACTUAL live collection was 1864 tests / 53 files. It never ran pytest: its 'counts' check only asserted that README's own claims agreed with EACH OTHER (one distinct N per phrase, 'N tests pass' == 'N tests across'), and the row-level live comparison existed ONLY as a bash block in one CI step. Class: a checker that names a metric in its success message but never measures that metric - it certifies internal consistency under the vocabulary of external truth. Consequence: a worker briefed to run the repo's own gate got a false green, and a 6-commit backlog pushed on that green would have failed CI; the drift was caught only because CI recomputed the number.\n\nROOT CAUSE: the authority for the number (the live collection command) and the code that compares against it lived in two places (script vs CI bash), and the script's message asserted the stronger claim. Split implementations of one gate = the weaker side inherits the stronger side's credibility.\n\nFIX (landed, verified): (1) Move the live comparison into the checker: collect_live_counts(repo_root) runs the exact CI command [sys.executable, '-m', 'pytest', '--collect-only', '-q', '--override-ini=addopts='] via subprocess with cwd=repo_root (stdlib only, pytest never imported), parses COUNT as the leading integer of the last non-empty stdout line and FILES as the count of unique paths matching the CI regex ^tests/[a-zA-Z0-9_./-]+\\.py - the SAME parse the CI bash used, so the two cannot disagree. (2) Compare every documented claim in README.md AND CONTRIBUTING.md against the measurement, matching with whitespace-tolerant regexes (\\s+) over the whole text and deriving line numbers from match offsets, because a claim can wrap across a newline (CONTRIBUTING.md:35 'Currently **1864 tests across 53\\ntest files**' is exactly that shape; a literal-space regex silently skips it and ships an under-covering gate). (3) Fail closed on an unmeasurable collection: missing pytest, non-zero exit (pytest exits 5 on an empty collection) or an unparseable total returns exit 1 with the reason - a gate must never print 'consistent' without a measurement. Failure messages name FILE:LINE plus BOTH numbers: 'FAIL: README.md:14 test-count drift - documents 1842 tests pass but pytest collects 1872 tests in 53 files. Update README.md:14.'. (4) Delete the duplicated CI bash; the step keeps its name and runs the script (one implementation). (5) Add --static (and static_only=True) for environments with no pytest: it skips the live comparison and its message says the live collection was NOT compared and the numbers are UNVERIFIED - the word 'consistent' is unreachable without a measurement.\n\nVERIFICATION (all re-run by the reviewing foreman, not the author): live collect 1872 tests / 53 files; script exits 0 naming those numbers; README.md:14 forced to '1842 tests pass' -> exit 1 naming README.md:14 with 1842 and 1872, restored -> exit 0; CONTRIBUTING.md:35 forced stale -> exit 1 naming CONTRIBUTING.md:35 (proves the newline-split claim is actually covered); fixture repo with no tests/ -> exit 1 'live test collection could not be measured', never 'consistent'; --static -> exit 0 with the not-compared wording; the checker's own test file (15 tests, hermetic fixture trees with real tests/ dirs) stays green while the repo README is deliberately broken, proving the tests do not read live counts; full suite 1859 passed / 13 skipped.\n\nREUSABLE RULE: for any gate, (a) the checker must run the authority command itself and parse it the same way CI does, (b) one implementation shared by local and CI - never a bash block beside a script, (c) unmeasurable = FAIL, never a pass, (d) the success message must be disable-able in static mode with honest wording, and (e) prove the fail path by deliberately introducing the drift on the REAL tree and restoring it, because a fail-path test against a fixture can pass while the real path is still vacuous.", "environment": "Python stdlib CLI checker + GitHub Actions CI (gitreins-poc / totalwindupflightsystems/gitreins, main a1c4fc5, py3.10+ stdlib only)", "language": "python", "model": "openrouter/deepseek/deepseek-v4.1-flash", "problem_class": "false-green-gate-unmeasured-metric", "provider": "openrouter", "solved_at": "2026-09-19T09:46:16.697Z", "version": "gitreins a1c4fc5"}I reproduced the checker, the failure modes, and the fix locally, then wrote the deliverable to ~/solution-false-green-gate-unmeasured-metric.md. The code in the report was diff-checked against the tested files and its test suite passes.
Below is the solution.
check_docs_drift.py certified test counts it never measuredProblem class: false-green-gate-unmeasured-metric
Repo: totalwindupflightsystems/gitreins @ a1c4fc5
Files: scripts/check_docs_drift.py, .github/workflows/ci.yml, README.md, CONTRIBUTING.md, tests/test_docs_drift.py
The gate printed:
docs drift check OK: version 0.14.0 matches README banner; README test counts consistent (1842 tests / 52 test files)
exit 0 — while python -m pytest --collect-only -q --override-ini=addopts= collected 1864 tests / 53 files. It never ran pytest.
Two defects combined into a false green:
N tests pass == N tests across). Internal consistency was reported under the vocabulary of external truth. The word consistent was unreachable from any measurement.A hidden second failure: CONTRIBUTING.md:35 reads Currently **1864 tests across 53\ntest files**. A literal-space regex silently skips that wrapped claim.
Fix principle: the checker runs the authority itself, parses it exactly as CI does, compares every documented claim against the measurement, and treats unmeasurable as FAIL. One implementation for local and CI.
scripts/check_docs_drift.py#!/usr/bin/env python3
"""Check documentation drift: test counts (LIVE) and version banner.
The test-count claim is checked against a live pytest collection performed by
THIS script, using exactly the command and parse the CI job uses. There is no
static path that can print "consistent": if the collection cannot be measured
the gate fails closed (exit 1).
"""
from __future__ import annotations
import argparse
import re
import subprocess
import sys
from pathlib import Path
# --- the authority: same command CI runs, pytest never imported -------------- #
COLLECT_CMD = [
sys.executable,
"-m",
"pytest",
"--collect-only",
"-q",
"--override-ini=addopts=",
]
TEST_PATH_RE = re.compile(r"^tests/[a-zA-Z0-9_./-]+\.py")
TOTAL_RE = re.compile(r"^(\d+)\b")
DOCS = ("README.md", "CONTRIBUTING.md")
# whitespace-tolerant (\s+) so a claim wrapped across a newline is still found
COUNT_CLAIMS = (
re.compile(r"(\d+)\s+tests?\s+pass\b"),
re.compile(r"(\d+)\s+tests?\s+across\b"),
)
FILE_CLAIMS = (re.compile(r"(\d+)\s+test\s+files?\b"),)
VERSION_RE = re.compile(r"__version__\s*=\s*[\"']([^\"']+)[\"']")
README_VERSION_RE = re.compile(r"version\s+(\d+\.\d+\.\d+)")
class MeasurementError(RuntimeError):
"""Raised when live collection cannot be measured - always a FAIL."""
def _line_of(text: str, offset: int) -> int:
return text.count("\n", 0, offset) + 1
def collect_live_counts(repo_root: Path) -> tuple[int, int]:
"""Run the CI collection command; return (test_count, test_file_count)."""
try:
proc = subprocess.run(
COLLECT_CMD,
cwd=str(repo_root),
capture_output=True,
text=True,
)
except OSError as exc: # pragma: no cover - only on broken envs
raise MeasurementError(f"live test collection could not be measured: {exc}")
if proc.returncode != 0:
detail = [ln for ln in (proc.stderr or proc.stdout).splitlines() if ln.strip()]
reason = detail[-1] if detail else "no output"
raise MeasurementError(
f"live test collection could not be measured "
f"(pytest exit {proc.returncode}): {reason}"
)
lines = [ln for ln in proc.stdout.splitlines() if ln.strip()]
if not lines:
raise MeasurementError("live test collection could not be measured: no output")
m = TOTAL_RE.match(lines[-1].strip())
if not m:
raise MeasurementError(
f"live test collection could not be measured: unparseable total "
f"{lines[-1]!r}"
)
count = int(m.group(1))
files = {
pm.group(0)
for ln in proc.stdout.splitlines()
if (pm := TEST_PATH_RE.match(ln)) is not None
}
return count, len(files)
def documented_claims(repo_root: Path):
"""Yield (file, line, kind, documented_value) for every count claim."""
for name in DOCS:
path = repo_root / name
if not path.exists():
continue
text = path.read_text(encoding="utf-8")
for pattern in COUNT_CLAIMS:
for match in pattern.finditer(text):
yield name, _line_of(text, match.start()), "tests", int(match.group(1))
for pattern in FILE_CLAIMS:
for match in pattern.finditer(text):
yield name, _line_of(text, match.start()), "files", int(match.group(1))
def read_package_version(repo_root: Path):
init = repo_root / "gitreins" / "__init__.py"
if not init.exists():
return None
m = VERSION_RE.search(init.read_text(encoding="utf-8"))
return m.group(1) if m else None
def check_version(repo_root: Path):
version = read_package_version(repo_root)
if version is None:
return None
readme = repo_root / "README.md"
if not readme.exists():
return f"FAIL: README.md missing; cannot verify version {version}."
if not README_VERSION_RE.search(readme.read_text(encoding="utf-8")):
return (
f"FAIL: README.md version banner drift - package is {version} "
f"but README.md has no matching version banner."
)
return None
def check_docs_drift(repo_root: Path, static_only: bool = False) -> int:
version_msg = check_version(repo_root)
if version_msg:
print(version_msg)
return 1
if static_only:
print(
"docs drift check (static): version banner OK; live test collection "
"NOT compared - documented test counts are UNVERIFIED."
)
return 0
try:
count, files = collect_live_counts(repo_root)
except MeasurementError as exc:
print(f"FAIL: {exc}")
return 1
failed = False
for name, line, kind, documented in documented_claims(repo_root):
observed = count if kind == "tests" else files
if documented == observed:
continue
failed = True
noun = "tests" if kind == "tests" else "test files"
print(
f"FAIL: {name}:{line} test-count drift - documents {documented} "
f"{noun} but pytest collects {observed} {noun} in {files} files."
)
if failed:
return 1
print(
f"docs drift check OK: version banner matches; live README/CONTRIBUTING "
f"test counts consistent ({count} tests / {files} test files)."
)
return 0
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--static", action="store_true", help="skip live collection")
parser.add_argument("repo_root", nargs="?", default=".")
args = parser.parse_args(argv)
return check_docs_drift(Path(args.repo_root).resolve(), static_only=args.static)
if __name__ == "__main__":
sys.exit(main())
Key points:
- collect_live_counts() runs [sys.executable, "-m", "pytest", "--collect-only", "-q", "--override-ini=addopts=] with cwd=repo_root; no pytest import.
- Same parse as CI: COUNT = leading integer of the last non-empty line (^(\d+)\b); FILES = unique ^tests/[a-zA-Z0-9_./-]+\.py paths.
- \s+ whole-text claim regexes cover the 53\ntest files wrap; line numbers from match offsets yield FILE:LINE.
- Fail closed: missing interpreter, non-zero exit (pytest exits 5 on empty collection), no output, or unparseable total -> exit 1 live test collection could not be measured. consistent is unreachable without a measurement.
- --static / static_only=True says NOT compared / UNVERIFIED.
- Failures name both numbers: FAIL: README.md:14 test-count drift - documents 1842 tests but pytest collects 1872 tests in 53 files.
# BEFORE: a second implementation beside the script
- name: Docs drift
run: |
python scripts/check_docs_drift.py
python -m pytest --collect-only -q --override-ini=addopts= > /tmp/collect.txt
COUNT=$(tail -n1 /tmp/collect.txt | grep -oE '^[0-9]+')
FILES=$(grep -oE '^tests/[a-zA-Z0-9_./-]+\.py' /tmp/collect.txt | sort -u | wc -l)
# ...compare with README claims...
# AFTER: the script is the single implementation, locally and in CI
- name: Docs drift
run: python scripts/check_docs_drift.py
tests/test_docs_drift.py)Each test builds its own fixture tree with a real tests/ directory, so the suite never reads the surrounding repo's live counts.
"""Hermetic tests for scripts/check_docs_drift.py.
Every test builds its own fixture tree with a REAL tests/ directory, so the
suite never reads the surrounding repository's live counts. Each test proves a
fail path on a tree it owns; the real-tree prove-and-restore is a CI/manual step.
"""
import importlib.util
import io
from contextlib import redirect_stdout
from pathlib import Path
MOD = Path(__file__).resolve().parents[1] / "scripts" / "check_docs_drift.py"
def load_checker():
spec = importlib.util.spec_from_file_location("check_docs_drift", MOD)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def build_repo(root: Path, *, tests: int = 5, files: int = 2,
readme_tests: int = 5, contrib_tests: int = 5,
contrib_newline: bool = False) -> Path:
(root / "gitreins").mkdir(parents=True, exist_ok=True)
(root / "gitreins" / "__init__.py").write_text('__version__ = "0.14.0"\n')
(root / "README.md").write_text(
"# docs\n\nversion 0.14.0\n\nEverything works: "
f"{readme_tests} tests pass.\n"
)
wrapped = "\n" if contrib_newline else " "
(root / "CONTRIBUTING.md").write_text(
"# Contributing\n\nCurrently "
f"**{contrib_tests} tests across {files}{wrapped}test files**.\n"
)
tests_dir = root / "tests"
tests_dir.mkdir(exist_ok=True)
per_file = max(1, tests // files)
remaining = tests
for i in range(files):
n = min(per_file if i < files - 1 else remaining, remaining)
body = "\n".join(f"def test_{i}_{j}(): assert True" for j in range(n))
(tests_dir / f"test_{i}.py").write_text(body + "\n")
remaining -= n
return root
def run(module, root, **kwargs):
buf = io.StringIO()
with redirect_stdout(buf):
code = module.check_docs_drift(root, **kwargs)
return code, buf.getvalue()
def test_ok(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path))
assert code == 0
assert "consistent" in out
assert "5 tests / 2 test files" in out
def test_readme_drift_names_file_line_and_both_numbers(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path, readme_tests=1842))
assert code == 1
assert "README.md:5" in out
assert "1842" in out and "5" in out
def test_wrapped_contributing_claim_is_covered(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path, contrib_tests=1864,
contrib_newline=True))
assert code == 1
assert "CONTRIBUTING.md:3" in out
assert "1864" in out
def test_unmeasurable_fails_closed(tmp_path):
root = build_repo(tmp_path)
(root / "tests" / "test_0.py").unlink()
(root / "tests" / "test_1.py").unlink()
code, out = run(load_checker(), root)
assert code == 1
assert "could not be measured" in out
assert "consistent" not in out
def test_static_never_says_consistent(tmp_path):
code, out = run(load_checker(), build_repo(tmp_path), static_only=True)
assert code == 0
assert "UNVERIFIED" in out
assert "consistent" not in out
Verified against a faithful fixture repo. Real-repo values from the fix record (1872 tests / 53 files) behave identically.
$ python3 scripts/check_docs_drift.py .
docs drift check OK: version banner matches; live README/CONTRIBUTING test counts consistent (10 tests / 3 test files).
EXIT=0
# force README drift, then restore
$ sed -i 's/10 tests pass/1842 tests pass/' README.md
$ python3 scripts/check_docs_drift.py .
FAIL: README.md:7 test-count drift - documents 1842 tests but pytest collects 10 tests in 3 files.
EXIT=1
$ sed -i 's/1842 tests pass/10 tests pass/' README.md
... consistent (10 tests / 3 test files). EXIT=0
# newline-split CONTRIBUTING claim is covered
$ sed -i 's/\*\*10 tests across 3/**1864 tests across 53/' CONTRIBUTING.md
$ python3 scripts/check_docs_drift.py .
FAIL: CONTRIBUTING.md:4 test-count drift - documents 1864 tests but pytest collects 10 tests in 3 files.
FAIL: CONTRIBUTING.md:4 test-count drift - documents 53 test files but pytest collects 3 test files in 3 files.
EXIT=1
# unmeasurable -> FAIL, never "consistent"
$ python3 scripts/check_docs_drift.py empty
FAIL: live test collection could not be measured (pytest exit 5): no tests collected in 0.00s
EXIT=1
# static wording is honest
$ python3 scripts/check_docs_drift.py --static empty
docs drift check (static): version banner OK; live test collection NOT compared - documented test counts are UNVERIFIED.
EXIT=0
# hermetic tests stay green while the surrounding repo README is broken
$ python3 -m pytest tests/test_docs_drift.py -q
5 passed
$ python3 -m pytest -q
10 passed
On the real repo the reviewing foreman observed: live collect 1872 tests / 53 files; exit 0 naming those numbers; README.md:14 forced stale -> exit 1 naming README.md:14 with 1842 and 1872; restore -> exit 0; CONTRIBUTING.md:35 forced stale -> exit 1 naming CONTRIBUTING.md:35; fixture with no tests/ -> exit 1 live test collection could not be measured; --static -> exit 0 with not-compared wording.
For any gate: - (a) The checker must run the authority command itself and parse it the same way CI does. - (b) One implementation shared by local and CI — never a bash block beside a script. - (c) Unmeasurable = FAIL, never a pass. - (d) The success message must be disable-able in static mode with honest wording; the strong word is unreachable without a measurement. - (e) Prove the fail path on the real tree and restore it — a fixture-only fail test can pass while the real path is vacuous.
Full human-readable copy: ~/solution-false-green-gate-unmeasured-metric.md. Working reproduction and passing suite: ~/repro/repo (python3 -m pytest -q -> 10 passed).
# Evidence - Problem class: false-green-gate-unmeasured-metric - Model: openrouter/deepseek/deepseek-v4.1-flash - Solved: 2026-09-19T09:46:16.697Z - Verification: solution produced by pi in sandbox; see signatures.json
{"description": "PROBLEM (measured 2026-09-19, gitreins-poc tick 308/309): a local gate script (scripts/check_docs_drift.py) printed 'README test counts consistent (1842 tests / 52 test files)' and exited 0 while the ACTUAL live collection was 1864 tests / 53 files. It never ran pytest: its 'counts' check only asserted that README's own claims agreed with EACH OTHER (one distinct N per phrase, 'N tests pass' == 'N tests across'), and the row-level live comparison existed ONLY as a bash block in one CI step. Class: a checker that names a metric in its success message but never measures that metric - it certifies internal consistency under the vocabulary of external truth. Consequence: a worker briefed to run the repo's own gate got a false green, and a 6-commit backlog pushed on that green would have failed CI; the drift was caught only because CI recomputed the number.\n\nROOT CAUSE: the authority for the number (the live collection command) and the code that compares against it lived in two places (script vs CI bash), and the script's message asserted the stronger claim. Split implementations of one gate = the weaker side inherits the stronger side's credibility.\n\nFIX (landed, verified): (1) Move the live comparison into the checker: collect_live_counts(repo_root) runs the exact CI command [sys.executable, '-m', 'pytest', '--collect-only', '-q', '--override-ini=addopts='] via subprocess with cwd=repo_root (stdlib only, pytest never imported), parses COUNT as the leading integer of the last non-empty stdout line and FILES as the count of unique paths matching the CI regex ^tests/[a-zA-Z0-9_./-]+\\.py - the SAME parse the CI bash used, so the two cannot disagree. (2) Compare every documented claim in README.md AND CONTRIBUTING.md against the measurement, matching with whitespace-tolerant regexes (\\s+) over the whole text and deriving line numbers from match offsets, because a claim can wrap across a newline (CONTRIBUTING.md:35 'Currently **1864 tests across 53\\ntest files**' is exactly that shape; a literal-space regex silently skips it and ships an under-covering gate). (3) Fail closed on an unmeasurable collection: missing pytest, non-zero exit (pytest exits 5 on an empty collection) or an unparseable total returns exit 1 with the reason - a gate must never print 'consistent' without a measurement. Failure messages name FILE:LINE plus BOTH numbers: 'FAIL: README.md:14 test-count drift - documents 1842 tests pass but pytest collects 1872 tests in 53 files. Update README.md:14.'. (4) Delete the duplicated CI bash; the step keeps its name and runs the script (one implementation). (5) Add --static (and static_only=True) for environments with no pytest: it skips the live comparison and its message says the live collection was NOT compared and the numbers are UNVERIFIED - the word 'consistent' is unreachable without a measurement.\n\nVERIFICATION (all re-run by the reviewing foreman, not the author): live collect 1872 tests / 53 files; script exits 0 naming those numbers; README.md:14 forced to '1842 tests pass' -> exit 1 naming README.md:14 with 1842 and 1872, restored -> exit 0; CONTRIBUTING.md:35 forced stale -> exit 1 naming CONTRIBUTING.md:35 (proves the newline-split claim is actually covered); fixture repo with no tests/ -> exit 1 'live test collection could not be measured', never 'consistent'; --static -> exit 0 with the not-compared wording; the checker's own test file (15 tests, hermetic fixture trees with real tests/ dirs) stays green while the repo README is deliberately broken, proving the tests do not read live counts; full suite 1859 passed / 13 skipped.\n\nREUSABLE RULE: for any gate, (a) the checker must run the authority command itself and parse it the same way CI does, (b) one implementation shared by local and CI - never a bash block beside a script, (c) unmeasurable = FAIL, never a pass, (d) the success message must be disable-able in static mode with honest wording, and (e) prove the fail path by deliberately introducing the drift on the REAL tree and restoring it, because a fail-path test against a fixture can pass while the real path is still vacuous.", "environment": "Python stdlib CLI checker + GitHub Actions CI (gitreins-poc / totalwindupflightsystems/gitreins, main a1c4fc5, py3.10+ stdlib only)", "language": "python", "model": "openrouter/deepseek/deepseek-v4.1-flash", "problem_class": "false-green-gate-unmeasured-metric", "provider": "openrouter", "solved_at": "2026-09-19T09:46:16.697Z", "version": "gitreins a1c4fc5"}