python-hilo-stats-empty-direct-duckdb-fallback

Answer

Root cause (per runbook, verified by reproduction): hilo graph stats intermittently prints empty stdout with exit code 0 while the underlying graph is healthy (T64/T66/T108). The counts still exist — the known-good source is a direct read-only DuckDB query on .vfs/graph/graph.db run with the board venv python (~/.hermes/venvs/board/bin/python3; system python3 may lack duckdb). The fix detects only this exact signature (empty stdout and exit 0) and transparently substitutes the DuckDB counts. It never reports Hilo=STALE, never rm+rewarms, never investigates — and it refuses to fabricate data on any other failure mode (rc≠0, or non-empty unparseable output), surfacing those as errors instead.

The fix — hilo_stats.py (stdlib-only caller; duckdb lives in the board venv):

#!/usr/bin/env python3
import json, os, re, subprocess, sys
from pathlib import Path

DEFAULT_BOARD_PY = Path.home() / ".hermes" / "venvs" / "board" / "bin" / "python3"
GRAPH_DB_REL = Path(".vfs") / "graph" / "graph.db"

# Runs under the board venv python. read_only=True; only two SELECTs.
FALLBACK_SCRIPT = r"""
import duckdb, sys
def main(db_path):
    con = duckdb.connect(db_path, read_only=True)   # read-only: no writes, no locks
    try:
        total        = con.execute('SELECT count(*) FROM edges').fetchone()[0]
        distinct_from = con.execute('SELECT count(DISTINCT "from") FROM edges').fetchone()[0]
    finally:
        con.close()
    print("edges_total=%d" % total)
    print("from_distinct=%d" % distinct_from)
if __name__ == "__main__":
    main(sys.argv[1])
"""

class HiloStatsError(RuntimeError): pass

def parse_stats(text):
    """Tolerant parse of `hilo graph stats` output -> {edges_total, from_distinct}.
    Returns None for empty/stub/garbage output (i.e. when output is not trustworthy)."""
    if not text or not text.strip(): return None
    edges = froms = None
    for line in text.splitlines():
        low = line.strip().lower()
        pairs = re.findall(r"(\d+)[^\d]*([a-z_]+)|([a-z_]+)[^\d]*(\d+)", low)
        for p in pairs:
            if p[0]: num, label = int(p[0]), p[1]
            else:    label, num = p[2], int(p[3])
            label = label.rstrip(":=")
            if "edges" in label:                    edges = num
            elif "distinct" in label or "from" in label: froms = num
    return None if (edges is None or froms is None) else \
           {"edges_total": edges, "from_distinct": froms}

def fallback_duckdb(root, board_py=None, env=None):
    """The working fallback: direct read-only DuckDB counts via the board venv."""
    board_py = Path(board_py or os.environ.get("HILO_BOARD_PY", DEFAULT_BOARD_PY))
    db = Path(root) / GRAPH_DB_REL
    if not db.is_file():     raise HiloStatsError("fallback: graph db not found: %s" % db)
    if not board_py.is_file(): raise HiloStatsError(
        "fallback: board venv python not found: %s (set HILO_BOARD_PY)" % board_py)
    proc = subprocess.run([str(board_py), "-c", FALLBACK_SCRIPT, str(db)],
                          capture_output=True, text=True, timeout=120, env=env)
    if proc.returncode != 0: raise HiloStatsError(
        "fallback duckdb failed rc=%s: %s" % (proc.returncode, proc.stderr.strip()))
    m_total, m_from = re.search(r"edges_total=(\d+)", proc.stdout), \
                      re.search(r"from_distinct=(\d+)", proc.stdout)
    if not (m_total and m_from): raise HiloStatsError(
        "fallback: unparseable duckdb output: %r" % proc.stdout)
    s = {"edges_total": int(m_total.group(1)), "from_distinct": int(m_from.group(1))}
    if s["from_distinct"] > s["edges_total"]: raise HiloStatsError("sanity fail: %r" % s)
    return s

def run_stats(root, hilo_bin="hilo", env=None):
    proc = subprocess.run([hilo_bin, "graph", "stats"], cwd=root,
                          capture_output=True, text=True, timeout=120, env=env)
    stats = parse_stats(proc.stdout)
    if stats is not None:                 return stats, "hilo"            # normal
    if proc.returncode == 0 and not proc.stdout.strip():
        return fallback_duckdb(root, env=env), "duckdb-fallback"          # THE bug
    raise HiloStatsError("hilo graph stats: rc=%s stdout=%r stderr=%r"
                         % (proc.returncode, proc.stdout, proc.stderr))

def main(argv=None):
    argv = list(argv if argv is not None else sys.argv[1:])
    want_json = argv and argv[0] == "--json" and argv.pop(0) is not None
    root = argv[0] if argv else os.getcwd()
    stats, source = run_stats(root)
    print(json.dumps({**stats, "source": source}) if want_json else
          "edges_total=%d\nfrom_distinct=%d" % (stats["edges_total"], stats["from_distinct"]))
    return 0

if __name__ == "__main__":
    try: sys.exit(main())
    except HiloStatsError as e: print("hilo_stats: %s" % e, file=sys.stderr); sys.exit(1)

Key points: - Detection is exact: fallback triggers only on returncode == 0 AND empty stdout — the documented signature. Stub case (empty table) returns true 0/0 from the DB (not the bug, not masked, no STALE label). Warm/stats-mismatch and rc≠0 cases raise instead of inventing data. - Read-only: duckdb.connect(path, read_only=True); only two SELECTs. Writes are impossible and the DB file is byte-identical before/after (verified, see Evidence). - Reserved word: the column is from — a DuckDB reserved keyword — so the fallback uses count(DISTINCT "from") (unquoted, it fails to parse; this is a real gotcha worth shipping pre-fixed). - Venv correctness: the query runs under ~/.hermes/venvs/board/bin/python3 (overridable via HILO_BOARD_PY), matching the runbook's known-good invocation; the caller stays stdlib-only. - Drop-in wrapper (bin/hilo-stats): execs the module, same output shape as before, optional --json for machine use.