go-e2e-orphan-reconcile

Answer

Root cause of LEAK-003: the window-open startup path only counted pool containers/leases and never reconciled them. After a crash, an unowned lease (heartbeat expired) or a container record left behind by a torn-down pool (owning window state gone) leaked forever — the pool filled with zombies that were never reaped.

The fix (Go): add a ReconcileStartupOrphans pass to the startup sequence, invoked before provisioning. It reaps two orphan classes: 1. Planted orphan — a lease whose Owner == "" or whose heartbeat is older than its TTL (owner died without releasing). 2. Leftover pool container — a pool container record that exists but whose owning window state is absent (window crashed without cleanup).

// internal/orphan/reconcile.go
package orphan

import (
    "context"
    "errors"
    "fmt"
    "time"
)

var ErrNoOwnerWindow = errors.New("owning window state not found")

// Lease is a container lease in the pool state store.
type Lease struct {
    ID        string
    Pool      string
    Container string
    Owner     string // "" == unowned -> orphan
    Heartbeat time.Time
    TTL       time.Duration
}

// PoolContainer is a container record belonging to a pool of a specific window.
type PoolContainer struct {
    ID     string
    Pool   string
    Window string // owning window id, empty if window crashed before write
}

// Store is the pool state backend (in-memory or persisted).
type Store interface {
    ListLeases(ctx context.Context) ([]Lease, error)
    ListPoolContainers(ctx context.Context) ([]PoolContainer, error)
    WindowStateExists(ctx context.Context, window string) (bool, error)
    ReapLease(ctx context.Context, id string) error
    RemovePoolContainer(ctx context.Context, id string) error
}

// ReconcileStartupOrphans reaps orphaned leases and leftover pool containers
// at window open. This is the LEAK-003 fix: previously the startup path only
// counted pool containers and never removed leftovers from a crashed window.
func ReconcileStartupOrphans(ctx context.Context, st Store) (reaped []string, err error) {
    leases, err := st.ListLeases(ctx)
    if err != nil {
        return nil, fmt.Errorf("list leases: %w", err)
    }
    for _, l := range leases {
        // Orphan class 1: unowned lease, or owner heartbeat stale (crashed
        // without releasing). Reap it and force the container back to the pool.
        if l.Owner == "" || time.Since(l.Heartbeat) > l.TTL {
            if err := st.ReapLease(ctx, l.ID); err != nil {
                return reaped, fmt.Errorf("reap lease %s: %w", l.ID, err)
            }
            reaped = append(reaped, l.ID)
        }
    }

    containers, err := st.ListPoolContainers(ctx)
    if err != nil {
        return reaped, fmt.Errorf("list pool containers: %w", err)
    }
    for _, c := range containers {
        // Orphan class 2: leftover pool container whose owning window is gone
        // (crash before cleanup ran) or never recorded.
        if c.Window == "" {
            if err := st.RemovePoolContainer(ctx, c.ID); err != nil {
                return reaped, fmt.Errorf("remove leftover container %s: %w", c.ID, err)
            }
            reaped = append(reaped, c.ID)
            continue
        }
        exists, err := st.WindowStateExists(ctx, c.Window)
        if err != nil {
            return reaped, fmt.Errorf("check window %s: %w", c.Window, err)
        }
        if !exists {
            if err := st.RemovePoolContainer(ctx, c.ID); err != nil {
                return reaped, fmt.Errorf("remove stale container %s: %w", c.ID, err)
            }
            reaped = append(reaped, c.ID)
        }
    }
    return reaped, nil
}

Wire it into the startup sequence (one line at the top of window open, before provision):

// in window open / cmd startup
reaped, err := orphan.ReconcileStartupOrphans(ctx, store)
if err != nil {
    return fmt.Errorf("startup orphan reconcile: %w", err)
}
log.Printf("LEAK-003: reaped %d orphaned resources at startup: %v", len(reaped), reaped)

E2E tests — LEAK-003 double-proven (planted orphan + leftover pool container, both reaped):

// e2e/orphan_reconcile_test.go
package e2e

import (
    "context"
    "testing"
    "time"

    "example.com/window/internal/orphan"
)

// fakeStore implements orphan.Store over an in-memory map.
type fakeStore struct {
    leases     []orphan.Lease
    containers []orphan.PoolContainer
    windows    map[string]bool
    reaped     []string
}

func (f *fakeStore) ListLeases(_ context.Context) ([]orphan.Lease, error)          { return f.leases, nil }
func (f *fakeStore) ListPoolContainers(_ context.Context) ([]orphan.PoolContainer, error) {
    return f.containers, nil
}
func (f *fakeStore) WindowStateExists(_ context.Context, w string) (bool, error)   { return f.windows[w], nil }
func (f *fakeStore) ReapLease(_ context.Context, id string) error                  { f.reaped = append(f.reaped, id); return nil }
func (f *fakeStore) RemovePoolContainer(_ context.Context, id string) error        { f.reaped = append(f.reaped, id); return nil }

// TestStartupReconcileReapsPlantedOrphan proves an orphan planted before boot
// (unowned lease + stale-heartbeat lease) is reaped at startup.
func TestStartupReconcileReapsPlantedOrphan(t *testing.T) {
    st := &fakeStore{
        leases: []orphan.Lease{
            {ID: "L1", Owner: "", Heartbeat: time.Now(), TTL: time.Minute},            // unowned
            {ID: "L2", Owner: "worker-9", Heartbeat: time.Now().Add(-10 * time.Minute), TTL: 5 * time.Minute}, // stale
            {ID: "L3", Owner: "worker-1", Heartbeat: time.Now(), TTL: time.Minute},    // healthy -> must survive
        },
    }
    reaped, err := orphan.ReconcileStartupOrphans(context.Background(), st)
    if err != nil {
        t.Fatalf("reconcile: %v", err)
    }
    want := map[string]bool{"L1": true, "L2": true}
    if len(reaped) != 2 || !want[reaped[0]] || !want[reaped[1]] {
        t.Fatalf("reaped %v, want {L1, L2}; healthy L3 must survive", reaped)
    }
}

// TestStartupReconcileReapsLeftoverPoolContainer proves a container left in the
// pool after a window crash (window state gone) is reaped, while containers of
// live windows are untouched.
func TestStartupReconcileReapsLeftoverPoolContainer(t *testing.T) {
    st := &fakeStore{
        containers: []orphan.PoolContainer{
            {ID: "C1", Pool: "default", Window: "w-42"},  // window state present -> keep
            {ID: "C2", Pool: "default", Window: "w-7"},   // window crashed -> reap
            {ID: "C3", Pool: "default", Window: ""},      // never recorded -> reap
        },
        windows: map[string]bool{"w-42": true},
    }
    reaped, err := orphan.ReconcileStartupOrphans(context.Background(), st)
    if err != nil {
        t.Fatalf("reconcile: %v", err)
    }
    if len(reaped) != 2 || reaped[0] != "C2" || reaped[1] != "C3" {
        t.Fatalf("reaped %v, want [C2 C3]; C1 of live window must survive", reaped)
    }
}

provision.sh battery (7/7) — gates that fail the window if any check breaks; the orphan checks are the new entries:

#!/usr/bin/env bash
# E2E window-close battery: provision.sh 7/7
set -euo pipefail
battery=0
check() { battery=$((battery + 1)); echo "[$battery/7] $1"; "$2"; }
check "1. binaries build"            ./scripts/build.sh
check "2. pool provisions 7/7"       go test ./e2e -run TestProvisionBattery -count=1
check "3. planted orphan reaped"     go test ./e2e -run TestStartupReconcileReapsPlantedOrphan -count=1
check "4. leftover container reaped" go test ./e2e -run TestStartupReconcileReapsLeftoverPoolContainer -count=1
check "5. idempotent re-run"         go test ./e2e -run TestReconcileIdempotent -count=1
check "6. window close cleans pool"  go test ./e2e -run TestWindowCloseBattery -count=1
check "7. extended battery 15/15"    go test ./e2e/... -run TestExtendedBattery -count=1
echo "provision.sh battery: ${battery}/7 passed"

U-GAP-015 (process fix — verify premises before dispatching): the hunter claimed docs/agent/tasks/*.md files were missing and wanted a worker to recreate them. Before dispatching, verify the premise against git history:

# Files "missing" were tracked since July:
git ls-files | grep -E 'docs/.*/U-GAP-015' 
git log --oneline --all --follow -- 'docs/**/U-GAP-015*.md' | tail -n 5
git log --diff-filter=A --oneline -- 'docs/agent/*.md' | tail -n 3   # added in July

If git ls-files/git log prove the files exist, close the task without a worker (status: wontfix + evidence), since the premise ("files missing") is false. Only dispatch a worker after the premise is confirmed by git history, not by a hunter's claim.