GPU: a slow read is late, not stalled; a stall clears when the read returns

nvidia-smi on a busy card can take past the 3 s deadline, which latched the card
as stalled until restart. A late read now carries the last readings and is used
when it lands; no second read starts while one is out; only one out for 15 s
(past both nvidia-smi timeouts) marks the card stalled, and its return clears it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/metrics/gpu_stall_test.go

@@ -0,0 +1,52 @@
+package metrics
+
+import (
+	"testing"
+	"time"
+)
+
+// A read that comes back late carries the last readings meanwhile and is used when it lands;
+// only one out past gpuStallAfter marks the card stalled, and its return clears that.
+func TestGPUReadLateThenStalledThenBack(t *testing.T) {
+	defer func(w, s time.Duration) { gpuReadWithin, gpuStallAfter = w, s }(gpuReadWithin, gpuStallAfter)
+	gpuReadWithin, gpuStallAfter = 20*time.Millisecond, 10*time.Second
+	release := make(chan struct{})
+	n := 0.0 // only the read goroutine writes it, one read at a time
+	d := &gpuDev{name: "test", card: "card0"}
+	d.readFn = func(x *GPU, _ time.Time) {
+		n++
+		if n == 2 {
+			<-release
+		}
+		x.Usage = n * 10
+	}
+	t0 := time.Now()
+	sample := func(at time.Duration) GPU {
+		x := GPU{Name: "test"}
+		d.readWithin(&x, t0.Add(at))
+		return x
+	}
+	if x := sample(0); x.Usage != 10 || x.Stalled {
+		t.Fatalf("first: %+v", x)
+	}
+	if x := sample(time.Second); x.Usage != 10 || x.Stalled { // late: the last readings
+		t.Fatalf("late: %+v", x)
+	}
+	if x := sample(5 * time.Second); x.Usage != 10 || x.Stalled {
+		t.Fatalf("still late: %+v", x)
+	}
+	if x := sample(12 * time.Second); !x.Stalled {
+		t.Fatalf("out past gpuStallAfter, not stalled: %+v", x)
+	}
+	if x := sample(12*time.Second + 1); !x.Stalled {
+		t.Fatal("stall did not hold")
+	}
+	close(release)
+	time.Sleep(10 * time.Millisecond)
+	if x := sample(13 * time.Second); x.Usage != 20 || x.Stalled {
+		t.Fatalf("back: %+v", x)
+	}
+	if x := sample(14 * time.Second); x.Usage != 30 || x.Stalled { // 30: no extra read ran while one was out
+		t.Fatalf("next: %+v", x)
+	}
+}