Mitchell-JohnsonandClaude Opus 5.5
574d9855ce
nvidia-smi on a busy card can take past the 3 s deadline, which latched the card as stalled until restart. A late read now carries the last readings and is used when it lands; no second read starts while one is out; only one out for 15 s (past both nvidia-smi timeouts) marks the card stalled, and its return clears it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
53 lines
1.6 KiB
Go
53 lines
1.6 KiB
Go
package metrics
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// A read that comes back late carries the last readings meanwhile and is used when it lands;
|
|
// only one out past gpuStallAfter marks the card stalled, and its return clears that.
|
|
func TestGPUReadLateThenStalledThenBack(t *testing.T) {
|
|
defer func(w, s time.Duration) { gpuReadWithin, gpuStallAfter = w, s }(gpuReadWithin, gpuStallAfter)
|
|
gpuReadWithin, gpuStallAfter = 20*time.Millisecond, 10*time.Second
|
|
release := make(chan struct{})
|
|
n := 0.0 // only the read goroutine writes it, one read at a time
|
|
d := &gpuDev{name: "test", card: "card0"}
|
|
d.readFn = func(x *GPU, _ time.Time) {
|
|
n++
|
|
if n == 2 {
|
|
<-release
|
|
}
|
|
x.Usage = n * 10
|
|
}
|
|
t0 := time.Now()
|
|
sample := func(at time.Duration) GPU {
|
|
x := GPU{Name: "test"}
|
|
d.readWithin(&x, t0.Add(at))
|
|
return x
|
|
}
|
|
if x := sample(0); x.Usage != 10 || x.Stalled {
|
|
t.Fatalf("first: %+v", x)
|
|
}
|
|
if x := sample(time.Second); x.Usage != 10 || x.Stalled { // late: the last readings
|
|
t.Fatalf("late: %+v", x)
|
|
}
|
|
if x := sample(5 * time.Second); x.Usage != 10 || x.Stalled {
|
|
t.Fatalf("still late: %+v", x)
|
|
}
|
|
if x := sample(12 * time.Second); !x.Stalled {
|
|
t.Fatalf("out past gpuStallAfter, not stalled: %+v", x)
|
|
}
|
|
if x := sample(12*time.Second + 1); !x.Stalled {
|
|
t.Fatal("stall did not hold")
|
|
}
|
|
close(release)
|
|
time.Sleep(10 * time.Millisecond)
|
|
if x := sample(13 * time.Second); x.Usage != 20 || x.Stalled {
|
|
t.Fatalf("back: %+v", x)
|
|
}
|
|
if x := sample(14 * time.Second); x.Usage != 30 || x.Stalled { // 30: no extra read ran while one was out
|
|
t.Fatalf("next: %+v", x)
|
|
}
|
|
}
|