mitchell/hamlab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 14 Wiki Activity

A second download source, for when the first one refuses

Browse Source 
Cloudflare rate-limits by volume, and the block it handed out while this was
being built lasted over twenty minutes — too fragile to be the only thing behind
a button. So the download phase tries its sources in order: the anycast endpoint
first, then a plain 100 MB file on Linode's Seattle mirror, which has no opinion
about how much has been pulled today.

Only a refusal moves to the next one; anything else would fail the same way
twice. Every result names the source it used, and the page says "from
linode-seattle" rather than "via SEA" when it was the fallback — that mirror is
in one fixed place, so somewhere else it reads low, and a figure that quietly
came from a different continent would be a lie.

Upload stays on Cloudflare, which has never been the one to refuse.

Also: the too-short-window guard was refusing to divide 512 MiB by ten
milliseconds on a link fast enough to hit the runaway budget before the first
tick. It now guards against dividing by zero and nothing more.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-09-27 20:15:33 -07:00
co-authored by Claude Opus 5
parent 2f59a54678
commit 48d3887514
5 changed files with 146 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/network/speedtest.go
+51 -12
View File
@@ -35,6 +35,7 @@ import (
"errors"
"fmt"
"io"
"log"
"math"
"net"
"net/http"
@@ -47,9 +48,27 @@ import (
// The endpoints, and the shape of a phase. Variables rather than constants only so the tests
// can point them at a local server and shrink the windows; nothing else rebinds them.
// Source is somewhere to pull bytes from. Cloudflare's endpoint is the first choice because
// it is anycast — near wherever Ham Lab is running — and because the same host answers for
// upload and for the trace. The fallback exists because that endpoint rate-limits by volume
// and will refuse a connection that has been tested hard: a plain 100 MB file on a public
// mirror has no such opinion. It is in Seattle, so on a machine far from there it will read
// low, which is exactly why every result names the source it used.
type Source struct {
Name string
// URL takes the number of bytes wanted, where the source can be asked; a fixed file
// ignores it and is fetched again each time round.
URL func(bytes int) string
}
var downSources = []Source{
{"cloudflare", func(b int) string { return fmt.Sprintf(cfDownURL, b) }},
{"linode-seattle", func(int) string { return "https://speedtest.seattle.linode.com/100MB-seattle.bin" }},
}
var (
downURL = "https://speed.cloudflare.com/__down?bytes=%d"
upURL = "https://speed.cloudflare.com/__up"
cfDownURL = "https://speed.cloudflare.com/__down?bytes=%d"
upURL = "https://speed.cloudflare.com/__up"
// traceURL is Cloudflare's standard trace: a few lines of key=value naming the address it
// sees, the edge that answered and the country. A few hundred bytes.
traceURL = "https://speed.cloudflare.com/cdn-cgi/trace"
@@ -100,6 +119,9 @@ type Result struct {
// address as that edge saw it.
Colo string `json:"colo,omitempty"`
IP string `json:"ip,omitempty"`
// Source is where the download came from, which matters because the fallback is not in the
// same place as the anycast one. Empty means the usual.
Source string `json:"source,omitempty"`
// Note is set when the figures are real but something went wrong along the way — a stream
// that was refused part-way, most often. The number is what was actually seen; the note is
// why it may be low.
@@ -230,12 +252,21 @@ func (t *Tester) measure(ctx context.Context, cancel context.CancelFunc) {
var note string
if err == nil {
t.set(func(r *Run) { r.Phase, r.Done, r.Mbps = "download", 0.1, 0 })
res.DownMbps, res.DownBytes, note, err = t.phase(ctx, client, "download", downStreams, downFor, 0.1, 0.6)
// Each source in turn: a refusal from the first is exactly what the second is for, and
// anything else (no route, no DNS) would fail the same way twice, so it stops there.
for i, src := range downSources {
res.Source = src.Name
res.DownMbps, res.DownBytes, note, err = t.phase(ctx, client, "download", downStreams, downFor, 0.1, 0.6, src)
if err == nil || !refused(err) || i == len(downSources)-1 {
break
}
log.Printf("speed test: %s refused the download, trying %s", src.Name, downSources[i+1].Name)
}
res.Note = join(res.Note, note)
}
if err == nil {
t.set(func(r *Run) { r.Phase, r.Done, r.Mbps = "upload", 0.7, 0 })
res.UpMbps, res.UpBytes, note, err = t.phase(ctx, client, "upload", upStreams, upFor, 0.7, 1.0)
res.UpMbps, res.UpBytes, note, err = t.phase(ctx, client, "upload", upStreams, upFor, 0.7, 1.0, Source{Name: "cloudflare"})
res.Note = join(res.Note, note)
}
res.TookSec = round(time.Since(started).Seconds(), 1)
@@ -307,7 +338,7 @@ func (t *Tester) latency(ctx context.Context, client *http.Client) ([]float64, m
var out []float64
for i := 0; i <= latencyProbes; i++ { // the first is the handshake, and is not counted
start := time.Now()
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(downURL, 0), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(cfDownURL, 0), nil)
if err != nil {
return nil, m, err
}
@@ -344,7 +375,7 @@ func (t *Tester) latency(ctx context.Context, client *http.Client) ([]float64, m
// refusing part-way, the figure is what was actually moved while it was moving, and the note
// says what happened. Coasting to the end of the clock with no bytes arriving would report a
// number a third of the truth.
func (t *Tester) phase(parent context.Context, client *http.Client, dir string, streams int, forHow time.Duration, from, to float64) (float64, int64, string, error) {
func (t *Tester) phase(parent context.Context, client *http.Client, dir string, streams int, forHow time.Duration, from, to float64, src Source) (float64, int64, string, error) {
// parent is the test's own context, which is cancelled when somebody presses Stop. ctx is
// this phase's, which *this function* cancels to shut the streams down — so the two are
// kept apart: asking ctx whether the test was cancelled would always say yes by then, and
@@ -371,7 +402,7 @@ func (t *Tester) phase(parent context.Context, client *http.Client, dir string,
for ctx.Err() == nil && moved.Load() < phaseBudget {
var err error
if dir == "download" {
err = stream(ctx, client, &moved)
err = stream(ctx, client, src, &moved)
} else {
err = upstream(ctx, client, &moved)
}
@@ -446,7 +477,8 @@ func (t *Tester) phase(parent context.Context, client *http.Client, dir string,
}
return 0, moved.Load(), "", fmt.Errorf("nothing moved in the %s phase", dir)
}
if secs < 0.1 { // too short a window to divide by
if secs < 0.02 { // too short a window to divide by; the guard is against dividing by zero,
// not a judgement about how long a good measurement takes
return 0, moved.Load(), "", fmt.Errorf("the %s phase was over before it could be measured", dir)
}
note := ""
@@ -467,9 +499,9 @@ func join(a, b string) string {
return a + "; " + b
}
// stream fetches one chunk, counting bytes as they arrive.
func stream(ctx context.Context, client *http.Client, n *atomic.Int64) error {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(downURL, downChunk), nil)
// stream fetches one chunk from src, counting bytes as they arrive.
func stream(ctx context.Context, client *http.Client, src Source, n *atomic.Int64) error {
req, err := http.NewRequestWithContext(ctx, "GET", src.URL(downChunk), nil)
if err != nil {
return err
}
@@ -522,11 +554,18 @@ func upstream(ctx context.Context, client *http.Client, n *atomic.Int64) error {
// the only failure of this test anyone here is likely to meet.
func answered(code int, status string) error {
if code == 429 || code == 403 {
return errors.New("the speed endpoint is rate-limiting this connection — it does that after a lot of testing; leave it a few minutes")
return errRefused
}
return fmt.Errorf("the speed endpoint answered %s", status)
}
// errRefused is the far end rate-limiting this connection, which is the only failure of this
// test anyone here is likely to meet: it limits by volume, and a test moves a few hundred
// megabytes. It is recognisable so the download phase can try its other source.
var errRefused = errors.New("the speed endpoint is rate-limiting this connection — it does that after a lot of testing; leave it a few minutes")
func refused(err error) bool { return errors.Is(err, errRefused) }
// zeros is a request body of a fixed size that costs nothing to produce, counting itself as
// it is read out onto the wire.
type zeros struct {