chore(stdiscosrv): smooth retry-after delays over a slightly larger normal distribution

Signed-off-by: Jakob Borg <jakob@kastelo.net>

cmd/stdiscosrv/apisrv.go

@@ -564,5 +564,17 @@ func (t *retryAfterTracker) retryAfterS() int {
 	}
 	t.curCount++
 	t.mut.Unlock()
-	return t.currentDelay + rand.Intn(t.currentDelay/4)
+
+	// Skewed normal distribution with the mean at currentDelay and the
+	// limits (50% and 150%) at 3 standard deviations
+	nf := rand.NormFloat64()
+	minD := max(notFoundRetryUnknownMinSeconds, t.currentDelay/2)
+	maxD := min(notFoundRetryUnknownMaxSeconds, t.currentDelay*3/2)
+	intv := float64(maxD - t.currentDelay)
+	if nf < 0 {
+		intv = float64(t.currentDelay - minD)
+	}
+	nf = min(max(nf*intv/3+float64(t.currentDelay), notFoundRetryUnknownMinSeconds), notFoundRetryUnknownMaxSeconds)
+
+	return int(nf)
 }

cmd/stdiscosrv/apisrv_test.go

@@ -18,6 +18,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -106,6 +107,53 @@ func addr(host string, port int) *net.TCPAddr {
 	}
 }
 
+func TestRetryAfterSHistogram(t *testing.T) {
+	tracker := &retryAfterTracker{
+		name:         "test",
+		bucketStarts: time.Now(),
+		desiredRate:  100,
+		currentDelay: 1800,
+	}
+
+	const n = 1000
+	bucketSize := 60 // seconds per histogram bucket
+	numBuckets := (notFoundRetryUnknownMaxSeconds + bucketSize - 1) / bucketSize
+	buckets := make([]int, numBuckets)
+
+	for i := 0; i < n; i++ {
+		v := tracker.retryAfterS()
+		if v < notFoundRetryUnknownMinSeconds || v > notFoundRetryUnknownMaxSeconds {
+			t.Fatalf("retryAfterS() = %d, out of range [%d, %d]", v, notFoundRetryUnknownMinSeconds, notFoundRetryUnknownMaxSeconds)
+		}
+		b := (v - 1) / bucketSize
+		if b >= numBuckets {
+			b = numBuckets - 1
+		}
+		buckets[b]++
+	}
+
+	// Print a horizontal histogram
+	maxCount := 0
+	for _, c := range buckets {
+		if c > maxCount {
+			maxCount = c
+		}
+	}
+	barWidth := 60
+	for i, c := range buckets {
+		lo := i*bucketSize + 1
+		hi := (i + 1) * bucketSize
+		if hi > notFoundRetryUnknownMaxSeconds {
+			hi = notFoundRetryUnknownMaxSeconds
+		}
+		bar := ""
+		if maxCount > 0 {
+			bar = strings.Repeat("#", c*barWidth/maxCount)
+		}
+		t.Logf("%4d-%4ds | %-*s %d", lo, hi, barWidth, bar, c)
+	}
+}
+
 func BenchmarkAPIRequests(b *testing.B) {
 	db := newInMemoryStore(b.TempDir(), 0, nil)
 	ctx, cancel := context.WithCancel(context.Background())