Merge branch 'infrastructure'

* infrastructure:
  build: let infra containers builds fail individually
  chore(ur): move structs to reduce dependency chain
  chore(stcrashreceiver): add profiler on metrics port
  chore(stcrashreceiver): compact diskstore in-memory representation
  chore(stcrashreceiver): better source cache & metrics
  chore(stcrashreceiver): metrics on ignore matches

.github/workflows/build-infra-dockers.yaml

@@ -22,6 +22,7 @@ jobs:
     if: github.repository_owner == 'syncthing'
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         pkg:
           - stcrashreceiver

cmd/infra/stcrashreceiver/diskstore.go

@@ -136,15 +136,25 @@ func (d *diskStore) Exists(path string) bool {
 }
 
 func (d *diskStore) clean() {
-	for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
+	numDeleted := 0
-		f := d.currentFiles[0]
+	for idx := range d.currentFiles {
+		if len(d.currentFiles)-numDeleted < d.maxFiles && d.currentSize < d.maxBytes {
+			break
+		}
+
+		f := d.currentFiles[idx]
 		log.Println("Removing", f.path)
 		if err := os.Remove(f.path); err != nil {
 			log.Println("Failed to remove file:", err)
 		}
-		d.currentFiles = d.currentFiles[1:]
 		d.currentSize -= f.size
+		numDeleted = idx + 1
 	}
+
+	// Compact currentFiles
+	copy(d.currentFiles, d.currentFiles[numDeleted:])
+	d.currentFiles = d.currentFiles[:len(d.currentFiles)-numDeleted]
+
 	var oldest time.Duration
 	if len(d.currentFiles) > 0 {
 		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
@@ -158,7 +168,7 @@ func (d *diskStore) clean() {
 }
 
 func (d *diskStore) inventory() error {
-	d.currentFiles = nil
+	d.currentFiles = d.currentFiles[:0]
 	d.currentSize = 0
 	err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {

cmd/infra/stcrashreceiver/main.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -29,7 +30,7 @@ import (
 	raven "github.com/getsentry/raven-go"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/ur"
+	"github.com/syncthing/syncthing/lib/ur/contract"
 )
 
 const maxRequestSize = 1 << 20 // 1 MiB
@@ -89,6 +90,7 @@ func main() {
 	if params.MetricsListen != "" {
 		mmux := http.NewServeMux()
 		mmux.Handle("/metrics", promhttp.Handler())
+		mmux.HandleFunc("/debug/pprof/", pprof.Index)
 		go func() {
 			if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
 				log.Fatalln("HTTP serve metrics:", err)
@@ -123,12 +125,13 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 			return
 		}
 
-		if _, ok := ignore.match(bs); ok {
+		if pat, ok := ignore.match(bs); ok {
+			metricIgnoreMatchesTotal.WithLabelValues(pat).Inc()
 			result = "ignored"
 			return
 		}
 
-		var reports []ur.FailureReport
+		var reports []contract.FailureReport
 		err = json.Unmarshal(bs, &reports)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
@@ -176,7 +179,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 	}
 }
 
-func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
+func saveFailureWithGoroutines(data contract.FailureData, failureDir string) (string, error) {
 	bs := make([]byte, len(data.Description)+len(data.Goroutines))
 	copy(bs, data.Description)
 	copy(bs[len(data.Description):], data.Goroutines)

cmd/infra/stcrashreceiver/metrics.go

@@ -42,4 +42,19 @@ var (
 		Subsystem: "crashreceiver",
 		Name:      "sentry_reports_total",
 	}, []string{"result"})
+	metricIgnoreMatchesTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "crashreceiver",
+		Name:      "ignore_matches_total",
+	}, []string{"pattern"})
+	metricSourceCodeLoadsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "crashreceiver",
+		Name:      "source_code_loads_total",
+	}, []string{"result"})
+	metricSourceCodeCacheSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "syncthing",
+		Subsystem: "crashreceiver",
+		Name:      "source_code_cache_size",
+	})
 )

cmd/infra/stcrashreceiver/sourcecodeloader.go

@@ -15,23 +15,33 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	lru "github.com/hashicorp/golang-lru/v2"
 )
 
 const (
-	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	urlPrefix       = "https://raw.githubusercontent.com/syncthing/syncthing/"
-	httpTimeout = 10 * time.Second
+	httpTimeout     = 10 * time.Second
+	maxCacheEntries = 1000
 )
 
+type cacheKey struct {
+	version string
+	file    string
+}
+
 type githubSourceCodeLoader struct {
 	mut     sync.Mutex
 	version string
-	cache   map[string]map[string][][]byte // version -> file -> lines
+
-	client  *http.Client
+	cache  *lru.TwoQueueCache[cacheKey, [][]byte] // version & file -> lines
+	client *http.Client
 }
 
 func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	cache, _ := lru.New2Q[cacheKey, [][]byte](maxCacheEntries)
 	return &githubSourceCodeLoader{
-		cache:  make(map[string]map[string][][]byte),
+		cache:  cache,
 		client: &http.Client{Timeout: httpTimeout},
 	}
 }
@@ -39,9 +49,6 @@ func newGithubSourceCodeLoader() *githubSourceCodeLoader {
 func (l *githubSourceCodeLoader) LockWithVersion(version string) {
 	l.mut.Lock()
 	l.version = version
-	if _, ok := l.cache[version]; !ok {
-		l.cache[version] = make(map[string][][]byte)
-	}
 }
 
 func (l *githubSourceCodeLoader) Unlock() {
@@ -50,11 +57,13 @@ func (l *githubSourceCodeLoader) Unlock() {
 
 func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
 	filename = filepath.ToSlash(filename)
-	lines, ok := l.cache[l.version][filename]
+	key := cacheKey{version: l.version, file: filename}
+	lines, ok := l.cache.Get(key)
 	if !ok {
 		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
 		defer func() {
-			l.cache[l.version][filename] = lines
+			l.cache.Add(key, lines)
+			metricSourceCodeCacheSize.Set(float64(l.cache.Len()))
 		}()
 
 		knownPrefixes := []string{"/lib/", "/cmd/"}
@@ -73,19 +82,25 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
 		resp, err := l.client.Get(url)
 		if err != nil {
 			fmt.Println("Loading source:", err)
+			metricSourceCodeLoadsTotal.WithLabelValues("failed").Inc()
 			return nil, 0
 		}
 		if resp.StatusCode != http.StatusOK {
 			fmt.Println("Loading source:", resp.Status)
+			metricSourceCodeLoadsTotal.WithLabelValues("failed").Inc()
 			return nil, 0
 		}
 		data, err := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
 		if err != nil {
 			fmt.Println("Loading source:", err.Error())
+			metricSourceCodeLoadsTotal.WithLabelValues("failed").Inc()
 			return nil, 0
 		}
 		lines = bytes.Split(data, []byte{'\n'})
+		metricSourceCodeLoadsTotal.WithLabelValues("loaded").Inc()
+	} else {
+		metricSourceCodeLoadsTotal.WithLabelValues("cached").Inc()
 	}
 
 	return getLineFromLines(lines, line, context)

cmd/infra/stcrashreceiver/stcrashreceiver.go

@@ -90,6 +90,7 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 	first := string(bytes.TrimSpace(bytes.Split(bs, []byte("\n"))[0]))
 
 	if pat, ok := r.ignore.match(bs); ok {
+		metricIgnoreMatchesTotal.WithLabelValues(pat).Inc()
 		result = "ignored"
 		log.Printf("Ignored report %s, matched: %s (%s)", reportID[:8], pat, first)
 		return

lib/model/indexhandler.go

@@ -20,7 +20,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/svcutil"
-	"github.com/syncthing/syncthing/lib/ur"
+	"github.com/syncthing/syncthing/lib/ur/contract"
 )
 
 type indexHandler struct {
@@ -470,7 +470,7 @@ func (s *indexHandler) logSequenceAnomaly(msg string, extra map[string]any) {
 		extraStrs[k] = fmt.Sprint(v)
 	}
 
-	s.evLogger.Log(events.Failure, ur.FailureData{
+	s.evLogger.Log(events.Failure, contract.FailureData{
 		Description: msg,
 		Extra:       extraStrs,
 	})

lib/ur/contract/contract.go

@@ -282,3 +282,16 @@ func clear(v interface{}, since int) error {
 	}
 	return nil
 }
+
+type FailureReport struct {
+	FailureData
+
+	Count   int
+	Version string
+}
+
+type FailureData struct {
+	Description string
+	Goroutines  string
+	Extra       map[string]string
+}

lib/ur/failurereporting.go

@@ -23,6 +23,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/svcutil"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"github.com/syncthing/syncthing/lib/ur/contract"
 
 	"github.com/thejerf/suture/v4"
 )
@@ -39,23 +40,10 @@ var (
 	invalidEventDataType = "failure event data is not a string"
 )
 
-type FailureReport struct {
+func FailureDataWithGoroutines(description string) contract.FailureData {
-	FailureData
-
-	Count   int
-	Version string
-}
-
-type FailureData struct {
-	Description string
-	Goroutines  string
-	Extra       map[string]string
-}
-
-func FailureDataWithGoroutines(description string) FailureData {
 	var buf strings.Builder
 	pprof.Lookup("goroutine").WriteTo(&buf, 1)
-	return FailureData{
+	return contract.FailureData{
 		Description: description,
 		Goroutines:  buf.String(),
 		Extra:       make(map[string]string),
@@ -86,7 +74,7 @@ type failureHandler struct {
 type failureStat struct {
 	first, last time.Time
 	count       int
-	data        FailureData
+	data        contract.FailureData
 }
 
 func (h *failureHandler) Serve(ctx context.Context) error {
@@ -105,24 +93,24 @@ func (h *failureHandler) Serve(ctx context.Context) error {
 			if !ok {
 				// Just to be safe - shouldn't ever happen, as
 				// evChan is set to nil when unsubscribing.
-				h.addReport(FailureData{Description: evChanClosed}, time.Now())
+				h.addReport(contract.FailureData{Description: evChanClosed}, time.Now())
 				evChan = nil
 				continue
 			}
-			var data FailureData
+			var data contract.FailureData
 			switch d := e.Data.(type) {
 			case string:
 				data.Description = d
-			case FailureData:
+			case contract.FailureData:
 				data = d
 			default:
 				// Same here, shouldn't ever happen.
-				h.addReport(FailureData{Description: invalidEventDataType}, time.Now())
+				h.addReport(contract.FailureData{Description: invalidEventDataType}, time.Now())
 				continue
 			}
 			h.addReport(data, e.Time)
 		case <-timer.C:
-			reports := make([]FailureReport, 0, len(h.buf))
+			reports := make([]contract.FailureReport, 0, len(h.buf))
 			now := time.Now()
 			for descr, stat := range h.buf {
 				if now.Sub(stat.last) > minDelay || now.Sub(stat.first) > maxDelay {
@@ -152,7 +140,7 @@ func (h *failureHandler) Serve(ctx context.Context) error {
 	if sub != nil {
 		sub.Unsubscribe()
 		if len(h.buf) > 0 {
-			reports := make([]FailureReport, 0, len(h.buf))
+			reports := make([]contract.FailureReport, 0, len(h.buf))
 			for _, stat := range h.buf {
 				reports = append(reports, newFailureReport(stat))
 			}
@@ -179,7 +167,7 @@ func (h *failureHandler) applyOpts(opts config.OptionsConfiguration, sub events.
 	return url, nil, nil
 }
 
-func (h *failureHandler) addReport(data FailureData, evTime time.Time) {
+func (h *failureHandler) addReport(data contract.FailureData, evTime time.Time) {
 	if stat, ok := h.buf[data.Description]; ok {
 		stat.last = evTime
 		stat.count++
@@ -204,7 +192,7 @@ func (*failureHandler) String() string {
 	return "FailureHandler"
 }
 
-func sendFailureReports(ctx context.Context, reports []FailureReport, url string) {
+func sendFailureReports(ctx context.Context, reports []contract.FailureReport, url string) {
 	var b bytes.Buffer
 	if err := json.NewEncoder(&b).Encode(reports); err != nil {
 		panic(err)
@@ -235,8 +223,8 @@ func sendFailureReports(ctx context.Context, reports []FailureReport, url string
 	resp.Body.Close()
 }
 
-func newFailureReport(stat *failureStat) FailureReport {
+func newFailureReport(stat *failureStat) contract.FailureReport {
-	return FailureReport{
+	return contract.FailureReport{
 		FailureData: stat.data,
 		Count:       stat.count,
 		Version:     build.LongVersion,