Merge branch 'infrastructure'

* infrastructure:
  fix(stcrashreceiver): allow extra pre/post data in version line
  chore(stcrashreceiver): improve logging
  chore(stdiscosrv): prewarm counters at startup

cmd/infra/stcrashreceiver/main.go

@@ -123,7 +123,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 			return
 		}
 
-		if ignore.match(bs) {
+		if _, ok := ignore.match(bs); ok {
 			result = "ignored"
 			return
 		}
@@ -216,14 +216,14 @@ func loadIgnorePatterns(path string) (*ignorePatterns, error) {
 	return &ignorePatterns{patterns: patterns}, nil
 }
 
-func (i *ignorePatterns) match(report []byte) bool {
+func (i *ignorePatterns) match(report []byte) (string, bool) {
 	if i == nil {
-		return false
+		return "", false
 	}
 	for _, re := range i.patterns {
 		if re.Match(report) {
-			return true
+			return re.String(), true
 		}
 	}
-	return false
+	return "", false
 }

cmd/infra/stcrashreceiver/metrics.go

@@ -37,4 +37,9 @@ var (
 		Subsystem: "crashreceiver",
 		Name:      "diskstore_oldest_age_seconds",
 	})
+	metricSentryReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "crashreceiver",
+		Name:      "sentry_reports_total",
+	}, []string{"result"})
 )

cmd/infra/stcrashreceiver/sentry.go

@@ -10,6 +10,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"regexp"
@@ -52,11 +53,15 @@ func (s *sentryService) Serve(ctx context.Context) {
 			pkt, err := parseCrashReport(req.reportID, req.data)
 			if err != nil {
 				log.Println("Failed to parse crash report:", err)
+				metricSentryReportsTotal.WithLabelValues("parse_failure").Inc()
 				continue
 			}
 			if err := sendReport(s.dsn, pkt, req.userID); err != nil {
 				log.Println("Failed to send crash report:", err)
+				metricSentryReportsTotal.WithLabelValues("send_failure").Inc()
+				continue
 			}
+			metricSentryReportsTotal.WithLabelValues("success").Inc()
 
 		case <-ctx.Done():
 			return
@@ -69,6 +74,7 @@ func (s *sentryService) Send(reportID, userID string, data []byte) bool {
 	case s.inbox <- sentryRequest{reportID, userID, data}:
 		return true
 	default:
+		metricCrashReportsTotal.WithLabelValues("overflow").Inc()
 		return false
 	}
 }
@@ -108,7 +114,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 
 	version, err := build.ParseVersion(string(parts[0]))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w in %q", err, parts[0])
 	}
 	report = parts[1]
 

cmd/infra/stcrashreceiver/stcrashreceiver.go

@@ -7,21 +7,18 @@
 package main
 
 import (
+	"bytes"
 	"io"
 	"log"
 	"net/http"
 	"path"
 	"strings"
-	"sync"
 )
 
 type crashReceiver struct {
 	store  *diskStore
 	sentry *sentryService
 	ignore *ignorePatterns
-
-	ignoredMut sync.RWMutex
-	ignored    map[string]struct{}
 }
 
 func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -69,12 +66,6 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
 func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		return // found
-	}
 	if !r.store.Exists(reportID) {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
@@ -87,17 +78,7 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		metricCrashReportsTotal.WithLabelValues(result).Inc()
 	}()
 
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		result = "ignored_cached"
-		io.Copy(io.Discard, req.Body)
-		return // found
-	}
-
 	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
 	bs, err := io.ReadAll(lr)
 	if err != nil {
@@ -106,14 +87,11 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		return
 	}
 
-	if r.ignore.match(bs) {
+	first := string(bytes.TrimSpace(bytes.Split(bs, []byte("\n"))[0]))
-		r.ignoredMut.Lock()
+
-		if r.ignored == nil {
+	if pat, ok := r.ignore.match(bs); ok {
-			r.ignored = make(map[string]struct{})
-		}
-		r.ignored[reportID] = struct{}{}
-		r.ignoredMut.Unlock()
 		result = "ignored"
+		log.Printf("Ignored report %s, matched: %s (%s)", reportID[:8], pat, first)
 		return
 	}
 
@@ -121,13 +99,15 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 
 	// Store the report
 	if !r.store.Put(reportID, bs) {
-		log.Println("Failed to store report (queue full):", reportID)
+		log.Println("Failed to store report (queue full):", reportID[:8])
 		result = "queue_failure"
 	}
 
 	// Send the report to Sentry
 	if !r.sentry.Send(reportID, userIDFor(req), bs) {
-		log.Println("Failed to send report to sentry (queue full):", reportID)
+		log.Println("Failed to send report to sentry (queue full):", reportID[:8])
 		result = "sentry_failure"
 	}
+
+	log.Printf("Received report %s (%s)", reportID[:8], first)
 }

cmd/stdiscosrv/stats.go

@@ -7,6 +7,8 @@
 package main
 
 import (
+	"net/http"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -113,14 +115,11 @@ var (
 )
 
 const (
-	dbOpGet             = "get"
+	dbOpGet       = "get"
-	dbOpPut             = "put"
+	dbOpPut       = "put"
-	dbOpMerge           = "merge"
+	dbOpMerge     = "merge"
-	dbOpDelete          = "delete"
+	dbResSuccess  = "success"
-	dbResSuccess        = "success"
+	dbResNotFound = "not_found"
-	dbResNotFound       = "not_found"
-	dbResError          = "error"
-	dbResUnmarshalError = "unmarsh_err"
 )
 
 func init() {
@@ -132,4 +131,24 @@ func init() {
 		databaseOperations, databaseOperationSeconds,
 		databaseWriteSeconds, databaseLastWritten,
 		retryAfterLevel)
+
+	// Prewarm important counters so they're available with zero values at
+	// startup
+
+	apiRequestsTotal.WithLabelValues(http.MethodGet, "200")
+	apiRequestsTotal.WithLabelValues(http.MethodGet, "404")
+	apiRequestsTotal.WithLabelValues(http.MethodPost, "204")
+	apiRequestsTotal.WithLabelValues(http.MethodPost, "400")
+	apiRequestsTotal.WithLabelValues(http.MethodPost, "403")
+
+	lookupRequestsTotal.WithLabelValues("success")
+	lookupRequestsTotal.WithLabelValues("not_found_ever")
+	lookupRequestsTotal.WithLabelValues("not_found_recent")
+
+	announceRequestsTotal.WithLabelValues("success")
+	announceRequestsTotal.WithLabelValues("bad_request")
+	announceRequestsTotal.WithLabelValues("no_certificate")
+
+	replicationSendsTotal.WithLabelValues("success")
+	replicationRecvsTotal.WithLabelValues("success")
 }

lib/build/parse.go

@@ -16,7 +16,7 @@ import (
 // or, somewhere along the way the "+" in the version tag disappeared:
 // syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
 var (
-	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?`)
 	gitExtraRE    = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
 	gitExtraSepRE = regexp.MustCompile(`[.-]`)              // dot or dash
 )

lib/build/parse_test.go

@@ -57,6 +57,20 @@ func TestParseVersion(t *testing.T) {
 				Extra:    []string{"Some Wrapper", "purego", "stnoupgrade"},
 			},
 		},
+		{
+			longVersion: `2026-05-18 14:53:32 INF syncthing v2.0.3 "Hafnium Hornet" (go1.25.0 darwin-amd64) builder@github.syncthing.net 2025-08-22 07:00:05 UTC [stnoupgrade] (log.pkg=main)`,
+			parsed: VersionParts{
+				Version:  "v2.0.3",
+				Tag:      "v2.0.3",
+				Commit:   "",
+				Codename: "Hafnium Hornet",
+				Runtime:  "go1.25.0",
+				GOOS:     "darwin",
+				GOARCH:   "amd64",
+				Builder:  "builder@github.syncthing.net",
+				Extra:    []string{"stnoupgrade"},
+			},
+		},
 	}
 
 	for _, tc := range cases {