diff --git a/cmd/infra/stcrashreceiver/main.go b/cmd/infra/stcrashreceiver/main.go index 315bcd0ea..30c834c2a 100644 --- a/cmd/infra/stcrashreceiver/main.go +++ b/cmd/infra/stcrashreceiver/main.go @@ -123,7 +123,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http return } - if ignore.match(bs) { + if _, ok := ignore.match(bs); ok { result = "ignored" return } @@ -216,14 +216,14 @@ func loadIgnorePatterns(path string) (*ignorePatterns, error) { return &ignorePatterns{patterns: patterns}, nil } -func (i *ignorePatterns) match(report []byte) bool { +func (i *ignorePatterns) match(report []byte) (string, bool) { if i == nil { - return false + return "", false } for _, re := range i.patterns { if re.Match(report) { - return true + return re.String(), true } } - return false + return "", false } diff --git a/cmd/infra/stcrashreceiver/metrics.go b/cmd/infra/stcrashreceiver/metrics.go index 8ccbc8fa3..c073c8c39 100644 --- a/cmd/infra/stcrashreceiver/metrics.go +++ b/cmd/infra/stcrashreceiver/metrics.go @@ -37,4 +37,9 @@ var ( Subsystem: "crashreceiver", Name: "diskstore_oldest_age_seconds", }) + metricSentryReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "syncthing", + Subsystem: "crashreceiver", + Name: "sentry_reports_total", + }, []string{"result"}) ) diff --git a/cmd/infra/stcrashreceiver/sentry.go b/cmd/infra/stcrashreceiver/sentry.go index dcc60f069..fd5ed2bfb 100644 --- a/cmd/infra/stcrashreceiver/sentry.go +++ b/cmd/infra/stcrashreceiver/sentry.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "log" "regexp" @@ -52,11 +53,15 @@ func (s *sentryService) Serve(ctx context.Context) { pkt, err := parseCrashReport(req.reportID, req.data) if err != nil { log.Println("Failed to parse crash report:", err) + metricSentryReportsTotal.WithLabelValues("parse_failure").Inc() continue } if err := sendReport(s.dsn, pkt, req.userID); err != nil { log.Println("Failed to send crash report:", err) + metricSentryReportsTotal.WithLabelValues("send_failure").Inc() + continue } + metricSentryReportsTotal.WithLabelValues("success").Inc() case <-ctx.Done(): return @@ -69,6 +74,7 @@ func (s *sentryService) Send(reportID, userID string, data []byte) bool { case s.inbox <- sentryRequest{reportID, userID, data}: return true default: + metricCrashReportsTotal.WithLabelValues("overflow").Inc() return false } } @@ -108,7 +114,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { version, err := build.ParseVersion(string(parts[0])) if err != nil { - return nil, err + return nil, fmt.Errorf("%w in %q", err, parts[0]) } report = parts[1] diff --git a/cmd/infra/stcrashreceiver/stcrashreceiver.go b/cmd/infra/stcrashreceiver/stcrashreceiver.go index d45e057ed..18776f338 100644 --- a/cmd/infra/stcrashreceiver/stcrashreceiver.go +++ b/cmd/infra/stcrashreceiver/stcrashreceiver.go @@ -7,21 +7,18 @@ package main import ( + "bytes" "io" "log" "net/http" "path" "strings" - "sync" ) type crashReceiver struct { store *diskStore sentry *sentryService ignore *ignorePatterns - - ignoredMut sync.RWMutex - ignored map[string]struct{} } func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -69,12 +66,6 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http // serveHead responds to HEAD requests by checking if the named report // already exists in the system. func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) { - r.ignoredMut.RLock() - _, ignored := r.ignored[reportID] - r.ignoredMut.RUnlock() - if ignored { - return // found - } if !r.store.Exists(reportID) { http.Error(w, "Not found", http.StatusNotFound) } @@ -87,17 +78,7 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht metricCrashReportsTotal.WithLabelValues(result).Inc() }() - r.ignoredMut.RLock() - _, ignored := r.ignored[reportID] - r.ignoredMut.RUnlock() - if ignored { - result = "ignored_cached" - io.Copy(io.Discard, req.Body) - return // found - } - // Read at most maxRequestSize of report data. - log.Println("Receiving report", reportID) lr := io.LimitReader(req.Body, maxRequestSize) bs, err := io.ReadAll(lr) if err != nil { @@ -106,14 +87,11 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht return } - if r.ignore.match(bs) { - r.ignoredMut.Lock() - if r.ignored == nil { - r.ignored = make(map[string]struct{}) - } - r.ignored[reportID] = struct{}{} - r.ignoredMut.Unlock() + first := string(bytes.TrimSpace(bytes.Split(bs, []byte("\n"))[0])) + + if pat, ok := r.ignore.match(bs); ok { result = "ignored" + log.Printf("Ignored report %s, matched: %s (%s)", reportID[:8], pat, first) return } @@ -121,13 +99,15 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht // Store the report if !r.store.Put(reportID, bs) { - log.Println("Failed to store report (queue full):", reportID) + log.Println("Failed to store report (queue full):", reportID[:8]) result = "queue_failure" } // Send the report to Sentry if !r.sentry.Send(reportID, userIDFor(req), bs) { - log.Println("Failed to send report to sentry (queue full):", reportID) + log.Println("Failed to send report to sentry (queue full):", reportID[:8]) result = "sentry_failure" } + + log.Printf("Received report %s (%s)", reportID[:8], first) } diff --git a/cmd/stdiscosrv/stats.go b/cmd/stdiscosrv/stats.go index ddc062d19..32b962ca5 100644 --- a/cmd/stdiscosrv/stats.go +++ b/cmd/stdiscosrv/stats.go @@ -7,6 +7,8 @@ package main import ( + "net/http" + "github.com/prometheus/client_golang/prometheus" ) @@ -113,14 +115,11 @@ var ( ) const ( - dbOpGet = "get" - dbOpPut = "put" - dbOpMerge = "merge" - dbOpDelete = "delete" - dbResSuccess = "success" - dbResNotFound = "not_found" - dbResError = "error" - dbResUnmarshalError = "unmarsh_err" + dbOpGet = "get" + dbOpPut = "put" + dbOpMerge = "merge" + dbResSuccess = "success" + dbResNotFound = "not_found" ) func init() { @@ -132,4 +131,24 @@ func init() { databaseOperations, databaseOperationSeconds, databaseWriteSeconds, databaseLastWritten, retryAfterLevel) + + // Prewarm important counters so they're available with zero values at + // startup + + apiRequestsTotal.WithLabelValues(http.MethodGet, "200") + apiRequestsTotal.WithLabelValues(http.MethodGet, "404") + apiRequestsTotal.WithLabelValues(http.MethodPost, "204") + apiRequestsTotal.WithLabelValues(http.MethodPost, "400") + apiRequestsTotal.WithLabelValues(http.MethodPost, "403") + + lookupRequestsTotal.WithLabelValues("success") + lookupRequestsTotal.WithLabelValues("not_found_ever") + lookupRequestsTotal.WithLabelValues("not_found_recent") + + announceRequestsTotal.WithLabelValues("success") + announceRequestsTotal.WithLabelValues("bad_request") + announceRequestsTotal.WithLabelValues("no_certificate") + + replicationSendsTotal.WithLabelValues("success") + replicationRecvsTotal.WithLabelValues("success") } diff --git a/lib/build/parse.go b/lib/build/parse.go index 59474d260..85194d486 100644 --- a/lib/build/parse.go +++ b/lib/build/parse.go @@ -16,7 +16,7 @@ import ( // or, somewhere along the way the "+" in the version tag disappeared: // syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade] var ( - longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`) + longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?`) gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618" gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash ) diff --git a/lib/build/parse_test.go b/lib/build/parse_test.go index 5cab988dd..9b6fae028 100644 --- a/lib/build/parse_test.go +++ b/lib/build/parse_test.go @@ -57,6 +57,20 @@ func TestParseVersion(t *testing.T) { Extra: []string{"Some Wrapper", "purego", "stnoupgrade"}, }, }, + { + longVersion: `2026-05-18 14:53:32 INF syncthing v2.0.3 "Hafnium Hornet" (go1.25.0 darwin-amd64) builder@github.syncthing.net 2025-08-22 07:00:05 UTC [stnoupgrade] (log.pkg=main)`, + parsed: VersionParts{ + Version: "v2.0.3", + Tag: "v2.0.3", + Commit: "", + Codename: "Hafnium Hornet", + Runtime: "go1.25.0", + GOOS: "darwin", + GOARCH: "amd64", + Builder: "builder@github.syncthing.net", + Extra: []string{"stnoupgrade"}, + }, + }, } for _, tc := range cases {