chore(stcrashreceiver): improve logging

Signed-off-by: Jakob Borg <jakob@kastelo.net>

cmd/infra/stcrashreceiver/main.go

@@ -123,7 +123,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 			return
 		}
 
-		if ignore.match(bs) {
+		if _, ok := ignore.match(bs); ok {
 			result = "ignored"
 			return
 		}
@@ -216,14 +216,14 @@ func loadIgnorePatterns(path string) (*ignorePatterns, error) {
 	return &ignorePatterns{patterns: patterns}, nil
 }
 
-func (i *ignorePatterns) match(report []byte) bool {
+func (i *ignorePatterns) match(report []byte) (string, bool) {
 	if i == nil {
-		return false
+		return "", false
 	}
 	for _, re := range i.patterns {
 		if re.Match(report) {
-			return true
+			return re.String(), true
 		}
 	}
-	return false
+	return "", false
 }

cmd/infra/stcrashreceiver/metrics.go

@@ -37,4 +37,9 @@ var (
 		Subsystem: "crashreceiver",
 		Name:      "diskstore_oldest_age_seconds",
 	})
+	metricSentryReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "crashreceiver",
+		Name:      "sentry_reports_total",
+	}, []string{"result"})
 )

cmd/infra/stcrashreceiver/sentry.go

@@ -10,6 +10,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"regexp"
@@ -52,11 +53,15 @@ func (s *sentryService) Serve(ctx context.Context) {
 			pkt, err := parseCrashReport(req.reportID, req.data)
 			if err != nil {
 				log.Println("Failed to parse crash report:", err)
+				metricSentryReportsTotal.WithLabelValues("parse_failure").Inc()
 				continue
 			}
 			if err := sendReport(s.dsn, pkt, req.userID); err != nil {
 				log.Println("Failed to send crash report:", err)
+				metricSentryReportsTotal.WithLabelValues("send_failure").Inc()
+				continue
 			}
+			metricSentryReportsTotal.WithLabelValues("success").Inc()
 
 		case <-ctx.Done():
 			return
@@ -69,6 +74,7 @@ func (s *sentryService) Send(reportID, userID string, data []byte) bool {
 	case s.inbox <- sentryRequest{reportID, userID, data}:
 		return true
 	default:
+		metricCrashReportsTotal.WithLabelValues("overflow").Inc()
 		return false
 	}
 }
@@ -108,7 +114,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 
 	version, err := build.ParseVersion(string(parts[0]))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w in %q", err, parts[0])
 	}
 	report = parts[1]
 

cmd/infra/stcrashreceiver/stcrashreceiver.go

@@ -7,21 +7,18 @@
 package main
 
 import (
+	"bytes"
 	"io"
 	"log"
 	"net/http"
 	"path"
 	"strings"
-	"sync"
 )
 
 type crashReceiver struct {
 	store  *diskStore
 	sentry *sentryService
 	ignore *ignorePatterns
-
-	ignoredMut sync.RWMutex
-	ignored    map[string]struct{}
 }
 
 func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -69,12 +66,6 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
 func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		return // found
-	}
 	if !r.store.Exists(reportID) {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
@@ -87,17 +78,7 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		metricCrashReportsTotal.WithLabelValues(result).Inc()
 	}()
 
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		result = "ignored_cached"
-		io.Copy(io.Discard, req.Body)
-		return // found
-	}
-
 	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
 	bs, err := io.ReadAll(lr)
 	if err != nil {
@@ -106,14 +87,11 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		return
 	}
 
-	if r.ignore.match(bs) {
+	first := string(bytes.TrimSpace(bytes.Split(bs, []byte("\n"))[0]))
-		r.ignoredMut.Lock()
+
-		if r.ignored == nil {
+	if pat, ok := r.ignore.match(bs); ok {
-			r.ignored = make(map[string]struct{})
-		}
-		r.ignored[reportID] = struct{}{}
-		r.ignoredMut.Unlock()
 		result = "ignored"
+		log.Printf("Ignored report %s, matched: %s (%s)", reportID[:8], pat, first)
 		return
 	}
 
@@ -121,13 +99,15 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 
 	// Store the report
 	if !r.store.Put(reportID, bs) {
-		log.Println("Failed to store report (queue full):", reportID)
+		log.Println("Failed to store report (queue full):", reportID[:8])
 		result = "queue_failure"
 	}
 
 	// Send the report to Sentry
 	if !r.sentry.Send(reportID, userIDFor(req), bs) {
-		log.Println("Failed to send report to sentry (queue full):", reportID)
+		log.Println("Failed to send report to sentry (queue full):", reportID[:8])
 		result = "sentry_failure"
 	}
+
+	log.Printf("Received report %s (%s)", reportID[:8], first)
 }