all: Send deadlocks as failures, crash only as a last resort (#7785)

cmd/stcrashreceiver/main.go

@@ -19,10 +19,9 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net"
 	"net/http"
 	"os"
-	"time"
+	"path/filepath"
 
 	"github.com/syncthing/syncthing/lib/sha256"
 	"github.com/syncthing/syncthing/lib/ur"
@@ -33,7 +32,7 @@ import (
 const maxRequestSize = 1 << 20 // 1 MiB
 
 func main() {
-	dir := flag.String("dir", ".", "Directory to store reports in")
+	dir := flag.String("dir", ".", "Parent directory to store crash and failure reports in")
 	dsn := flag.String("dsn", "", "Sentry DSN")
 	listen := flag.String("listen", ":22039", "HTTP listen address")
 	flag.Parse()
@@ -41,13 +40,13 @@ func main() {
 	mux := http.NewServeMux()
 
 	cr := &crashReceiver{
-		dir: *dir,
+		dir: filepath.Join(*dir, "crash_reports"),
 		dsn: *dsn,
 	}
 	mux.Handle("/", cr)
 
 	if *dsn != "" {
-		mux.HandleFunc("/newcrash/failure", handleFailureFn(*dsn))
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(*dsn, filepath.Join(*dir, "failure_reports")))
 	}
 
 	log.SetOutput(os.Stdout)
@@ -56,7 +55,7 @@ func main() {
 	}
 }
 
-func handleFailureFn(dsn string) func(w http.ResponseWriter, req *http.Request) {
+func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
 		lr := io.LimitReader(req.Body, maxRequestSize)
 		bs, err := ioutil.ReadAll(lr)
@@ -89,6 +88,18 @@ func handleFailureFn(dsn string) func(w http.ResponseWriter, req *http.Request)
 			pkt.Extra = raven.Extra{
 				"count": r.Count,
 			}
+			for k, v := range r.Extra {
+				pkt.Extra[k] = v
+			}
+			if len(r.Goroutines) != 0 {
+				url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
+				if err != nil {
+					log.Println("Saving failure report:", err)
+					http.Error(w, "Internal server error", http.StatusInternalServerError)
+					return
+				}
+				pkt.Extra["goroutinesURL"] = url
+			}
 			message := sanitizeMessageLDB(r.Description)
 			pkt.Fingerprint = []string{message}
 
@@ -101,19 +112,15 @@ func handleFailureFn(dsn string) func(w http.ResponseWriter, req *http.Request)
 	}
 }
 
-// userIDFor returns a string we can use as the user ID for the purpose of
-// counting affected users. It's the truncated hash of a salt, the user
-// remote IP, and the current month.
-func userIDFor(req *http.Request) string {
-	addr := req.RemoteAddr
-	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
-		addr = fwd
+func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
+	bs := make([]byte, len(data.Description)+len(data.Goroutines))
+	copy(bs, data.Description)
+	copy(bs[len(data.Description):], data.Goroutines)
+	id := fmt.Sprintf("%x", sha256.Sum256(bs))
+	path := fullPathCompressed(failureDir, id)
+	err := compressAndWrite(bs, path)
+	if err != nil {
+		return "", err
 	}
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-	now := time.Now().Format("200601")
-	salt := "stcrashreporter"
-	hash := sha256.Sum256([]byte(salt + addr + now))
-	return fmt.Sprintf("%x", hash[:8])
+	return reportServer + path, nil
 }

cmd/stcrashreceiver/util.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}
+
+// 01234567890abcdef... => 01/23
+func dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}
+
+func fullPathCompressed(root, reportID string) string {
+	return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
+}
+
+func compressAndWrite(bs []byte, fullPath string) error {
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	return ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+}