all: Send deadlocks as failures, crash only as a last resort (#7785)

lib/model/model.go

@@ -324,7 +324,7 @@ func (m *model) fatal(err error) {
 // period.
 func (m *model) StartDeadlockDetector(timeout time.Duration) {
 	l.Infof("Starting deadlock detector with %v timeout", timeout)
-	detector := newDeadlockDetector(timeout)
+	detector := newDeadlockDetector(timeout, m.evLogger, m.fatal)
 	detector.Watch("fmut", m.fmut)
 	detector.Watch("pmut", m.pmut)
 }

lib/model/util.go

@@ -9,61 +9,100 @@ package model
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/ur"
 )
 
 type Holdable interface {
 	Holders() string
 }
 
-func newDeadlockDetector(timeout time.Duration) *deadlockDetector {
+func newDeadlockDetector(timeout time.Duration, evLogger events.Logger, fatal func(error)) *deadlockDetector {
 	return &deadlockDetector{
-		timeout: timeout,
-		lockers: make(map[string]sync.Locker),
+		warnTimeout:  timeout,
+		fatalTimeout: 10 * timeout,
+		lockers:      make(map[string]sync.Locker),
+		evLogger:     evLogger,
+		fatal:        fatal,
 	}
 }
 
 type deadlockDetector struct {
-	timeout time.Duration
-	lockers map[string]sync.Locker
+	warnTimeout, fatalTimeout time.Duration
+	lockers                   map[string]sync.Locker
+	evLogger                  events.Logger
+	fatal                     func(error)
 }
 
 func (d *deadlockDetector) Watch(name string, mut sync.Locker) {
 	d.lockers[name] = mut
 	go func() {
 		for {
-			time.Sleep(d.timeout / 4)
-			ok := make(chan bool, 2)
+			time.Sleep(d.warnTimeout / 4)
+			done := make(chan struct{}, 1)
 
 			go func() {
 				mut.Lock()
 				_ = 1 // empty critical section
 				mut.Unlock()
-				ok <- true
+				done <- struct{}{}
 			}()
 
-			go func() {
-				time.Sleep(d.timeout)
-				ok <- false
-			}()
-
-			if r := <-ok; !r {
-				msg := fmt.Sprintf("deadlock detected at %s", name)
-				for otherName, otherMut := range d.lockers {
-					if otherHolder, ok := otherMut.(Holdable); ok {
-						msg += "\n===" + otherName + "===\n" + otherHolder.Holders()
-					}
-				}
-				panic(msg)
-			}
+			d.watchInner(name, done)
 		}
 	}()
 }
 
+func (d *deadlockDetector) watchInner(name string, done chan struct{}) {
+	warn := time.NewTimer(d.warnTimeout)
+	fatal := time.NewTimer(d.fatalTimeout)
+	defer func() {
+		warn.Stop()
+		fatal.Stop()
+	}()
+
+	select {
+	case <-warn.C:
+		failure := ur.FailureDataWithGoroutines(fmt.Sprintf("potential deadlock detected at %s (short timeout)", name))
+		failure.Extra["timeout"] = d.warnTimeout.String()
+		d.evLogger.Log(events.Failure, failure)
+	case <-done:
+		return
+	}
+
+	select {
+	case <-fatal.C:
+		err := fmt.Errorf("potential deadlock detected at %s (long timeout)", name)
+		failure := ur.FailureDataWithGoroutines(err.Error())
+		failure.Extra["timeout"] = d.fatalTimeout.String()
+		others := d.otherHolders()
+		failure.Extra["other-holders"] = others
+		d.evLogger.Log(events.Failure, failure)
+		d.fatal(err)
+		// Give it a minute to shut down gracefully, maybe shutting down
+		// can get out of the deadlock (or it's not really a deadlock).
+		time.Sleep(time.Minute)
+		panic(fmt.Sprintf("%v:\n%v", err, others))
+	case <-done:
+	}
+}
+
+func (d *deadlockDetector) otherHolders() string {
+	var b strings.Builder
+	for otherName, otherMut := range d.lockers {
+		if otherHolder, ok := otherMut.(Holdable); ok {
+			b.WriteString("===" + otherName + "===\n" + otherHolder.Holders() + "\n")
+		}
+	}
+	return b.String()
+}
+
 // inWritableDir calls fn(path), while making sure that the directory
 // containing `path` is writable for the duration of the call.
 func inWritableDir(fn func(string) error, targetFs fs.Filesystem, path string, ignorePerms bool) error {