all: Handle errors opening db/creating file-set (ref #5907) (#7150)

lib/db/lowlevel.go

@@ -10,6 +10,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -19,6 +20,7 @@ import (
 	"github.com/dchest/siphash"
 	"github.com/greatroar/blobloom"
 	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
@@ -66,9 +68,10 @@ type Lowlevel struct {
 	indirectGCInterval time.Duration
 	recheckInterval    time.Duration
 	oneFileSetCreated  chan struct{}
+	evLogger           events.Logger
 }
 
-func NewLowlevel(backend backend.Backend, opts ...Option) *Lowlevel {
+func NewLowlevel(backend backend.Backend, evLogger events.Logger, opts ...Option) (*Lowlevel, error) {
 	// Only log restarts in debug mode.
 	spec := util.SpecWithDebugLogger(l)
 	db := &Lowlevel{
@@ -80,6 +83,7 @@ func NewLowlevel(backend backend.Backend, opts ...Option) *Lowlevel {
 		indirectGCInterval: indirectGCDefaultInterval,
 		recheckInterval:    recheckDefaultInterval,
 		oneFileSetCreated:  make(chan struct{}),
+		evLogger:           evLogger,
 	}
 	for _, opt := range opts {
 		opt(db)
@@ -89,11 +93,14 @@ func NewLowlevel(backend backend.Backend, opts ...Option) *Lowlevel {
 	if path := db.needsRepairPath(); path != "" {
 		if _, err := os.Lstat(path); err == nil {
 			l.Infoln("Database was marked for repair - this may take a while")
-			db.checkRepair()
+			if err := db.checkRepair(); err != nil {
+				db.handleFailure(err)
+				return nil, err
+			}
 			os.Remove(path)
 		}
 	}
-	return db
+	return db, nil
 }
 
 type Option func(*Lowlevel)
@@ -822,29 +829,22 @@ func (b *bloomFilter) hash(id []byte) uint64 {
 }
 
 // checkRepair checks folder metadata and sequences for miscellaneous errors.
-func (db *Lowlevel) checkRepair() {
+func (db *Lowlevel) checkRepair() error {
 	for _, folder := range db.ListFolders() {
-		_ = db.getMetaAndCheck(folder)
+		if _, err := db.getMetaAndCheck(folder); err != nil {
+			return err
+		}
 	}
+	return nil
 }
 
-func (db *Lowlevel) getMetaAndCheck(folder string) *metadataTracker {
+func (db *Lowlevel) getMetaAndCheck(folder string) (*metadataTracker, error) {
 	db.gcMut.RLock()
 	defer db.gcMut.RUnlock()
 
-	var err error
-	defer func() {
-		if err != nil && !backend.IsClosed(err) {
-			l.Warnf("Fatal error: %v", err)
-			obfuscateAndPanic(err)
-		}
-	}()
-
-	var fixed int
-	fixed, err = db.checkLocalNeed([]byte(folder))
+	fixed, err := db.checkLocalNeed([]byte(folder))
 	if err != nil {
-		err = fmt.Errorf("checking local need: %w", err)
-		return nil
+		return nil, fmt.Errorf("checking local need: %w", err)
 	}
 	if fixed != 0 {
 		l.Infof("Repaired %d local need entries for folder %v in database", fixed, folder)
@@ -852,24 +852,22 @@ func (db *Lowlevel) getMetaAndCheck(folder string) *metadataTracker {
 
 	meta, err := db.recalcMeta(folder)
 	if err != nil {
-		err = fmt.Errorf("recalculating metadata: %w", err)
-		return nil
+		return nil, fmt.Errorf("recalculating metadata: %w", err)
 	}
 
 	fixed, err = db.repairSequenceGCLocked(folder, meta)
 	if err != nil {
-		err = fmt.Errorf("repairing sequences: %w", err)
-		return nil
+		return nil, fmt.Errorf("repairing sequences: %w", err)
 	}
 	if fixed != 0 {
 		l.Infof("Repaired %d sequence entries for folder %v in database", fixed, folder)
 	}
 
-	return meta
+	return meta, nil
 }
 
-func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
-	meta := newMetadataTracker(db.keyer)
+func (db *Lowlevel) loadMetadataTracker(folder string) (*metadataTracker, error) {
+	meta := newMetadataTracker(db.keyer, db.evLogger)
 	if err := meta.fromDB(db, []byte(folder)); err != nil {
 		if err == errMetaInconsistent {
 			l.Infof("Stored folder metadata for %q is inconsistent; recalculating", folder)
@@ -881,7 +879,9 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
 	}
 
 	curSeq := meta.Sequence(protocol.LocalDeviceID)
-	if metaOK := db.verifyLocalSequence(curSeq, folder); !metaOK {
+	if metaOK, err := db.verifyLocalSequence(curSeq, folder); err != nil {
+		return nil, fmt.Errorf("verifying sequences: %w", err)
+	} else if !metaOK {
 		l.Infof("Stored folder metadata for %q is out of date after crash; recalculating", folder)
 		return db.getMetaAndCheck(folder)
 	}
@@ -891,13 +891,13 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
 		return db.getMetaAndCheck(folder)
 	}
 
-	return meta
+	return meta, nil
 }
 
 func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
 	folder := []byte(folderStr)
 
-	meta := newMetadataTracker(db.keyer)
+	meta := newMetadataTracker(db.keyer, db.evLogger)
 	if err := db.checkGlobals(folder); err != nil {
 		return nil, fmt.Errorf("checking globals: %w", err)
 	}
@@ -951,7 +951,7 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
 
 // Verify the local sequence number from actual sequence entries. Returns
 // true if it was all good, or false if a fixup was necessary.
-func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool {
+func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) (bool, error) {
 	// Walk the sequence index from the current (supposedly) highest
 	// sequence number and raise the alarm if we get anything. This recovers
 	// from the occasion where we have written sequence entries to disk but
@@ -964,20 +964,18 @@ func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool {
 
 	t, err := db.newReadOnlyTransaction()
 	if err != nil {
-		l.Warnf("Fatal error: %v", err)
-		obfuscateAndPanic(err)
+		return false, err
 	}
 	ok := true
 	if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi protocol.FileIntf) bool {
 		ok = false // we got something, which we should not have
 		return false
-	}); err != nil && !backend.IsClosed(err) {
-		l.Warnf("Fatal error: %v", err)
-		obfuscateAndPanic(err)
+	}); err != nil {
+		return false, err
 	}
 	t.close()
 
-	return ok
+	return ok, nil
 }
 
 // repairSequenceGCLocked makes sure the sequence numbers in the sequence keys
@@ -1177,6 +1175,17 @@ func (db *Lowlevel) needsRepairPath() string {
 	return path + needsRepairSuffix
 }
 
+func (db *Lowlevel) checkErrorForRepair(err error) {
+	if errors.Is(err, errEntryFromGlobalMissing) || errors.Is(err, errEmptyGlobal) {
+		// Inconsistency error, mark db for repair on next start.
+		if path := db.needsRepairPath(); path != "" {
+			if fd, err := os.Create(path); err == nil {
+				fd.Close()
+			}
+		}
+	}
+}
+
 // unchanged checks if two files are the same and thus don't need to be updated.
 // Local flags or the invalid bit might change without the version
 // being bumped.
@@ -1184,8 +1193,15 @@ func unchanged(nf, ef protocol.FileIntf) bool {
 	return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags()
 }
 
+func (db *Lowlevel) handleFailure(err error) {
+	db.checkErrorForRepair(err)
+	if shouldReportFailure(err) {
+		db.evLogger.Log(events.Failure, err)
+	}
+}
+
 var ldbPathRe = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
 
-func obfuscateAndPanic(err error) {
-	panic(ldbPathRe.ReplaceAllString(err.Error(), "$1 x: "))
+func shouldReportFailure(err error) bool {
+	return !ldbPathRe.MatchString(err.Error())
 }