feat: switch logging framework (#10220)

This updates our logging framework from legacy freetext strings using the `log` package to structured log entries using `log/slog`. I have updated all INFO or higher level entries, but not yet DEBUG (😓)... So, at a high level: There is a slight change in log levels, effectively adding a new warning level: - DEBUG is still debug (ideally not for users but developers, though this is something we need to work on) - INFO is still info, though I've added more data here, effectively making Syncthing more verbose by default (more on this below) - WARNING is a new log level that is different from the _old_ WARNING (more below) - ERROR is what was WARNING before -- problems that must be dealt with, and also bubbled as a popup in the GUI. A new feature is that the logging level can be set per package to something other than just debug or info, and hence I feel that we can add a bit more things into INFO while moving some (in fact, most) current INFO level warnings into WARNING. For example, I think it's justified to get a log of synced files in INFO and sync failures in WARNING. These are things that have historically been tricky to debug properly, and having more information by default will be useful to many, while still making it possible get close to told level of inscrutability by setting the log level to WARNING. I'd like to get to a stage where DEBUG is never necessary to just figure out what's going on, as opposed to trying to narrow down a likely bug. Code wise: - Our logging object, generally known as `l` in each package, is now a new adapter object that provides the old API on top of the newer one. (This should go away once all old log entries are migrated.) This is only for `l.Debugln` and `l.Debugf`. - There is a new level tracker that keeps the log level for each package. - There is a nested setup of handlers, since the structure mandated by `log/slog` is slightly convoluted (imho). We do this because we need to do formatting at a "medium" level internally so we can buffer log lines in text format but with separate timestamp and log level for the API/GUI to consume. - The `debug` API call becomes a `loglevels` API call, which can set the log level to `DEBUG`, `INFO`, `WARNING` or `ERROR` per package. The GUI is updated to handle this. - Our custom `sync` package provided some debugging of mutexes quite strongly integrated into the old logging framework, only turned on when `STTRACE` was set to certain values at startup, etc. It's been a long time since this has been useful; I removed it. - The `STTRACE` env var remains and can be used the same way as before, while additionally permitting specific log levels to be specified, `STTRACE=model:WARN,scanner:DEBUG`. - There is a new command line option `--log-level=INFO` to set the default log level. - The command line options `--log-flags` and `--verbose` go away, but are currently retained as hidden & ignored options since we set them by default in some of our startup examples and Syncthing would otherwise fail to start. Sample format messages: ``` 2009-02-13 23:31:30 INF A basic info line (attr1="val with spaces" attr2=2 attr3="val\"quote" a=a log.pkg=slogutil) 2009-02-13 23:31:30 INF An info line with grouped values (attr1=val1 foo.attr2=2 foo.bar.attr3=3 a=a log.pkg=slogutil) 2009-02-13 23:31:30 INF An info line with grouped values via logger (foo.attr1=val1 foo.attr2=2 a=a log.pkg=slogutil) 2009-02-13 23:31:30 INF An info line with nested grouped values via logger (bar.foo.attr1=val1 bar.foo.attr2=2 a=a log.pkg=slogutil) 2009-02-13 23:31:30 WRN A warning entry (a=a log.pkg=slogutil) 2009-02-13 23:31:30 ERR An error (a=a log.pkg=slogutil) ``` --------- Co-authored-by: Ross Smith II <ross@smithii.com>
2025-08-07 11:19:36 +02:00
parent 49462448d0
commit 836045ee87
149 changed files with 1797 additions and 2641 deletions
@@ -13,13 +13,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"github.com/syncthing/syncthing/internal/itererr"
+	"github.com/syncthing/syncthing/internal/slogutil"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
@@ -29,13 +32,12 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/semaphore"
-	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/versioner"
 )

 var (
 	blockStats    = make(map[string]int)
-	blockStatsMut = sync.NewMutex()
+	blockStatsMut sync.Mutex
 )

 func init() {
@@ -120,7 +122,7 @@ type dbUpdateJob struct {
 }

 type sendReceiveFolder struct {
-	folder
+	*folder

 	queue              *jobQueue
 	blockPullReorderer blockPullReorderer
@@ -210,7 +212,7 @@ func (f *sendReceiveFolder) pull() (bool, error) {
 	if pullErrNum > 0 {
 		f.pullErrors = make([]FileError, 0, len(f.tempPullErrors))
 		for path, err := range f.tempPullErrors {
-			l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
+			f.sl.Warn("Failed to sync", slogutil.FilePath(path), slogutil.Error(err))
 			f.pullErrors = append(f.pullErrors, FileError{
 				Err:  err,
 				Path: path,
@@ -221,7 +223,6 @@ func (f *sendReceiveFolder) pull() (bool, error) {
 	f.errorsMut.Unlock()

 	if pullErrNum > 0 {
-		l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum)
 		f.evLogger.Log(events.FolderErrors, map[string]interface{}{
 			"folder": f.folderID,
 			"errors": f.Errors(),
@@ -245,10 +246,10 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) (int, error)
 	finisherChan := make(chan *sharedPullerState)
 	dbUpdateChan := make(chan dbUpdateJob)

-	pullWg := sync.NewWaitGroup()
-	copyWg := sync.NewWaitGroup()
-	doneWg := sync.NewWaitGroup()
-	updateWg := sync.NewWaitGroup()
+	var pullWg sync.WaitGroup
+	var copyWg sync.WaitGroup
+	var doneWg sync.WaitGroup
+	var updateWg sync.WaitGroup

 	l.Debugln(f, "copiers:", f.Copiers, "pullerPendingKiB:", f.PullerMaxPendingKiB)

@@ -422,7 +423,6 @@ loop:
 			}

 		default:
-			l.Warnln(file)
 			panic("unhandleable item type, can't happen")
 		}
 	}
@@ -554,6 +554,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 	})

 	defer func() {
+		slog.Info("Created or updated directory", f.LogAttr(), file.LogAttr())
 		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   file.Name,
@@ -568,11 +569,10 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 		mode = 0o777
 	}

-	if shouldDebug() {
+	f.sl.Debug("Need dir", "file", file, "cur", slogutil.Expensive(func() any {
 		curFile, _, _ := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
-		l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
-	}
-
+		return curFile
+	}))
 	info, err := f.mtimefs.Lstat(file.Name)
 	switch {
 	// There is already something under that name, we need to handle that.
@@ -723,6 +723,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 	})

 	defer func() {
+		if err != nil {
+			slog.Warn("Failed to handle symlink", f.LogAttr(), file.LogAttr(), slogutil.Error(err))
+		} else {
+			slog.Info("Created or updated symlink", f.LogAttr(), file.LogAttr())
+		}
 		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   file.Name,
@@ -732,10 +737,10 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 		})
 	}()

-	if shouldDebug() {
-		curFile, ok, _ := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
-		l.Debugf("need symlink\n\t%v\n\t%v", file, curFile, ok)
-	}
+	f.sl.Debug("Need symlink", slogutil.FilePath(file.Name), slog.Any("cur", slogutil.Expensive(func() any {
+		curFile, _, _ := f.model.sdb.GetDeviceFile(f.folderID, protocol.LocalDeviceID, file.Name)
+		return curFile
+	})))

 	if len(file.SymlinkTarget) == 0 {
 		// Index entry from a Syncthing predating the support for including
@@ -814,6 +819,9 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
 	defer func() {
 		if err != nil {
 			f.newPullError(file.Name, fmt.Errorf("delete dir: %w", err))
+			slog.Info("Failed to delete directory", f.LogAttr(), file.LogAttr(), slogutil.Error(err))
+		} else {
+			slog.Info("Deleted directory", f.LogAttr(), file.LogAttr())
 		}
 		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
@@ -859,7 +867,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 	// care not declare another err.
 	var err error

-	l.Debugln(f, "Deleting file", file.Name)
+	l.Debugln(f, "Deleting file or symlink", file.Name)

 	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
@@ -869,8 +877,15 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 	})

 	defer func() {
+		kind := "file"
+		if file.IsSymlink() {
+			kind = "symlink"
+		}
 		if err != nil {
 			f.newPullError(file.Name, fmt.Errorf("delete file: %w", err))
+			slog.Info("Failed to delete "+kind, f.LogAttr(), file.LogAttr(), slogutil.Error(err))
+		} else {
+			slog.Info("Deleted "+kind, f.LogAttr(), file.LogAttr())
 		}
 		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
@@ -927,6 +942,8 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 		err = nil
 		dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
 	}
+
+	slog.Info("Deleted file", f.LogAttr(), file.LogAttr())
 }

 // renameFile attempts to rename an existing file to a destination
@@ -950,6 +967,11 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 	})

 	defer func() {
+		if err != nil {
+			slog.Info("Failed to rename file", f.LogAttr(), target.LogAttr(), slog.String("from", source.Name), slogutil.Error(err))
+		} else {
+			slog.Info("Renamed file", f.LogAttr(), target.LogAttr(), slog.String("from", source.Name))
+		}
 		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   source.Name,
@@ -1237,13 +1259,20 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo, dbUpdateChan ch
 	})

 	var err error
-	defer f.evLogger.Log(events.ItemFinished, map[string]interface{}{
-		"folder": f.folderID,
-		"item":   file.Name,
-		"error":  events.Error(err),
-		"type":   "file",
-		"action": "metadata",
-	})
+	defer func() {
+		if err != nil {
+			slog.Info("Failed to update file metadata", f.LogAttr(), file.LogAttr(), slogutil.Error(err))
+		} else {
+			slog.Info("Updated file metadata", f.LogAttr(), file.LogAttr())
+		}
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
+			"folder": f.folderID,
+			"item":   file.Name,
+			"error":  events.Error(err),
+			"type":   "file",
+			"action": "metadata",
+		})
+	}()

 	f.queue.Done(file.Name)

@@ -1395,7 +1424,7 @@ func (f *sendReceiveFolder) copyBlockFromFolder(folderID string, block protocol.
 			// We just ignore this and continue pulling instead (though
 			// there's a good chance that will fail too, if the DB is
 			// unhealthy).
-			l.Debugf("Failed to get information from DB about block %v in copier (folderID %v, file %v): %v", block.Hash, f.folderID, state.file.Name)
+			l.Debugf("Failed to get information from DB about block %v in copier (folderID %v, file %v): %v", block.Hash, f.folderID, state.file.Name, err)
 			return false
 		}

@@ -1480,7 +1509,7 @@ func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) err

 func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
 	requestLimiter := semaphore.New(f.PullerMaxPendingKiB * 1024)
-	wg := sync.NewWaitGroup()
+	var wg sync.WaitGroup

 	for state := range in {
 		if state.failed() != nil {
@@ -1666,6 +1695,8 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpda
 			if err != nil {
 				f.newPullError(state.file.Name, fmt.Errorf("finishing: %w", err))
 			} else {
+				slog.Info("Synced file", f.LogAttr(), state.file.LogAttr(), slog.Group("blocks", slog.Int("local", state.reused+state.copyTotal), slog.Int("download", state.pullTotal)))
+
 				minBlocksPerBlock := state.file.BlockSize() / protocol.MinBlockSize
 				blockStatsMut.Lock()
 				blockStats["total"] += (state.reused + state.copyTotal + state.pullTotal) * minBlocksPerBlock
@@ -1673,8 +1704,7 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpda
 				blockStats["pulled"] += state.pullTotal * minBlocksPerBlock
 				// copyOriginShifted is counted towards copyOrigin due to progress bar reasons
 				// for reporting reasons we want to separate these.
-				blockStats["copyOrigin"] += (state.copyOrigin - state.copyOriginShifted) * minBlocksPerBlock
-				blockStats["copyOriginShifted"] += state.copyOriginShifted * minBlocksPerBlock
+				blockStats["copyOrigin"] += state.copyOrigin * minBlocksPerBlock
 				blockStats["copyElsewhere"] += (state.copyTotal - state.copyOrigin) * minBlocksPerBlock
 				blockStatsMut.Unlock()
 			}
@@ -1777,7 +1807,7 @@ loop:
 					// (resp. whatever caused the error) will cause this file to
 					// change. Log at info level to leave a trace if a user
 					// notices, but no need to warn
-					l.Infof("Error updating metadata for %v at database commit: %v", job.file.Name, err)
+					f.sl.Warn("Failed to update metadata at database commit", slogutil.FilePath(job.file.Name), slogutil.Error(err))
 				}
 			}
 			job.file.Sequence = 0
@@ -1831,7 +1861,7 @@ func (f *sendReceiveFolder) inConflict(current, replacement protocol.Vector) boo

 func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan chan<- string) error {
 	if isConflict(name) {
-		l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.")
+		f.sl.Info("Conflict on existing conflict copy; not copying again", slogutil.FilePath(name))
 		if err := f.mtimefs.Remove(name); err != nil && !fs.IsNotExist(err) {
 			return fmt.Errorf("%s: %w", contextRemovingOldItem, err)
 		}