feat: switch logging framework (#10220)
This updates our logging framework from legacy freetext strings using the `log` package to structured log entries using `log/slog`. I have updated all INFO or higher level entries, but not yet DEBUG (😓)... So, at a high level: There is a slight change in log levels, effectively adding a new warning level: - DEBUG is still debug (ideally not for users but developers, though this is something we need to work on) - INFO is still info, though I've added more data here, effectively making Syncthing more verbose by default (more on this below) - WARNING is a new log level that is different from the _old_ WARNING (more below) - ERROR is what was WARNING before -- problems that must be dealt with, and also bubbled as a popup in the GUI. A new feature is that the logging level can be set per package to something other than just debug or info, and hence I feel that we can add a bit more things into INFO while moving some (in fact, most) current INFO level warnings into WARNING. For example, I think it's justified to get a log of synced files in INFO and sync failures in WARNING. These are things that have historically been tricky to debug properly, and having more information by default will be useful to many, while still making it possible get close to told level of inscrutability by setting the log level to WARNING. I'd like to get to a stage where DEBUG is never necessary to just figure out what's going on, as opposed to trying to narrow down a likely bug. Code wise: - Our logging object, generally known as `l` in each package, is now a new adapter object that provides the old API on top of the newer one. (This should go away once all old log entries are migrated.) This is only for `l.Debugln` and `l.Debugf`. - There is a new level tracker that keeps the log level for each package. - There is a nested setup of handlers, since the structure mandated by `log/slog` is slightly convoluted (imho). We do this because we need to do formatting at a "medium" level internally so we can buffer log lines in text format but with separate timestamp and log level for the API/GUI to consume. - The `debug` API call becomes a `loglevels` API call, which can set the log level to `DEBUG`, `INFO`, `WARNING` or `ERROR` per package. The GUI is updated to handle this. - Our custom `sync` package provided some debugging of mutexes quite strongly integrated into the old logging framework, only turned on when `STTRACE` was set to certain values at startup, etc. It's been a long time since this has been useful; I removed it. - The `STTRACE` env var remains and can be used the same way as before, while additionally permitting specific log levels to be specified, `STTRACE=model:WARN,scanner:DEBUG`. - There is a new command line option `--log-level=INFO` to set the default log level. - The command line options `--log-flags` and `--verbose` go away, but are currently retained as hidden & ignored options since we set them by default in some of our startup examples and Syncthing would otherwise fail to start. Sample format messages: ``` 2009-02-13 23:31:30 INF A basic info line (attr1="val with spaces" attr2=2 attr3="val\"quote" a=a log.pkg=slogutil) 2009-02-13 23:31:30 INF An info line with grouped values (attr1=val1 foo.attr2=2 foo.bar.attr3=3 a=a log.pkg=slogutil) 2009-02-13 23:31:30 INF An info line with grouped values via logger (foo.attr1=val1 foo.attr2=2 a=a log.pkg=slogutil) 2009-02-13 23:31:30 INF An info line with nested grouped values via logger (bar.foo.attr1=val1 bar.foo.attr2=2 a=a log.pkg=slogutil) 2009-02-13 23:31:30 WRN A warning entry (a=a log.pkg=slogutil) 2009-02-13 23:31:30 ERR An error (a=a log.pkg=slogutil) ``` --------- Co-authored-by: Ross Smith II <ross@smithii.com>
This commit is contained in:
Jakob Borg
co-authored by
Ross Smith II
Ross Smith II
parent
49462448d0
commit
836045ee87
+40
-49
@@ -17,6 +17,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
@@ -42,6 +44,7 @@ import (
|
||||
"golang.org/x/text/unicode/norm"
|
||||
|
||||
"github.com/syncthing/syncthing/internal/db"
|
||||
"github.com/syncthing/syncthing/internal/slogutil"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/connections"
|
||||
@@ -49,12 +52,10 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/events"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/locations"
|
||||
"github.com/syncthing/syncthing/lib/logger"
|
||||
"github.com/syncthing/syncthing/lib/model"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/rand"
|
||||
"github.com/syncthing/syncthing/lib/svcutil"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
"github.com/syncthing/syncthing/lib/ur"
|
||||
@@ -95,8 +96,8 @@ type service struct {
|
||||
miscDB *db.Typed
|
||||
shutdownTimeout time.Duration
|
||||
|
||||
guiErrors logger.Recorder
|
||||
systemLog logger.Recorder
|
||||
guiErrors slogutil.Recorder
|
||||
systemLog slogutil.Recorder
|
||||
}
|
||||
|
||||
var _ config.Verifier = &service{}
|
||||
@@ -107,7 +108,7 @@ type Service interface {
|
||||
WaitForStart() error
|
||||
}
|
||||
|
||||
func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, noUpgrade bool, miscDB *db.Typed) Service {
|
||||
func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog slogutil.Recorder, noUpgrade bool, miscDB *db.Typed) Service {
|
||||
return &service{
|
||||
id: id,
|
||||
cfg: cfg,
|
||||
@@ -117,7 +118,6 @@ func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonNam
|
||||
DefaultEventMask: defaultSub,
|
||||
DiskEventMask: diskSub,
|
||||
},
|
||||
eventSubsMut: sync.NewMutex(),
|
||||
evLogger: evLogger,
|
||||
discoverer: discoverer,
|
||||
connectionsService: connectionsService,
|
||||
@@ -151,8 +151,10 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
|
||||
err = shouldRegenerateCertificate(cert)
|
||||
}
|
||||
if err != nil {
|
||||
l.Infoln("Loading HTTPS certificate:", err)
|
||||
l.Infoln("Creating new HTTPS certificate")
|
||||
if !os.IsNotExist(err) {
|
||||
slog.Warn("Failed to load HTTPS certificate", slogutil.Error(err))
|
||||
}
|
||||
slog.Info("Creating new HTTPS certificate")
|
||||
|
||||
// When generating the HTTPS certificate, use the system host name per
|
||||
// default. If that isn't available, use the "syncthing" default.
|
||||
@@ -222,7 +224,7 @@ func (s *service) Serve(ctx context.Context) error {
|
||||
case <-s.startedOnce:
|
||||
// We let this be a loud user-visible warning as it may be the only
|
||||
// indication they get that the GUI won't be available.
|
||||
l.Warnln("Starting API/GUI:", err)
|
||||
slog.ErrorContext(ctx, "Failed to start API/GUI", slogutil.Error(err))
|
||||
|
||||
default:
|
||||
// This is during initialization. A failure here should be fatal
|
||||
@@ -280,7 +282,7 @@ func (s *service) Serve(ctx context.Context) error {
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/status", s.getSystemStatus) // -
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/upgrade", s.getSystemUpgrade) // -
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/version", s.getSystemVersion) // -
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/debug", s.getSystemDebug) // -
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/loglevels", s.getSystemDebug) // -
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/log", s.getSystemLog) // [since]
|
||||
restMux.HandlerFunc(http.MethodGet, "/rest/system/log.txt", s.getSystemLogTxt) // [since]
|
||||
|
||||
@@ -300,7 +302,7 @@ func (s *service) Serve(ctx context.Context) error {
|
||||
restMux.HandlerFunc(http.MethodPost, "/rest/system/upgrade", s.postSystemUpgrade) // -
|
||||
restMux.HandlerFunc(http.MethodPost, "/rest/system/pause", s.makeDevicePauseHandler(true)) // [device]
|
||||
restMux.HandlerFunc(http.MethodPost, "/rest/system/resume", s.makeDevicePauseHandler(false)) // [device]
|
||||
restMux.HandlerFunc(http.MethodPost, "/rest/system/debug", s.postSystemDebug) // [enable] [disable]
|
||||
restMux.HandlerFunc(http.MethodPost, "/rest/system/loglevels", s.postSystemDebug) // [enable] [disable]
|
||||
|
||||
// The DELETE handlers
|
||||
restMux.HandlerFunc(http.MethodDelete, "/rest/cluster/pending/devices", s.deletePendingDevices) // device
|
||||
@@ -409,8 +411,8 @@ func (s *service) Serve(ctx context.Context) error {
|
||||
srv.ErrorLog = log.Default()
|
||||
}
|
||||
|
||||
l.Infoln("GUI and API listening on", listener.Addr())
|
||||
l.Infoln("Access the GUI via the following URL:", guiCfg.URL())
|
||||
slog.InfoContext(ctx, "GUI and API listening", slogutil.Address(listener.Addr()))
|
||||
slog.InfoContext(ctx, "Access the GUI via the following URL: "+guiCfg.URL()) //nolint:sloglint
|
||||
if s.started != nil {
|
||||
// only set when run by the tests
|
||||
select {
|
||||
@@ -443,14 +445,14 @@ func (s *service) Serve(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Shutting down permanently
|
||||
l.Debugln("shutting down (stop)")
|
||||
slog.DebugContext(ctx, "Shutting down (stop)")
|
||||
case <-s.configChanged:
|
||||
// Soft restart due to configuration change
|
||||
l.Debugln("restarting (config changed)")
|
||||
slog.DebugContext(ctx, "Restarting (config changed)")
|
||||
case err = <-s.exitChan:
|
||||
case err = <-serveError:
|
||||
// Restart due to listen/serve failure
|
||||
l.Warnln("GUI/API:", err, "(restarting)")
|
||||
slog.ErrorContext(ctx, "GUI/API error (restarting)", slogutil.Error(err))
|
||||
}
|
||||
// Give it a moment to shut down gracefully, e.g. if we are restarting
|
||||
// due to a config change through the API, let that finish successfully.
|
||||
@@ -749,31 +751,20 @@ func (*service) getSystemVersion(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
func (*service) getSystemDebug(w http.ResponseWriter, _ *http.Request) {
|
||||
names := l.Facilities()
|
||||
enabled := l.FacilityDebugging()
|
||||
slices.Sort(enabled)
|
||||
sendJSON(w, map[string]interface{}{
|
||||
"facilities": names,
|
||||
"enabled": enabled,
|
||||
sendJSON(w, map[string]any{
|
||||
"packages": slogutil.PackageDescrs(),
|
||||
"levels": slogutil.PackageLevels(),
|
||||
})
|
||||
}
|
||||
|
||||
func (*service) postSystemDebug(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
q := r.URL.Query()
|
||||
for _, f := range strings.Split(q.Get("enable"), ",") {
|
||||
if f == "" || l.ShouldDebug(f) {
|
||||
continue
|
||||
}
|
||||
l.SetDebug(f, true)
|
||||
l.Infof("Enabled debug data for %q", f)
|
||||
var levelRequest map[string]slog.Level
|
||||
if err := json.NewDecoder(r.Body).Decode(&levelRequest); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, f := range strings.Split(q.Get("disable"), ",") {
|
||||
if f == "" || !l.ShouldDebug(f) {
|
||||
continue
|
||||
}
|
||||
l.SetDebug(f, false)
|
||||
l.Infof("Disabled debug data for %q", f)
|
||||
for pkg, level := range levelRequest {
|
||||
slogutil.SetPackageLevel(pkg, level)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1106,7 +1097,7 @@ func (s *service) getSystemStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
func (s *service) getSystemError(w http.ResponseWriter, _ *http.Request) {
|
||||
sendJSON(w, map[string][]logger.Line{
|
||||
sendJSON(w, map[string][]slogutil.Line{
|
||||
"errors": s.guiErrors.Since(time.Time{}),
|
||||
})
|
||||
}
|
||||
@@ -1114,7 +1105,7 @@ func (s *service) getSystemError(w http.ResponseWriter, _ *http.Request) {
|
||||
func (*service) postSystemError(_ http.ResponseWriter, r *http.Request) {
|
||||
bs, _ := io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
l.Warnln(string(bs))
|
||||
slog.Error("External error report", slogutil.Error(string(bs)))
|
||||
}
|
||||
|
||||
func (s *service) postSystemErrorClear(_ http.ResponseWriter, _ *http.Request) {
|
||||
@@ -1127,7 +1118,7 @@ func (s *service) getSystemLog(w http.ResponseWriter, r *http.Request) {
|
||||
if err != nil {
|
||||
l.Debugln(err)
|
||||
}
|
||||
sendJSON(w, map[string][]logger.Line{
|
||||
sendJSON(w, map[string][]slogutil.Line{
|
||||
"messages": s.systemLog.Since(since),
|
||||
})
|
||||
}
|
||||
@@ -1156,7 +1147,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Redacted configuration as a JSON
|
||||
if jsonConfig, err := json.MarshalIndent(getRedactedConfig(s), "", " "); err != nil {
|
||||
l.Warnln("Support bundle: failed to create config.json:", err)
|
||||
slog.Warn("Failed to create config.json in support bundle", slogutil.Error(err))
|
||||
} else {
|
||||
files = append(files, fileEntry{name: "config.json.txt", data: jsonConfig})
|
||||
}
|
||||
@@ -1171,7 +1162,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
// Errors as a JSON
|
||||
if errs := s.guiErrors.Since(time.Time{}); len(errs) > 0 {
|
||||
if jsonError, err := json.MarshalIndent(errs, "", " "); err != nil {
|
||||
l.Warnln("Support bundle: failed to create errors.json:", err)
|
||||
slog.Warn("Failed to create errors.json in support bundle", slogutil.Error(err))
|
||||
} else {
|
||||
files = append(files, fileEntry{name: "errors.json.txt", data: jsonError})
|
||||
}
|
||||
@@ -1181,7 +1172,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
if panicFiles, err := filepath.Glob(filepath.Join(locations.GetBaseDir(locations.ConfigBaseDir), "panic*")); err == nil {
|
||||
for _, f := range panicFiles {
|
||||
if panicFile, err := os.ReadFile(f); err != nil {
|
||||
l.Warnf("Support bundle: failed to load %s: %s", filepath.Base(f), err)
|
||||
slog.Warn("Failed to load panic file for support bundle", slogutil.FilePath(filepath.Base(f)), slogutil.Error(err))
|
||||
} else {
|
||||
files = append(files, fileEntry{name: filepath.Base(f), data: panicFile})
|
||||
}
|
||||
@@ -1204,15 +1195,15 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
}, "", " "); err == nil {
|
||||
files = append(files, fileEntry{name: "version-platform.json.txt", data: versionPlatform})
|
||||
} else {
|
||||
l.Warnln("Failed to create versionPlatform.json: ", err)
|
||||
slog.Warn("Failed to create versionPlatform.json in support bundle", slogutil.Error(err))
|
||||
}
|
||||
|
||||
// Report Data as a JSON
|
||||
if r, err := s.urService.ReportDataPreview(r.Context(), ur.Version); err != nil {
|
||||
l.Warnln("Support bundle: failed to create usage-reporting.json.txt:", err)
|
||||
slog.Warn("Failed to create usage-reporting.json.txt in support bundle", slogutil.Error(err))
|
||||
} else {
|
||||
if usageReportingData, err := json.MarshalIndent(r, "", " "); err != nil {
|
||||
l.Warnln("Support bundle: failed to serialize usage-reporting.json.txt", err)
|
||||
slog.Warn("Failed to serialize usage-reporting.json.txt in support bundle", slogutil.Error(err))
|
||||
} else {
|
||||
files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData})
|
||||
}
|
||||
@@ -1227,7 +1218,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
// Connection data as JSON
|
||||
connStats := s.model.ConnectionStats()
|
||||
if connStatsJSON, err := json.MarshalIndent(connStats, "", " "); err != nil {
|
||||
l.Warnln("Support bundle: failed to serialize connection-stats.json.txt", err)
|
||||
slog.Warn("Failed to serialize connection-stats.json.txt in support bundle", slogutil.Error(err))
|
||||
} else {
|
||||
files = append(files, fileEntry{name: "connection-stats.json.txt", data: connStatsJSON})
|
||||
}
|
||||
@@ -1273,7 +1264,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
// Add buffer files to buffer zip
|
||||
var zipFilesBuffer bytes.Buffer
|
||||
if err := writeZip(&zipFilesBuffer, files); err != nil {
|
||||
l.Warnln("Support bundle: failed to create support bundle zip:", err)
|
||||
slog.Warn("Failed to create support bundle zip (buffer)", slogutil.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
@@ -1284,7 +1275,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Write buffer zip to local zip file (back up)
|
||||
if err := os.WriteFile(zipFilePath, zipFilesBuffer.Bytes(), 0o600); err != nil {
|
||||
l.Warnln("Support bundle: support bundle zip could not be created:", err)
|
||||
slog.Warn("Failed to create support bundle zip (file)", slogutil.FilePath(zipFilePath), slogutil.Error(err))
|
||||
}
|
||||
|
||||
// Serve the buffer zip to client for download
|
||||
@@ -1534,7 +1525,7 @@ func (s *service) postSystemUpgrade(w http.ResponseWriter, _ *http.Request) {
|
||||
if upgrade.CompareVersions(rel.Tag, build.Version) > upgrade.Equal {
|
||||
err = upgrade.To(rel)
|
||||
if err != nil {
|
||||
l.Warnln("upgrading:", err)
|
||||
slog.Error("Failed to upgrade", slogutil.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user