Merge branch 'main' into v2

* main:
  fix(config): properly apply defaults when reading folder configuration (#10034)
  chore(model): add metric for total number of conflicts (#10037)
  build: replace underscore in Debian version (#10032)

build.go

@@ -628,6 +628,9 @@ func buildDeb(target target) {
 		// than just 0.14.26. This rectifies that.
 		debver = strings.Replace(debver, "-", "~", -1)
 	}
+	if strings.Contains(debver, "_") {
+		debver = strings.Replace(debver, "_", "~", -1)
+	}
 	args := []string{
 		"-t", "deb",
 		"-s", "dir",

internal/db/sqlite/db_update.go

@@ -577,7 +577,6 @@ func (s *DB) periodicCheckpointLocked(fs []protocol.FileInfo) {
 		s.updatePoints += len(f.Blocks) * updatePointsPerBlock
 	}
 	if s.updatePoints > updatePointsThreshold {
-		l.Debugln("checkpoint at", s.updatePoints)
 		conn, err := s.sql.Conn(context.Background())
 		if err != nil {
 			l.Debugln("conn:", err)
@@ -585,10 +584,16 @@ func (s *DB) periodicCheckpointLocked(fs []protocol.FileInfo) {
 		}
 		defer conn.Close()
 		if _, err := conn.ExecContext(context.Background(), `PRAGMA journal_size_limit = 67108864`); err != nil {
-			l.Debugln("PRAGMA journal_size_limit(RESTART):", err)
+			l.Debugln("PRAGMA journal_size_limit:", err)
 		}
-		if _, err := conn.ExecContext(context.Background(), `PRAGMA wal_checkpoint(RESTART)`); err != nil {
+		row := conn.QueryRowContext(context.Background(), `PRAGMA wal_checkpoint(RESTART)`)
+		var res, modified, moved int
+		if row.Err() != nil {
 			l.Debugln("PRAGMA wal_checkpoint(RESTART):", err)
+		} else if err := row.Scan(&res, &modified, &moved); err != nil {
+			l.Debugln("PRAGMA wal_checkpoint(RESTART) (scan):", err)
+		} else {
+			l.Debugln("checkpoint at", s.updatePoints, "returned", res, modified, moved)
 		}
 		s.updatePoints = 0
 	}

lib/config/config_test.go

@@ -101,7 +101,7 @@ func TestDefaultValues(t *testing.T) {
 		Defaults: Defaults{
 			Folder: FolderConfiguration{
 				FilesystemType:   FilesystemTypeBasic,
-				Path:             "~",
+				Path:             "",
 				Type:             FolderTypeSendReceive,
 				Devices:          []FolderDeviceConfiguration{{DeviceID: device1}},
 				RescanIntervalS:  3600,
@@ -180,7 +180,7 @@ func TestDeviceConfig(t *testing.T) {
 				Devices:          []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
 				Type:             FolderTypeSendOnly,
 				RescanIntervalS:  600,
-				FSWatcherEnabled: false,
+				FSWatcherEnabled: true,
 				FSWatcherDelayS:  10,
 				Copiers:          0,
 				Hashers:          0,
@@ -188,13 +188,17 @@ func TestDeviceConfig(t *testing.T) {
 				MinDiskFree:      Size{1, "%"},
 				MaxConflicts:     -1,
 				Versioning: VersioningConfiguration{
-					Params: map[string]string{},
+					CleanupIntervalS: 3600,
+					FSType:           FilesystemTypeBasic,
+					Params:           map[string]string{},
 				},
 				MarkerName:          DefaultMarkerName,
 				JunctionsAsDirs:     true,
 				MaxConcurrentWrites: maxConcurrentWritesDefault,
 				XattrFilter: XattrFilter{
-					Entries: []XattrFilterEntry{},
+					MaxSingleEntrySize: 1024,
+					MaxTotalSize:       4096,
+					Entries:            []XattrFilterEntry{},
 				},
 			},
 		}

lib/config/folderconfiguration.go

@@ -9,6 +9,8 @@ package config
 import (
 	"bytes"
 	"crypto/sha256"
+	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"path"
@@ -22,6 +24,7 @@ import (
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/structutil"
 )
 
 var (
@@ -47,7 +50,7 @@ type FolderConfiguration struct {
 	ID                      string                      `json:"id" xml:"id,attr" nodefault:"true"`
 	Label                   string                      `json:"label" xml:"label,attr" restart:"false"`
 	FilesystemType          FilesystemType              `json:"filesystemType" xml:"filesystemType" default:"basic"`
-	Path                    string                      `json:"path" xml:"path,attr" default:"~"`
+	Path                    string                      `json:"path" xml:"path,attr"`
 	Type                    FolderType                  `json:"type" xml:"type,attr"`
 	Devices                 []FolderDeviceConfiguration `json:"devices" xml:"device"`
 	RescanIntervalS         int                         `json:"rescanIntervalS" xml:"rescanIntervalS,attr" default:"3600"`
@@ -391,3 +394,23 @@ func (f XattrFilter) GetMaxSingleEntrySize() int {
 func (f XattrFilter) GetMaxTotalSize() int {
 	return f.MaxTotalSize
 }
+
+func (f *FolderConfiguration) UnmarshalJSON(data []byte) error {
+	structutil.SetDefaults(f)
+
+	// avoid recursing into this method
+	type noCustomUnmarshal FolderConfiguration
+	ptr := (*noCustomUnmarshal)(f)
+
+	return json.Unmarshal(data, ptr)
+}
+
+func (f *FolderConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	structutil.SetDefaults(f)
+
+	// avoid recursing into this method
+	type noCustomUnmarshal FolderConfiguration
+	ptr := (*noCustomUnmarshal)(f)
+
+	return d.DecodeElement(ptr, &start)
+}

lib/model/folder_sendrecv.go

@@ -315,15 +315,23 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 	fileDeletions := map[string]protocol.FileInfo{}
 	buckets := map[string][]protocol.FileInfo{}
 
+	// Buffer the full list of needed files. This is somewhat wasteful and
+	// uses a lot of memory, but we need to keep the duration of the
+	// database read short and not do a bunch of file and data I/O inside
+	// the loop. If we forego the ability for users to repriorize the pull
+	// queue on the fly we could do this in batches, though that would also
+	// be a bit slower and less efficient in other ways.
+	files, err := itererr.Collect(f.model.sdb.AllNeededGlobalFiles(f.folderID, protocol.LocalDeviceID, f.Order, 0, 0))
+	if err != nil {
+		return changed, nil, nil, err
+	}
+
 	// Iterate the list of items that we need and sort them into piles.
 	// Regular files to pull goes into the file queue, everything else
 	// (directories, symlinks and deletes) goes into the "process directly"
 	// pile.
 loop:
-	for file, err := range itererr.Zip(f.model.sdb.AllNeededGlobalFiles(f.folderID, protocol.LocalDeviceID, f.Order, 0, 0)) {
-		if err != nil {
-			return changed, nil, nil, err
-		}
+	for _, file := range files {
 		select {
 		case <-f.ctx.Done():
 			break loop
@@ -1801,6 +1809,7 @@ func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan cha
 		return nil
 	}
 
+	metricFolderConflictsTotal.WithLabelValues(f.ID).Inc()
 	newName := conflictName(name, lastModBy)
 	err := f.mtimefs.Rename(name, newName)
 	if fs.IsNotExist(err) {

lib/model/indexhandler.go

@@ -295,7 +295,7 @@ func (s *indexHandler) sendIndexTo(ctx context.Context) error {
 	var f protocol.FileInfo
 	previousWasDelete := false
 
-	for fi, err := range itererr.Zip(s.sdb.AllLocalFilesBySequence(s.folder, protocol.LocalDeviceID, s.localPrevSequence+1, 5000)) {
+	for fi, err := range itererr.Zip(s.sdb.AllLocalFilesBySequence(s.folder, protocol.LocalDeviceID, s.localPrevSequence+1, MaxBatchSizeFiles+1)) {
 		if err != nil {
 			return err
 		}

lib/model/metrics.go

@@ -57,6 +57,13 @@ var (
 		Name:      "folder_processed_bytes_total",
 		Help:      "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/skipped)",
 	}, []string{"folder", "source"})
+
+	metricFolderConflictsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "model",
+		Name:      "folder_conflicts_total",
+		Help:      "Total number of conflicts",
+	}, []string{"folder"})
 )
 
 const (
@@ -88,4 +95,5 @@ func registerFolderMetrics(folderID string) {
 	metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOrigin)
 	metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOther)
 	metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceSkipped)
+	metricFolderConflictsTotal.WithLabelValues(folderID)
 }