feat: make block indexing configurable (#10608)

This adds a new folder-level configuration `FullBlockIndex`. It controls
whether we maintain the block index for a given folder -- currently
that's always true, now it becomes possible to turn off. The block index
is used for lookup of blocks across files and folders. Effectively, when
syncing a change, for each block, we check:

1. Is the block already present in the old version of the file? If so,
we can reuse (copy) it without network transfer. **This check is always
possible.**
2. Is the block already present in any other file in this folder or
other folders? If so we can copy it. **This check is only possible with
the full block index.**
3. We must transfer the block over the network.

Maintaining the full block index is costly in time, I/O and database
size. With this PR, maintaining the full block index becomes the default
for send-receive and receive-only folders only, with it disabled for
send-only and receive-encrypted folders. The block index is never useful
for encrypted folders, as blocks are encrypted separate for each file.
It is also not useful for send-only folders by themselves, though the
data in the send-only folder could be reused by other receive-type
folders if it were enabled.

For very large folders it may make sense to disable the full block index
regardless of folder type and just accept the resulting decrease in data
reuse.

Disabling or enabling the option in the GUI causes the index to be
destroyed or rebuilt accordingly.

https://github.com/syncthing/docs/pull/1005

---------

Signed-off-by: Jakob Borg <jakob@kastelo.net>

lib/config/config_test.go

@@ -126,6 +126,7 @@ func TestDefaultValues(t *testing.T) {
 					MaxSingleEntrySize: 1024,
 					MaxTotalSize:       4096,
 				},
+				BlockIndexing: true,
 			},
 			Device: DeviceConfiguration{
 				Addresses:       []string{"dynamic"},
@@ -204,6 +205,7 @@ func TestDeviceConfig(t *testing.T) {
 					MaxTotalSize:       4096,
 					Entries:            []XattrFilterEntry{},
 				},
+				BlockIndexing: true,
 			},
 		}
 

lib/config/folderconfiguration.go

@@ -87,6 +87,7 @@ type FolderConfiguration struct {
 	SendOwnership           bool                        `json:"sendOwnership" xml:"sendOwnership"`
 	SyncXattrs              bool                        `json:"syncXattrs" xml:"syncXattrs"`
 	SendXattrs              bool                        `json:"sendXattrs" xml:"sendXattrs"`
+	BlockIndexing           bool                        `json:"blockIndexing" xml:"blockIndexing" default:"true"`
 	XattrFilter             XattrFilter                 `json:"xattrFilter" xml:"xattrFilter"`
 	// Legacy deprecated
 	DeprecatedReadOnly       bool    `json:"-" xml:"ro,attr,omitempty"`        // Deprecated: Do not use.

lib/model/folder.go

@@ -153,12 +153,19 @@ func (f *folder) Serve(ctx context.Context) error {
 	f.sl.DebugContext(ctx, "Folder starting")
 	defer f.sl.DebugContext(ctx, "Folder exiting")
 
+	f.setState(FolderStarting)
+
 	defer func() {
 		f.scanTimer.Stop()
 		f.versionCleanupTimer.Stop()
 		f.setState(FolderIdle)
 	}()
 
+	if err := f.reconcileBlockIndex(ctx); err != nil {
+		f.setError(ctx, err)
+		return err // will get restarted by suture
+	}
+
 	if f.FSWatcherEnabled && f.getHealthErrorAndLoadIgnores() == nil {
 		f.startWatch(ctx)
 	}
@@ -175,6 +182,8 @@ func (f *folder) Serve(ctx context.Context) error {
 	pullTimer := time.NewTimer(0)
 	pullTimer.Stop()
 
+	f.setState(FolderIdle)
+
 	for {
 		var err error
 
@@ -256,6 +265,15 @@ func (f *folder) Serve(ctx context.Context) error {
 	}
 }
 
+func (f *folder) reconcileBlockIndex(ctx context.Context) error {
+	if !f.BlockIndexing {
+		f.sl.DebugContext(ctx, "Dropping block index (block indexing disabled)")
+		return f.db.DropBlockIndex(f.folderID)
+	}
+	f.sl.DebugContext(ctx, "Populating block index if empty")
+	return f.db.PopulateBlockIndex(f.folderID)
+}
+
 func (*folder) BringToFront(string) {}
 
 func (*folder) Override() {}
@@ -1273,7 +1291,11 @@ func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) error {
 }
 
 func (f *folder) updateLocals(fs []protocol.FileInfo) error {
-	if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs); err != nil {
+	var opts []db.UpdateOption
+	if !f.BlockIndexing {
+		opts = append(opts, db.WithSkipBlockIndex())
+	}
+	if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs, opts...); err != nil {
 		return err
 	}
 

lib/model/folder_sendrecv.go

@@ -1166,6 +1166,7 @@ func (f *sendReceiveFolder) handleFile(ctx context.Context, file protocol.FileIn
 		blocks:            blocks,
 		have:              len(have),
 	}
+
 	copyChan <- cs
 	return nil
 }
@@ -1322,7 +1323,7 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo, dbUpdateChan ch
 func (f *sendReceiveFolder) copierRoutine(ctx context.Context, in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) {
 	otherFolderFilesystems := make(map[string]fs.Filesystem)
 	for folder, cfg := range f.model.cfg.Folders() {
-		if folder == f.ID {
+		if folder == f.ID || !cfg.BlockIndexing {
 			continue
 		}
 		otherFolderFilesystems[folder] = cfg.Filesystem()
@@ -1390,13 +1391,26 @@ func (f *sendReceiveFolder) copyBlock(ctx context.Context, block protocol.BlockI
 	buf := protocol.BufferPool.Get(block.Size)
 	defer protocol.BufferPool.Put(buf)
 
-	// Hope that it's usually in the same folder, so start with that
-	// one. Also possibly more efficient copy (same filesystem).
-	if f.copyBlockFromFolder(ctx, f.ID, block, state, f.mtimefs, buf) {
-		return true
+	// Check for the block in the current version of the file
+	if idx, ok := state.curFileBlocks[string(block.Hash)]; ok {
+		if f.copyBlockFromFile(ctx, state.curFile.Name, state.curFile.Blocks[idx].Offset, state, f.mtimefs, block, buf) {
+			state.copiedFromOrigin(block.Size)
+			return true
+		}
+		if state.failed() != nil {
+			return false
+		}
 	}
-	if state.failed() != nil {
-		return false
+
+	if f.folder.BlockIndexing {
+		// Hope that it's usually in the same folder, so start with that
+		// one. Also possibly more efficient copy (same filesystem).
+		if f.copyBlockFromFolder(ctx, f.ID, block, state, f.mtimefs, buf) {
+			return true
+		}
+		if state.failed() != nil {
+			return false
+		}
 	}
 
 	for folderID, ffs := range otherFolderFilesystems {

lib/model/folderstate.go

@@ -27,12 +27,15 @@ const (
 	FolderCleaning
 	FolderCleanWaiting
 	FolderError
+	FolderStarting
 )
 
 func (s folderState) String() string {
 	switch s {
 	case FolderIdle:
 		return "idle"
+	case FolderStarting:
+		return "starting"
 	case FolderScanning:
 		return "scanning"
 	case FolderScanWaiting:

lib/model/model_test.go

@@ -1667,8 +1667,6 @@ func waitForState(t *testing.T, sub events.Subscription, folder, expected string
 				}
 				if err == expected {
 					return
-				} else {
-					t.Error(ev)
 				}
 			}
 		case <-timeout:

lib/model/sharedpullerstate.go

@@ -25,18 +25,19 @@ import (
 // updated along the way.
 type sharedPullerState struct {
 	// Immutable, does not require locking
-	file        protocol.FileInfo // The new file (desired end state)
-	fs          fs.Filesystem
-	folder      string
-	tempName    string
-	realName    string
-	reused      int // Number of blocks reused from temporary file
-	ignorePerms bool
-	hasCurFile  bool              // Whether curFile is set
-	curFile     protocol.FileInfo // The file as it exists now in our database
-	sparse      bool
-	created     time.Time
-	fsync       bool
+	file          protocol.FileInfo // The new file (desired end state)
+	fs            fs.Filesystem
+	folder        string
+	tempName      string
+	realName      string
+	reused        int // Number of blocks reused from temporary file
+	ignorePerms   bool
+	hasCurFile    bool              // Whether curFile is set
+	curFile       protocol.FileInfo // The file as it exists now in our database
+	curFileBlocks map[string]int    // block hash to index in curFile
+	sparse        bool
+	created       time.Time
+	fsync         bool
 
 	// Mutable, must be locked for access
 	err              error           // The first error we hit
@@ -54,6 +55,12 @@ type sharedPullerState struct {
 }
 
 func newSharedPullerState(file protocol.FileInfo, fs fs.Filesystem, folderID, tempName string, blocks []protocol.BlockInfo, reused []int, ignorePerms, hasCurFile bool, curFile protocol.FileInfo, sparse bool, fsync bool) *sharedPullerState {
+	// Map the existing blocks by hash to block index in the current file
+	blocksMap := make(map[string]int, len(curFile.Blocks))
+	for idx, block := range curFile.Blocks {
+		blocksMap[string(block.Hash)] = idx
+	}
+
 	return &sharedPullerState{
 		file:             file,
 		fs:               fs,
@@ -69,6 +76,7 @@ func newSharedPullerState(file protocol.FileInfo, fs fs.Filesystem, folderID, te
 		ignorePerms:      ignorePerms,
 		hasCurFile:       hasCurFile,
 		curFile:          curFile,
+		curFileBlocks:    blocksMap,
 		sparse:           sparse,
 		fsync:            fsync,
 		created:          time.Now(),

lib/model/testutils_test.go

@@ -110,6 +110,7 @@ func newFolderConfig() config.FolderConfiguration {
 	cfg.FSWatcherEnabled = false
 	cfg.PullerDelayS = 0
 	cfg.Devices = append(cfg.Devices, config.FolderDeviceConfiguration{DeviceID: device1})
+	cfg.BlockIndexing = true
 	return cfg
 }
 

lib/ur/contract/contract.go

@@ -127,6 +127,7 @@ type Report struct {
 		SyncXattrs              int            `json:"syncXattrs,omitempty" metric:"folder_feature{feature=SyncXattrs},summary" since:"3"`
 		SendOwnership           int            `json:"sendOwnership,omitempty" metric:"folder_feature{feature=SendOwnership},summary" since:"3"`
 		SyncOwnership           int            `json:"syncOwnership,omitempty" metric:"folder_feature{feature=SyncOwnership},summary" since:"3"`
+		NoBlockIndexing         int            `json:"noBlockIndexing,omitempty" metric:"folder_feature{feature=NoBlockIndexing},summary" since:"3"`
 	} `json:"folderUsesV3,omitzero" since:"3"`
 
 	DeviceUsesV3 struct {

lib/ur/usage_report.go

@@ -280,6 +280,9 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
 			if cfg.SyncOwnership {
 				report.FolderUsesV3.SyncOwnership++
 			}
+			if !cfg.BlockIndexing {
+				report.FolderUsesV3.NoBlockIndexing++
+			}
 		}
 		slices.Sort(report.FolderUsesV3.FsWatcherDelays)