feat: make block indexing configurable (#10608)

This adds a new folder-level configuration `FullBlockIndex`. It controls
whether we maintain the block index for a given folder -- currently
that's always true, now it becomes possible to turn off. The block index
is used for lookup of blocks across files and folders. Effectively, when
syncing a change, for each block, we check:

1. Is the block already present in the old version of the file? If so,
we can reuse (copy) it without network transfer. **This check is always
possible.**
2. Is the block already present in any other file in this folder or
other folders? If so we can copy it. **This check is only possible with
the full block index.**
3. We must transfer the block over the network.

Maintaining the full block index is costly in time, I/O and database
size. With this PR, maintaining the full block index becomes the default
for send-receive and receive-only folders only, with it disabled for
send-only and receive-encrypted folders. The block index is never useful
for encrypted folders, as blocks are encrypted separate for each file.
It is also not useful for send-only folders by themselves, though the
data in the send-only folder could be reused by other receive-type
folders if it were enabled.

For very large folders it may make sense to disable the full block index
regardless of folder type and just accept the resulting decrease in data
reuse.

Disabling or enabling the option in the GUI causes the index to be
destroyed or rebuilt accordingly.

https://github.com/syncthing/docs/pull/1005

---------

Signed-off-by: Jakob Borg <jakob@kastelo.net>

internal/db/sqlite/folderdb_update.go

@@ -14,6 +14,7 @@ import (
 	"slices"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syncthing/syncthing/internal/gen/dbproto"
 	"github.com/syncthing/syncthing/internal/itererr"
 	"github.com/syncthing/syncthing/internal/slogutil"
@@ -30,7 +31,7 @@ const (
 	updatePointsThreshold = 250_000
 )
 
-func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) error {
+func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo, options db.UpdateOptions) error {
 	s.updateLock.Lock()
 	defer s.updateLock.Unlock()
 
@@ -151,7 +152,7 @@ func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) erro
 			}
 			if _, err := insertBlockListStmt.Exec(f.BlocksHash, bs); err != nil {
 				return wrap(err, "insert blocklist")
-			} else if device == protocol.LocalDeviceID {
+			} else if device == protocol.LocalDeviceID && !options.SkipBlockIndex {
 				// Insert all blocks
 				if err := s.insertBlocksLocked(txp, f.BlocksHash, f.Blocks); err != nil {
 					return wrap(err, "insert blocks")
@@ -303,6 +304,86 @@ func (s *folderDB) DropFilesNamed(device protocol.DeviceID, names []string) erro
 	return wrap(tx.Commit())
 }
 
+func (s *folderDB) blockIndexEmpty() (bool, error) {
+	var exists bool
+	err := s.sql.Get(&exists, `SELECT EXISTS (SELECT 1 FROM blocks LIMIT 1)`)
+	if err != nil {
+		return false, wrap(err)
+	}
+	return !exists, nil
+}
+
+func (s *folderDB) DropBlockIndex() error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	empty, err := s.blockIndexEmpty()
+	if err != nil || empty {
+		return err
+	}
+
+	if _, err := s.sql.Exec(`DELETE FROM blocks`); err != nil {
+		return wrap(err)
+	}
+
+	return s.vacuumAndOptimize()
+}
+
+func (s *folderDB) PopulateBlockIndex() error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	empty, err := s.blockIndexEmpty()
+	if err != nil || !empty {
+		return err
+	}
+
+	tx, err := s.sql.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback()
+	txp := &txPreparedStmts{Tx: tx}
+
+	// Iterate all local files that have a blocklist
+	rows, err := tx.Queryx(`
+		SELECT f.blocklist_hash, bl.blprotobuf FROM files f
+		INNER JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		WHERE f.device_idx = ? AND f.blocklist_hash IS NOT NULL
+	`, s.localDeviceIdx)
+	if err != nil {
+		return wrap(err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var blocklistHash []byte
+		var blProtobuf []byte
+		if err := rows.Scan(&blocklistHash, &blProtobuf); err != nil {
+			return wrap(err)
+		}
+
+		var bl dbproto.BlockList
+		if err := proto.Unmarshal(blProtobuf, &bl); err != nil {
+			return wrap(err, "unmarshal blocklist")
+		}
+
+		blocks := make([]protocol.BlockInfo, len(bl.Blocks))
+		for i, b := range bl.Blocks {
+			blocks[i] = protocol.BlockInfoFromWire(b)
+		}
+
+		if err := s.insertBlocksLocked(txp, blocklistHash, blocks); err != nil {
+			return wrap(err, "insert blocks")
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return wrap(err)
+	}
+
+	return wrap(tx.Commit())
+}
+
 func (*folderDB) insertBlocksLocked(tx *txPreparedStmts, blocklistHash []byte, blocks []protocol.BlockInfo) error {
 	if len(blocks) == 0 {
 		return nil