chore(db): migration for previous commits (#10319)

Recreate the blocks and block lists tables.

---------

Co-authored-by: bt90 <btom1990@googlemail.com>

internal/db/sqlite/basedb.go

@@ -11,6 +11,7 @@ import (
 	"database/sql"
 	"embed"
 	"io/fs"
+	"log/slog"
 	"net/url"
 	"path/filepath"
 	"slices"
@@ -21,11 +22,12 @@ import (
 	"time"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/slogutil"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const currentSchemaVersion = 3
+const currentSchemaVersion = 4
 
 //go:embed sql/**
 var embedded embed.FS
@@ -91,6 +93,7 @@ func openBase(path string, maxConns int, pragmas, schemaScripts, migrationScript
 	}
 
 	ver, _ := db.getAppliedSchemaVersion()
+	shouldVacuum := false
 	if ver.SchemaVersion > 0 {
 		type migration struct {
 			script  string
@@ -117,6 +120,7 @@ func openBase(path string, maxConns int, pragmas, schemaScripts, migrationScript
 			if err := db.applyMigration(m.version, m.script); err != nil {
 				return nil, wrap(err)
 			}
+			shouldVacuum = true
 		}
 	}
 
@@ -125,6 +129,14 @@ func openBase(path string, maxConns int, pragmas, schemaScripts, migrationScript
 		return nil, wrap(err)
 	}
 
+	if shouldVacuum {
+		// We applied migrations and should take the opportunity to vaccuum
+		// the database.
+		if err := db.vacuumAndOptimize(); err != nil {
+			return nil, wrap(err)
+		}
+	}
+
 	return db, nil
 }
 
@@ -200,6 +212,20 @@ func (s *baseDB) expandTemplateVars(tpl string) string {
 	return sb.String()
 }
 
+func (s *baseDB) vacuumAndOptimize() error {
+	stmts := []string{
+		"VACUUM;",
+		"PRAGMA optimize;",
+		"PRAGMA wal_checkpoint(truncate);",
+	}
+	for _, stmt := range stmts {
+		if _, err := s.sql.Exec(stmt); err != nil {
+			return wrap(err, stmt)
+		}
+	}
+	return nil
+}
+
 type stmt interface {
 	Exec(args ...any) (sql.Result, error)
 	Get(dest any, args ...any) error
@@ -252,6 +278,8 @@ func (s *baseDB) applyMigration(ver int, script string) error {
 	}
 	defer tx.Rollback() //nolint:errcheck
 
+	slog.Info("Applying database migration", slogutil.FilePath(s.baseName), "toSchema", ver, "script", script)
+
 	if err := s.execScript(tx, script); err != nil {
 		return wrap(err)
 	}

internal/db/sqlite/sql/migrations/folder/04-alter-blocks-tables.sql

@@ -0,0 +1,51 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Copy blocks to new table with fewer indexes
+
+DROP TABLE IF EXISTS blocks_v4
+;
+
+CREATE TABLE blocks_v4 (
+    hash BLOB NOT NULL,
+    blocklist_hash BLOB NOT NULL,
+    idx INTEGER NOT NULL,
+    offset INTEGER NOT NULL,
+    size INTEGER NOT NULL,
+    PRIMARY KEY (hash, blocklist_hash, idx)
+) STRICT, WITHOUT ROWID
+;
+
+INSERT INTO blocks_v4 (hash, blocklist_hash, idx, offset, size)
+SELECT hash, blocklist_hash, idx, offset, size FROM blocks ORDER BY hash, blocklist_hash, idx
+;
+
+DROP TABLE blocks
+;
+
+ALTER TABLE blocks_v4 RENAME TO blocks
+;
+
+-- Copy blocklists to new table with fewer indexes
+
+DROP TABLE IF EXISTS blocklists_v4
+;
+
+CREATE TABLE blocklists_v4 (
+    blocklist_hash BLOB NOT NULL PRIMARY KEY,
+    blprotobuf BLOB NOT NULL
+) STRICT, WITHOUT ROWID
+;
+
+INSERT INTO blocklists_v4 (blocklist_hash, blprotobuf)
+SELECT blocklist_hash, blprotobuf FROM blocklists ORDER BY blocklist_hash
+;
+
+DROP TABLE blocklists
+;
+
+ALTER TABLE blocklists_v4 RENAME TO blocklists
+;

lib/syncthing/syncthing.go

@@ -191,6 +191,10 @@ func (a *App) startup() error {
 		if _, ok := cfgFolders[folder]; !ok {
 			slog.Info("Cleaning metadata for dropped folder", "folder", folder)
 			a.sdb.DropFolder(folder)
+		} else {
+			// Open the folder database, causing it to apply migrations
+			// early when appropriate.
+			_, _ = a.sdb.GetDeviceSequence(folder, protocol.LocalDeviceID)
 		}
 	}