chore(db): use one SQLite database per folder (#10042)

This changes the database structure to use one database per folder, with
a small main database to coordinate. Reverts the prior change to buffer
all files in memory when pulling, meaning there is now a phase where the
WAL file will grow significantly, at least for initial sync of folders
with many directories.

---------

Co-authored-by: bt90 <btom1990@googlemail.com>

internal/db/sqlite/db_service.go

@@ -11,7 +11,9 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/jmoiron/sqlx"
 	"github.com/syncthing/syncthing/internal/db"
+	"github.com/thejerf/suture/v4"
 )
 
 const (
@@ -21,6 +23,10 @@ const (
 	minDeleteRetention     = 24 * time.Hour
 )
 
+func (s *DB) Service(maintenanceInterval time.Duration) suture.Service {
+	return newService(s, maintenanceInterval)
+}
+
 type Service struct {
 	sdb                 *DB
 	maintenanceInterval time.Duration
@@ -80,14 +86,25 @@ func (s *Service) periodic(ctx context.Context) error {
 	t1 := time.Now()
 	defer func() { l.Debugln("Periodic done in", time.Since(t1), "+", t1.Sub(t0)) }()
 
-	if err := s.garbageCollectOldDeletedLocked(); err != nil {
-		return wrap(err)
-	}
-	if err := s.garbageCollectBlocklistsAndBlocksLocked(ctx); err != nil {
-		return wrap(err)
-	}
+	tidy(ctx, s.sdb.sql)
 
-	conn, err := s.sdb.sql.Conn(ctx)
+	return wrap(s.sdb.forEachFolder(func(fdb *folderDB) error {
+		fdb.updateLock.Lock()
+		defer fdb.updateLock.Unlock()
+
+		if err := garbageCollectOldDeletedLocked(fdb); err != nil {
+			return wrap(err)
+		}
+		if err := garbageCollectBlocklistsAndBlocksLocked(ctx, fdb); err != nil {
+			return wrap(err)
+		}
+		tidy(ctx, fdb.sql)
+		return nil
+	}))
+}
+
+func tidy(ctx context.Context, db *sqlx.DB) error {
+	conn, err := db.Conn(ctx)
 	if err != nil {
 		return wrap(err)
 	}
@@ -95,35 +112,34 @@ func (s *Service) periodic(ctx context.Context) error {
 	_, _ = conn.ExecContext(ctx, `ANALYZE`)
 	_, _ = conn.ExecContext(ctx, `PRAGMA optimize`)
 	_, _ = conn.ExecContext(ctx, `PRAGMA incremental_vacuum`)
-	_, _ = conn.ExecContext(ctx, `PRAGMA journal_size_limit = 67108864`)
+	_, _ = conn.ExecContext(ctx, `PRAGMA journal_size_limit = 8388608`)
 	_, _ = conn.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
-
 	return nil
 }
 
-func (s *Service) garbageCollectOldDeletedLocked() error {
-	if s.sdb.deleteRetention <= 0 {
-		l.Debugln("Delete retention is infinite, skipping cleanup")
+func garbageCollectOldDeletedLocked(fdb *folderDB) error {
+	if fdb.deleteRetention <= 0 {
+		l.Debugln(fdb.baseName, "delete retention is infinite, skipping cleanup")
 		return nil
 	}
 
 	// Remove deleted files that are marked as not needed (we have processed
 	// them) and they were deleted more than MaxDeletedFileAge ago.
-	l.Debugln("Forgetting deleted files older than", s.sdb.deleteRetention)
-	res, err := s.sdb.stmt(`
+	l.Debugln(fdb.baseName, "forgetting deleted files older than", fdb.deleteRetention)
+	res, err := fdb.stmt(`
 		DELETE FROM files
 		WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0
-	`).Exec(time.Now().Add(-s.sdb.deleteRetention).UnixNano())
+	`).Exec(time.Now().Add(-fdb.deleteRetention).UnixNano())
 	if err != nil {
 		return wrap(err)
 	}
 	if aff, err := res.RowsAffected(); err == nil {
-		l.Debugln("Removed old deleted file records:", aff)
+		l.Debugln(fdb.baseName, "removed old deleted file records:", aff)
 	}
 	return nil
 }
 
-func (s *Service) garbageCollectBlocklistsAndBlocksLocked(ctx context.Context) error {
+func garbageCollectBlocklistsAndBlocksLocked(ctx context.Context, fdb *folderDB) error {
 	// Remove all blocklists not referred to by any files and, by extension,
 	// any blocks not referred to by a blocklist. This is an expensive
 	// operation when run normally, especially if there are a lot of blocks
@@ -134,7 +150,7 @@ func (s *Service) garbageCollectBlocklistsAndBlocksLocked(ctx context.Context) e
 	// an explicit connection and disabling foreign keys before starting the
 	// transaction. We make sure to clean up on the way out.
 
-	conn, err := s.sdb.sql.Connx(ctx)
+	conn, err := fdb.sql.Connx(ctx)
 	if err != nil {
 		return wrap(err)
 	}
@@ -161,7 +177,7 @@ func (s *Service) garbageCollectBlocklistsAndBlocksLocked(ctx context.Context) e
 		return wrap(err, "delete blocklists")
 	} else if shouldDebug() {
 		rows, err := res.RowsAffected()
-		l.Debugln("Blocklist GC:", rows, err)
+		l.Debugln(fdb.baseName, "blocklist GC:", rows, err)
 	}
 
 	if res, err := tx.ExecContext(ctx, `
@@ -172,7 +188,7 @@ func (s *Service) garbageCollectBlocklistsAndBlocksLocked(ctx context.Context) e
 		return wrap(err, "delete blocks")
 	} else if shouldDebug() {
 		rows, err := res.RowsAffected()
-		l.Debugln("Blocks GC:", rows, err)
+		l.Debugln(fdb.baseName, "blocks GC:", rows, err)
 	}
 
 	return wrap(tx.Commit())