chore(db): use one SQLite database per folder (#10042)

This changes the database structure to use one database per folder, with
a small main database to coordinate. Reverts the prior change to buffer
all files in memory when pulling, meaning there is now a phase where the
WAL file will grow significantly, at least for initial sync of folders
with many directories.

---------

Co-authored-by: bt90 <btom1990@googlemail.com>

internal/db/sqlite/sql/schema/folder/00-indexes.sql

@@ -4,16 +4,9 @@
 -- License, v. 2.0. If a copy of the MPL was not distributed with this file,
 -- You can obtain one at https://mozilla.org/MPL/2.0/.
 
--- folders map folder IDs as used by Syncthing to database folder indexes
-CREATE TABLE IF NOT EXISTS folders (
-    idx INTEGER NOT NULL PRIMARY KEY,
-    folder_id TEXT NOT NULL UNIQUE COLLATE BINARY
-) STRICT
-;
-
 -- devices map device IDs as used by Syncthing to database device indexes
 CREATE TABLE IF NOT EXISTS devices (
-    idx INTEGER NOT NULL PRIMARY KEY,
+    idx INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     device_id TEXT NOT NULL UNIQUE COLLATE BINARY
 ) STRICT
 ;

internal/db/sqlite/sql/schema/folder/20-files.sql

@@ -22,7 +22,6 @@
 -- Need bit. This allows for very efficient lookup of files needing handling
 -- on this device, which is a common query.
 CREATE TABLE IF NOT EXISTS files (
-    folder_idx INTEGER NOT NULL,
     device_idx INTEGER NOT NULL, -- actual device ID or LocalDeviceID
     sequence INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, -- our local database sequence, for each and every entry
     remote_sequence INTEGER, -- remote device's sequence number, null for local or synthetic entries
@@ -35,8 +34,7 @@ CREATE TABLE IF NOT EXISTS files (
     invalid INTEGER NOT NULL, -- boolean
     local_flags INTEGER NOT NULL,
     blocklist_hash BLOB, -- null when there are no blocks
-    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE,
-    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE
 ) STRICT
 ;
 -- FileInfos store the actual protobuf object. We do this separately to keep
@@ -48,15 +46,15 @@ CREATE TABLE IF NOT EXISTS fileinfos (
 ) STRICT
 ;
 -- There can be only one file per folder, device, and remote sequence number
-CREATE UNIQUE INDEX IF NOT EXISTS files_remote_sequence ON files (folder_idx, device_idx, remote_sequence)
+CREATE UNIQUE INDEX IF NOT EXISTS files_remote_sequence ON files (device_idx, remote_sequence)
     WHERE remote_sequence IS NOT NULL
 ;
 -- There can be only one file per folder, device, and name
-CREATE UNIQUE INDEX IF NOT EXISTS files_device_name ON files (folder_idx, device_idx, name)
+CREATE UNIQUE INDEX IF NOT EXISTS files_device_name ON files (device_idx, name)
 ;
 -- We want to be able to look up & iterate files based on just folder and name
-CREATE INDEX IF NOT EXISTS files_name_only ON files (folder_idx, name)
+CREATE INDEX IF NOT EXISTS files_name_only ON files (name)
 ;
 -- We want to be able to look up & iterate files based on blocks hash
-CREATE INDEX IF NOT EXISTS files_blocklist_hash_only ON files (blocklist_hash, device_idx, folder_idx) WHERE blocklist_hash IS NOT NULL
+CREATE INDEX IF NOT EXISTS files_blocklist_hash_only ON files (blocklist_hash, device_idx) WHERE blocklist_hash IS NOT NULL
 ;

internal/db/sqlite/sql/schema/folder/30-indexids.sql

@@ -7,18 +7,16 @@
 -- indexids holds the index ID and maximum sequence for a given device and folder
 CREATE TABLE IF NOT EXISTS indexids (
     device_idx INTEGER NOT NULL,
-    folder_idx INTEGER NOT NULL,
     index_id TEXT NOT NULL COLLATE BINARY,
     sequence INTEGER NOT NULL DEFAULT 0,
-    PRIMARY KEY(device_idx, folder_idx),
-    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE,
+    PRIMARY KEY(device_idx),
     FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE
 ) STRICT, WITHOUT ROWID
 ;
 CREATE TRIGGER IF NOT EXISTS indexids_seq AFTER INSERT ON files
 BEGIN
-    INSERT INTO indexids (folder_idx, device_idx, index_id, sequence)
-        VALUES (NEW.folder_idx, NEW.device_idx, "", COALESCE(NEW.remote_sequence, NEW.sequence))
+    INSERT INTO indexids (device_idx, index_id, sequence)
+        VALUES (NEW.device_idx, "", COALESCE(NEW.remote_sequence, NEW.sequence))
         ON CONFLICT DO UPDATE SET sequence = COALESCE(NEW.remote_sequence, NEW.sequence);
 END
 ;

internal/db/sqlite/sql/schema/folder/40-counts.sql

@@ -9,16 +9,14 @@
 -- Counts and sizes are maintained for each device, folder, type, flag bits
 -- combination.
 CREATE TABLE IF NOT EXISTS counts (
-    folder_idx INTEGER NOT NULL,
     device_idx INTEGER NOT NULL,
     type INTEGER NOT NULL,
     local_flags INTEGER NOT NULL,
     count INTEGER NOT NULL,
     size INTEGER NOT NULL,
     deleted INTEGER NOT NULL, -- boolean
-    PRIMARY KEY(folder_idx, device_idx, type, local_flags, deleted),
-    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE,
-    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+    PRIMARY KEY(device_idx, type, local_flags, deleted),
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE
 ) STRICT, WITHOUT ROWID
 ;
 
@@ -26,28 +24,24 @@ CREATE TABLE IF NOT EXISTS counts (
 
 CREATE TRIGGER IF NOT EXISTS counts_insert AFTER INSERT ON files
 BEGIN
-    INSERT INTO counts (folder_idx, device_idx, type, local_flags, count, size, deleted)
-        VALUES (NEW.folder_idx, NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
+    INSERT INTO counts (device_idx, type, local_flags, count, size, deleted)
+        VALUES (NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
         ON CONFLICT DO UPDATE SET count = count + 1, size = size + NEW.size;
 END
 ;
 CREATE TRIGGER IF NOT EXISTS counts_delete AFTER DELETE ON files
 BEGIN
     UPDATE counts SET count = count - 1, size = size - OLD.size
-        WHERE folder_idx = OLD.folder_idx AND device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
+        WHERE device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
 END
 ;
 CREATE TRIGGER IF NOT EXISTS counts_update AFTER UPDATE OF local_flags ON files
 WHEN NEW.local_flags != OLD.local_flags
 BEGIN
-    INSERT INTO counts (folder_idx, device_idx, type, local_flags, count, size, deleted)
-        VALUES (NEW.folder_idx, NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
+    INSERT INTO counts (device_idx, type, local_flags, count, size, deleted)
+        VALUES (NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
         ON CONFLICT DO UPDATE SET count = count + 1, size = size + NEW.size;
     UPDATE counts SET count = count - 1, size = size - OLD.size
-        WHERE folder_idx = OLD.folder_idx AND device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
+        WHERE device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
 END
 ;
-DROP TRIGGER IF EXISTS counts_update_add -- tmp migration
-;
-DROP TRIGGER IF EXISTS counts_update_del -- tmp migration
-;

internal/db/sqlite/sql/schema/folder/50-mtimes.sql

@@ -6,11 +6,9 @@
 
 --- Backing for the MtimeFS
 CREATE TABLE IF NOT EXISTS mtimes (
-    folder_idx INTEGER NOT NULL,
     name TEXT NOT NULL,
     ondisk INTEGER NOT NULL, -- unix nanos
     virtual INTEGER NOT NULL, -- unix nanos
-    PRIMARY KEY(folder_idx, name),
-    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+    PRIMARY KEY(name)
 ) STRICT, WITHOUT ROWID
 ;

internal/db/sqlite/sql/schema/main/00-indexes.sql

@@ -0,0 +1,12 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- folders map folder IDs as used by Syncthing to database folder indexes
+CREATE TABLE IF NOT EXISTS folders (
+    idx INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    folder_id TEXT NOT NULL UNIQUE COLLATE BINARY
+) STRICT
+;