chore: switch database engine to sqlite (fixes #9954) (#9965)

Switch the database from LevelDB to SQLite, for greater stability and simpler code. Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl> Co-authored-by: bt90 <btom1990@googlemail.com>
2025-03-29 13:50:08 +01:00
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
@@ -0,0 +1,8 @@
+These SQL scripts are embedded in the binary.
+
+Scripts in `schema/` are run at every startup, in alphanumerical order.
+
+Scripts in `migrations/` are run when a migration is needed; the must begin
+with a number that equals the schema version that results from that
+migration. Migrations are not run on initial database creation, so the
+scripts in `schema/` should create the latest version.
@@ -0,0 +1,7 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- The next migration should be number two.
@@ -0,0 +1,19 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- folders map folder IDs as used by Syncthing to database folder indexes
+CREATE TABLE IF NOT EXISTS folders (
+    idx INTEGER NOT NULL PRIMARY KEY,
+    folder_id TEXT NOT NULL UNIQUE COLLATE BINARY
+) STRICT
+;
+
+-- devices map device IDs as used by Syncthing to database device indexes
+CREATE TABLE IF NOT EXISTS devices (
+    idx INTEGER NOT NULL PRIMARY KEY,
+    device_id TEXT NOT NULL UNIQUE COLLATE BINARY
+) STRICT
+;
@@ -0,0 +1,14 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Schema migrations hold the list of historical migrations applied
+CREATE TABLE IF NOT EXISTS schemamigrations (
+    schema_version INTEGER NOT NULL,
+    applied_at INTEGER NOT NULL, -- unix nanos
+    syncthing_version TEXT NOT NULL COLLATE BINARY,
+    PRIMARY KEY(schema_version)
+) STRICT
+;
@@ -0,0 +1,62 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Files
+--
+-- The files table contains all files announced by any device. Files present
+-- on this device are filed under the LocalDeviceID, not the actual current
+-- device ID, for simplicity, consistency and portability. One announced
+-- version of each file is considered the "global" version - the latest one,
+-- that all other devices strive to replicate. This instance gets the Global
+-- flag bit set. There may be other identical instances of this file
+-- announced by other devices, but only one onstance gets the Global flag;
+-- this simplifies accounting. If the current device has the Global version,
+-- the LocalDeviceID instance of the file is the one that has the Global
+-- bit.
+--
+-- If the current device does not have that version of the file it gets the
+-- Need bit set. Only Global files announced by another device can have the
+-- Need bit. This allows for very efficient lookup of files needing handling
+-- on this device, which is a common query.
+CREATE TABLE IF NOT EXISTS files (
+    folder_idx INTEGER NOT NULL,
+    device_idx INTEGER NOT NULL, -- actual device ID or LocalDeviceID
+    sequence INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, -- our local database sequence, for each and every entry
+    remote_sequence INTEGER, -- remote device's sequence number, null for local or synthetic entries
+    name TEXT NOT NULL COLLATE BINARY,
+    type INTEGER NOT NULL, -- protocol.FileInfoType
+    modified INTEGER NOT NULL, -- Unix nanos
+    size INTEGER NOT NULL,
+    version TEXT NOT NULL COLLATE BINARY,
+    deleted INTEGER NOT NULL, -- boolean
+    invalid INTEGER NOT NULL, -- boolean
+    local_flags INTEGER NOT NULL,
+    blocklist_hash BLOB, -- null when there are no blocks
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE,
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+) STRICT
+;
+-- FileInfos store the actual protobuf object. We do this separately to keep
+-- the files rows smaller and more efficient.
+CREATE TABLE IF NOT EXISTS fileinfos (
+    sequence INTEGER NOT NULL PRIMARY KEY, -- our local database sequence from the files table
+    fiprotobuf BLOB NOT NULL,
+    FOREIGN KEY(sequence) REFERENCES files(sequence) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
+) STRICT
+;
+-- There can be only one file per folder, device, and remote sequence number
+CREATE UNIQUE INDEX IF NOT EXISTS files_remote_sequence ON files (folder_idx, device_idx, remote_sequence)
+    WHERE remote_sequence IS NOT NULL
+;
+-- There can be only one file per folder, device, and name
+CREATE UNIQUE INDEX IF NOT EXISTS files_device_name ON files (folder_idx, device_idx, name)
+;
+-- We want to be able to look up & iterate files based on just folder and name
+CREATE INDEX IF NOT EXISTS files_name_only ON files (folder_idx, name)
+;
+-- We want to be able to look up & iterate files based on blocks hash
+CREATE INDEX IF NOT EXISTS files_blocklist_hash_only ON files (blocklist_hash, device_idx, folder_idx) WHERE blocklist_hash IS NOT NULL
+;
@@ -0,0 +1,24 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- indexids holds the index ID and maximum sequence for a given device and folder
+CREATE TABLE IF NOT EXISTS indexids (
+    device_idx INTEGER NOT NULL,
+    folder_idx INTEGER NOT NULL,
+    index_id TEXT NOT NULL COLLATE BINARY,
+    sequence INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY(device_idx, folder_idx),
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE,
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE
+) STRICT, WITHOUT ROWID
+;
+CREATE TRIGGER IF NOT EXISTS indexids_seq AFTER INSERT ON files
+BEGIN
+    INSERT INTO indexids (folder_idx, device_idx, index_id, sequence)
+        VALUES (NEW.folder_idx, NEW.device_idx, "", COALESCE(NEW.remote_sequence, NEW.sequence))
+        ON CONFLICT DO UPDATE SET sequence = COALESCE(NEW.remote_sequence, NEW.sequence);
+END
+;
@@ -0,0 +1,53 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Counts
+--
+-- Counts and sizes are maintained for each device, folder, type, flag bits
+-- combination.
+CREATE TABLE IF NOT EXISTS counts (
+    folder_idx INTEGER NOT NULL,
+    device_idx INTEGER NOT NULL,
+    type INTEGER NOT NULL,
+    local_flags INTEGER NOT NULL,
+    count INTEGER NOT NULL,
+    size INTEGER NOT NULL,
+    deleted INTEGER NOT NULL, -- boolean
+    PRIMARY KEY(folder_idx, device_idx, type, local_flags, deleted),
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE,
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+) STRICT, WITHOUT ROWID
+;
+
+--- Maintain counts when files are added and removed using triggers
+
+CREATE TRIGGER IF NOT EXISTS counts_insert AFTER INSERT ON files
+BEGIN
+    INSERT INTO counts (folder_idx, device_idx, type, local_flags, count, size, deleted)
+        VALUES (NEW.folder_idx, NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
+        ON CONFLICT DO UPDATE SET count = count + 1, size = size + NEW.size;
+END
+;
+CREATE TRIGGER IF NOT EXISTS counts_delete AFTER DELETE ON files
+BEGIN
+    UPDATE counts SET count = count - 1, size = size - OLD.size
+        WHERE folder_idx = OLD.folder_idx AND device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
+END
+;
+CREATE TRIGGER IF NOT EXISTS counts_update AFTER UPDATE OF local_flags ON files
+WHEN NEW.local_flags != OLD.local_flags
+BEGIN
+    INSERT INTO counts (folder_idx, device_idx, type, local_flags, count, size, deleted)
+        VALUES (NEW.folder_idx, NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
+        ON CONFLICT DO UPDATE SET count = count + 1, size = size + NEW.size;
+    UPDATE counts SET count = count - 1, size = size - OLD.size
+        WHERE folder_idx = OLD.folder_idx AND device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
+END
+;
+DROP TRIGGER IF EXISTS counts_update_add -- tmp migration
+;
+DROP TRIGGER IF EXISTS counts_update_del -- tmp migration
+;
@@ -0,0 +1,34 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Block lists
+--
+-- The block lists are extracted from FileInfos and stored separately. This
+-- reduces the database size by reusing the same block list entry for all
+-- devices announcing the same file. Doing it for all block lists instead of
+-- using a size cutoff simplifies queries. Block lists are garbage collected
+-- "manually", not using a trigger as that was too performance impacting.
+CREATE TABLE IF NOT EXISTS blocklists (
+    blocklist_hash BLOB NOT NULL PRIMARY KEY,
+    blprotobuf BLOB NOT NULL
+) STRICT
+;
+
+-- Blocks
+--
+-- For all local files we store the blocks individually for quick lookup. A
+-- given block can exist in multiple blocklists and at multiple offsets in a
+-- blocklist.
+CREATE TABLE IF NOT EXISTS blocks (
+    hash BLOB NOT NULL,
+    blocklist_hash BLOB NOT NULL,
+    idx INTEGER NOT NULL,
+    offset INTEGER NOT NULL,
+    size INTEGER NOT NULL,
+    PRIMARY KEY (hash, blocklist_hash, idx),
+    FOREIGN KEY(blocklist_hash) REFERENCES blocklists(blocklist_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
+) STRICT
+;
@@ -0,0 +1,16 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+--- Backing for the MtimeFS
+CREATE TABLE IF NOT EXISTS mtimes (
+    folder_idx INTEGER NOT NULL,
+    name TEXT NOT NULL,
+    ondisk INTEGER NOT NULL, -- unix nanos
+    virtual INTEGER NOT NULL, -- unix nanos
+    PRIMARY KEY(folder_idx, name),
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+) STRICT, WITHOUT ROWID
+;
@@ -0,0 +1,13 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+--- Simple KV store. This backs the "miscDB" we use for certain minor pieces
+--  of data.
+CREATE TABLE IF NOT EXISTS kv (
+    key TEXT NOT NULL PRIMARY KEY COLLATE BINARY,
+    value BLOB NOT NULL
+) STRICT
+;