felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: switch database engine to sqlite (fixes #9954) (#9965)

Browse Source 
Switch the database from LevelDB to SQLite, for greater stability and
simpler code.

Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl>
Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
2025-03-29 13:50:08 +01:00
committed by GitHub
co-authored by Tommy van der Vorst bt90
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/syncthing/globalmigrations.go
+4 -103
View File
@@ -7,13 +7,8 @@
package syncthing
import (
"encoding/binary"
"io"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
const (
@@ -21,8 +16,8 @@ const (
globalMigrationDBKey = "globalMigrationVersion"
)
func globalMigration(ll *db.Lowlevel, cfg config.Wrapper) error {
miscDB := db.NewMiscDataNamespace(ll)
func globalMigration(kv db.KV, cfg config.Wrapper) error {
miscDB := db.NewMiscDB(kv)
prevVersion, _, err := miscDB.Int64(globalMigrationDBKey)
if err != nil {
return err
@@ -32,101 +27,7 @@ func globalMigration(ll *db.Lowlevel, cfg config.Wrapper) error {
return nil
}
if prevVersion < 1 {
if err := encryptionTrailerSizeMigration(ll, cfg); err != nil {
return err
}
}
// currently no migrations
return miscDB.PutInt64(globalMigrationDBKey, globalMigrationVersion)
}
func encryptionTrailerSizeMigration(ll *db.Lowlevel, cfg config.Wrapper) error {
encFolders := cfg.Folders()
for folderID, folderCfg := range cfg.Folders() {
if folderCfg.Type != config.FolderTypeReceiveEncrypted {
delete(encFolders, folderID)
}
}
if len(encFolders) == 0 {
return nil
}
l.Infoln("Running global migration to fix encryption file sizes")
// Trigger index re-transfer with fixed up sizes
db.DropDeltaIndexIDs(ll)
for folderID, folderCfg := range encFolders {
fset, err := db.NewFileSet(folderID, ll)
if err != nil {
return err
}
snap, err := fset.Snapshot()
if err != nil {
return err
}
batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
// As we can't touch the version, we need to first invalidate the
// files, and then re-add the modified valid files
invalidFiles := make([]protocol.FileInfo, len(files))
for i, f := range files {
f.SetUnsupported()
invalidFiles[i] = f
}
fset.Update(protocol.LocalDeviceID, invalidFiles)
fset.Update(protocol.LocalDeviceID, files)
return nil
})
filesystem := folderCfg.Filesystem(fset)
var innerErr error
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
size, err := sizeOfEncryptedTrailer(filesystem, fi.Name)
if err != nil {
// Best effort: If we fail to read a file, it will show as
// locally changed on next scan.
return true
}
fi.EncryptionTrailerSize = size
batch.Append(fi)
err = batch.FlushIfFull()
if err != nil {
innerErr = err
return false
}
return true
})
snap.Release()
if innerErr != nil {
return innerErr
}
err = batch.Flush()
if err != nil {
return err
}
}
return nil
}
// sizeOfEncryptedTrailer returns the size of the encrypted trailer on disk.
// This amount of bytes should be subtracted from the file size to get the
// original file size.
func sizeOfEncryptedTrailer(fs fs.Filesystem, name string) (int, error) {
f, err := fs.Open(name)
if err != nil {
return 0, err
}
defer f.Close()
if _, err := f.Seek(-4, io.SeekEnd); err != nil {
return 0, err
}
var buf [4]byte
if _, err := io.ReadFull(f, buf[:]); err != nil {
return 0, err
}
// The stored size is the size of the encrypted data.
size := int(binary.BigEndian.Uint32(buf[:]))
// We add the size of the length word itself as well.
return size + 4, nil
}
lib/syncthing/internals.go
+32 -5
View File
@@ -8,9 +8,10 @@ package syncthing
import (
"context"
"iter"
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stats"
@@ -23,6 +24,8 @@ type Internals struct {
model model.Model
}
type Counts = db.Counts
func newInternals(model model.Model) *Internals {
return &Internals{
model: model,
@@ -77,14 +80,38 @@ func (m *Internals) PendingFolders(deviceID protocol.DeviceID) (map[string]db.Pe
return m.model.PendingFolders(deviceID)
}
func (m *Internals) DBSnapshot(folderID string) (*db.Snapshot, error) {
return m.model.DBSnapshot(folderID)
}
func (m *Internals) ScanFolderSubdirs(folderID string, paths []string) error {
return m.model.ScanFolderSubdirs(folderID, paths)
}
func (m *Internals) GlobalSize(folder string) (Counts, error) {
counts, err := m.model.GlobalSize(folder)
if err != nil {
return Counts{}, err
}
return counts, nil
}
func (m *Internals) LocalSize(folder string) (Counts, error) {
counts, err := m.model.LocalSize(folder, protocol.LocalDeviceID)
if err != nil {
return Counts{}, err
}
return counts, nil
}
func (m *Internals) NeedSize(folder string, device protocol.DeviceID) (Counts, error) {
counts, err := m.model.NeedSize(folder, device)
if err != nil {
return Counts{}, err
}
return counts, nil
}
func (m *Internals) AllGlobalFiles(folder string) (iter.Seq[db.FileMetadata], func() error) {
return m.model.AllGlobalFiles(folder)
}
func (m *Internals) FolderProgressBytesCompleted(folder string) int64 {
return m.model.FolderProgressBytesCompleted(folder)
}
lib/syncthing/syncthing.go
+33 -34
View File
@@ -22,13 +22,12 @@ import (
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/lib/api"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/connections/registry"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/locations"
@@ -52,21 +51,19 @@ const (
)
type Options struct {
AuditWriter io.Writer
NoUpgrade bool
ProfilerAddr string
ResetDeltaIdxs bool
Verbose bool
// null duration means use default value
DBRecheckInterval time.Duration
DBIndirectGCInterval time.Duration
AuditWriter io.Writer
NoUpgrade bool
ProfilerAddr string
ResetDeltaIdxs bool
Verbose bool
DBMaintenanceInterval time.Duration
}
type App struct {
myID protocol.DeviceID
mainService *suture.Supervisor
cfg config.Wrapper
ll *db.Lowlevel
sdb db.DB
evLogger events.Logger
cert tls.Certificate
opts Options
@@ -80,14 +77,10 @@ type App struct {
Internals *Internals
}
func New(cfg config.Wrapper, dbBackend backend.Backend, evLogger events.Logger, cert tls.Certificate, opts Options) (*App, error) {
ll, err := db.NewLowlevel(dbBackend, evLogger, db.WithRecheckInterval(opts.DBRecheckInterval), db.WithIndirectGCInterval(opts.DBIndirectGCInterval))
if err != nil {
return nil, err
}
func New(cfg config.Wrapper, sdb db.DB, evLogger events.Logger, cert tls.Certificate, opts Options) (*App, error) {
a := &App{
cfg: cfg,
ll: ll,
sdb: sdb,
evLogger: evLogger,
opts: opts,
cert: cert,
@@ -124,7 +117,7 @@ func (a *App) Start() error {
func (a *App) startup() error {
a.mainService.Add(ur.NewFailureHandler(a.cfg, a.evLogger))
a.mainService.Add(a.ll)
a.mainService.Add(a.sdb.Service(a.opts.DBMaintenanceInterval))
if a.opts.AuditWriter != nil {
a.mainService.Add(newAuditService(a.opts.AuditWriter, a.evLogger))
@@ -180,14 +173,12 @@ func (a *App) startup() error {
perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond)
l.Infof("Hashing performance is %.02f MB/s", perf)
if err := db.UpdateSchema(a.ll); err != nil {
l.Warnln("Database schema:", err)
return err
}
if a.opts.ResetDeltaIdxs {
l.Infoln("Reinitializing delta index IDs")
db.DropDeltaIndexIDs(a.ll)
if err := a.sdb.DropAllIndexIDs(); err != nil {
l.Warnln("Drop index IDs:", err)
return err
}
}
protectedFiles := []string{
@@ -198,17 +189,22 @@ func (a *App) startup() error {
}
// Remove database entries for folders that no longer exist in the config
folders := a.cfg.Folders()
for _, folder := range a.ll.ListFolders() {
if _, ok := folders[folder]; !ok {
cfgFolders := a.cfg.Folders()
dbFolders, err := a.sdb.ListFolders()
if err != nil {
l.Warnln("Listing folders:", err)
return err
}
for _, folder := range dbFolders {
if _, ok := cfgFolders[folder]; !ok {
l.Infof("Cleaning metadata for dropped folder %q", folder)
db.DropFolder(a.ll, folder)
a.sdb.DropFolder(folder)
}
}
// Grab the previously running version string from the database.
miscDB := db.NewMiscDataNamespace(a.ll)
miscDB := db.NewMiscDB(a.sdb)
prevVersion, _, err := miscDB.String("prevVersion")
if err != nil {
l.Warnln("Database:", err)
@@ -229,7 +225,10 @@ func (a *App) startup() error {
if a.cfg.Options().SendFullIndexOnUpgrade {
// Drop delta indexes in case we've changed random stuff we
// shouldn't have. We will resend our index on next connect.
db.DropDeltaIndexIDs(a.ll)
if err := a.sdb.DropAllIndexIDs(); err != nil {
l.Warnln("Drop index IDs:", err)
return err
}
}
}
@@ -238,13 +237,13 @@ func (a *App) startup() error {
miscDB.PutString("prevVersion", build.Version)
}
if err := globalMigration(a.ll, a.cfg); err != nil {
if err := globalMigration(a.sdb, a.cfg); err != nil {
l.Warnln("Global migration:", err)
return err
}
keyGen := protocol.NewKeyGenerator()
m := model.NewModel(a.cfg, a.myID, a.ll, protectedFiles, a.evLogger, keyGen)
m := model.NewModel(a.cfg, a.myID, a.sdb, protectedFiles, a.evLogger, keyGen)
a.Internals = newInternals(m)
a.mainService.Add(m)
@@ -333,7 +332,7 @@ func (a *App) wait(errChan <-chan error) {
done := make(chan struct{})
go func() {
a.ll.Close()
a.sdb.Close()
close(done)
}()
select {
@@ -399,7 +398,7 @@ func (a *App) stopWithErr(stopReason svcutil.ExitStatus, err error) svcutil.Exit
return a.exitStatus
}
func (a *App) setupGUI(m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, errors, systemLog logger.Recorder, miscDB *db.NamespacedKV) error {
func (a *App) setupGUI(m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, errors, systemLog logger.Recorder, miscDB *db.Typed) error {
guiCfg := a.cfg.GUI()
if !guiCfg.Enabled {
lib/syncthing/syncthing_test.go
+12 -6
View File
@@ -8,11 +8,12 @@ package syncthing
import (
"os"
"strings"
"testing"
"time"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/svcutil"
@@ -71,8 +72,14 @@ func TestStartupFail(t *testing.T) {
}, protocol.LocalDeviceID, events.NoopLogger)
defer os.Remove(cfg.ConfigPath())
db := backend.OpenMemory()
app, err := New(cfg, db, events.NoopLogger, cert, Options{})
sdb, err := sqlite.OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
sdb.Close()
})
app, err := New(cfg, sdb, events.NoopLogger, cert, Options{})
if err != nil {
t.Fatal(err)
}
@@ -102,10 +109,9 @@ func TestStartupFail(t *testing.T) {
t.Errorf(`Got different errors "%v" from Start and "%v" from Error`, startErr, err)
}
if trans, err := db.NewReadTransaction(); err == nil {
if _, err := sdb.ListFolders(); err == nil {
t.Error("Expected error due to db being closed, got nil")
trans.Release()
} else if !backend.IsClosed(err) {
} else if !strings.Contains(err.Error(), "closed") {
t.Error("Expected error due to db being closed, got", err)
}
}
lib/syncthing/utils.go
+129 -3
View File
@@ -12,9 +12,16 @@ import (
"fmt"
"io"
"os"
"sync"
"time"
"github.com/syncthing/syncthing/internal/db"
newdb "github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/olddb"
"github.com/syncthing/syncthing/internal/db/olddb/backend"
"github.com/syncthing/syncthing/internal/db/sqlite"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/locations"
@@ -150,6 +157,125 @@ func copyFile(src, dst string) error {
return nil
}
func OpenDBBackend(path string, tuning config.Tuning) (backend.Backend, error) {
return backend.Open(path, backend.Tuning(tuning))
// Opens a database
func OpenDatabase(path string) (newdb.DB, error) {
sql, err := sqlite.Open(path)
if err != nil {
return nil, err
}
sdb := newdb.MetricsWrap(sql)
return sdb, nil
}
// Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type
func TryMigrateDatabase() error {
oldDBDir := locations.Get(locations.LegacyDatabase)
if _, err := os.Lstat(oldDBDir); err != nil {
// No old database
return nil
}
be, err := backend.OpenLevelDBRO(oldDBDir)
if err != nil {
// Apparently, not a valid old database
return nil
}
sdb, err := sqlite.OpenForMigration(locations.Get(locations.Database))
if err != nil {
return err
}
miscDB := db.NewMiscDB(sdb)
if when, ok, err := miscDB.Time("migrated-from-leveldb-at"); err == nil && ok {
l.Warnf("Old-style database present but already migrated at %v; please manually move or remove %s.", when, oldDBDir)
return nil
}
l.Infoln("Migrating old-style database to SQLite; this may take a while...")
t0 := time.Now()
ll, err := olddb.NewLowlevel(be)
if err != nil {
return err
}
totFiles, totBlocks := 0, 0
for _, folder := range ll.ListFolders() {
// Start a writer routine
fis := make(chan protocol.FileInfo, 50)
var writeErr error
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
var batch []protocol.FileInfo
files, blocks := 0, 0
t0 := time.Now()
t1 := time.Now()
for fi := range fis {
batch = append(batch, fi)
files++
blocks += len(fi.Blocks)
if len(batch) == 1000 {
writeErr = sdb.Update(folder, protocol.LocalDeviceID, batch)
if writeErr != nil {
return
}
batch = batch[:0]
if time.Since(t1) > 10*time.Second {
d := time.Since(t0) + 1
t1 = time.Now()
l.Infof("Migrating folder %s... (%d files and %dk blocks in %v, %.01f files/s)", folder, files, blocks/1000, d.Truncate(time.Second), float64(files)/d.Seconds())
}
}
}
if len(batch) > 0 {
writeErr = sdb.Update(folder, protocol.LocalDeviceID, batch)
}
d := time.Since(t0) + 1
l.Infof("Migrated folder %s; %d files and %dk blocks in %v, %.01f files/s", folder, files, blocks/1000, d.Truncate(time.Second), float64(files)/d.Seconds())
totFiles += files
totBlocks += blocks
}()
// Iterate the existing files
fs, err := olddb.NewFileSet(folder, ll)
if err != nil {
return err
}
snap, err := fs.Snapshot()
if err != nil {
return err
}
_ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool {
fis <- fi
return true
})
close(fis)
snap.Release()
// Wait for writes to complete
wg.Wait()
if writeErr != nil {
return writeErr
}
}
l.Infoln("Migrating virtual mtimes...")
if err := ll.IterateMtimes(sdb.PutMtime); err != nil {
l.Warnln("Failed to migrate mtimes:", err)
}
_ = miscDB.PutTime("migrated-from-leveldb-at", time.Now())
_ = miscDB.PutString("migrated-from-leveldb-by", build.LongVersion)
be.Close()
sdb.Close()
_ = os.Rename(oldDBDir, oldDBDir+"-migrated")
l.Infof("Migration complete, %d files and %dk blocks in %s", totFiles, totBlocks/1000, time.Since(t0).Truncate(time.Second))
return nil
}