felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: switch database engine to sqlite (fixes #9954) (#9965)

Browse Source 
Switch the database from LevelDB to SQLite, for greater stability and
simpler code.

Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl>
Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
2025-03-29 13:50:08 +01:00
committed by GitHub
co-authored by Tommy van der Vorst bt90
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/folder_sendrecv_test.go
+56 -66
View File
@@ -19,6 +19,7 @@ import (
"testing"
"time"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
@@ -149,7 +150,7 @@ func TestHandleFile(t *testing.T) {
copyChan := make(chan copyBlocksState, 1)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(requiredFile, copyChan)
// Receive the results
toCopy := <-copyChan
@@ -189,13 +190,13 @@ func TestHandleFileWithTemp(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t, existingFile)
defer wcfgCancel()
if _, err := prepareTmpFile(f.Filesystem(nil)); err != nil {
if _, err := prepareTmpFile(f.Filesystem()); err != nil {
t.Fatal(err)
}
copyChan := make(chan copyBlocksState, 1)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(requiredFile, copyChan)
// Receive the results
toCopy := <-copyChan
@@ -239,7 +240,7 @@ func TestCopierFinder(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t, existingFile)
defer wcfgCancel()
if _, err := prepareTmpFile(f.Filesystem(nil)); err != nil {
if _, err := prepareTmpFile(f.Filesystem()); err != nil {
t.Fatal(err)
}
@@ -251,7 +252,7 @@ func TestCopierFinder(t *testing.T) {
go f.copierRoutine(copyChan, pullChan, finisherChan)
defer close(copyChan)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(requiredFile, copyChan)
timeout := time.After(10 * time.Second)
pulls := make([]pullBlockState, 4)
@@ -272,8 +273,9 @@ func TestCopierFinder(t *testing.T) {
defer cleanupSharedPullerState(finish)
select {
case <-pullChan:
t.Fatal("Pull channel has data to be read")
case v := <-pullChan:
t.Logf("%+v\n", v)
t.Fatal("Pull channel had data to be read")
case <-finisherChan:
t.Fatal("Finisher channel has data to be read")
default:
@@ -299,7 +301,7 @@ func TestCopierFinder(t *testing.T) {
}
// Verify that the fetched blocks have actually been written to the temp file
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil)
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(), tempFile, protocol.MinBlockSize, nil)
if err != nil {
t.Log(err)
}
@@ -313,10 +315,6 @@ func TestCopierFinder(t *testing.T) {
// Test that updating a file removes its old blocks from the blockmap
func TestCopierCleanup(t *testing.T) {
iterFn := func(folder, file string, index int32) bool {
return true
}
// Create a file
file := setupFile("test", []int{0})
file.Size = 1
@@ -328,11 +326,11 @@ func TestCopierCleanup(t *testing.T) {
// Update index (removing old blocks)
f.updateLocalsFromScanning([]protocol.FileInfo{file})
if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[0].Hash)); err != nil || len(vals) > 0 {
t.Error("Unexpected block found")
}
if !m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[1].Hash)); err != nil || len(vals) == 0 {
t.Error("Expected block not found")
}
@@ -341,11 +339,11 @@ func TestCopierCleanup(t *testing.T) {
// Update index (removing old blocks)
f.updateLocalsFromScanning([]protocol.FileInfo{file})
if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[0].Hash)); err != nil || len(vals) == 0 {
t.Error("Unexpected block found")
}
if m.finder.Iterate(folders, blocks[1].Hash, iterFn) {
if vals, err := itererr.Collect(m.sdb.AllLocalBlocksWithHash(blocks[1].Hash)); err != nil || len(vals) > 0 {
t.Error("Expected block not found")
}
}
@@ -371,10 +369,9 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState, 1)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := fsetSnapshot(t, f.fset)
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
defer func() {
close(copyChan)
@@ -384,7 +381,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
close(finisherChan)
}()
f.handleFile(file, snap, copyChan)
f.handleFile(file, copyChan)
// Receive a block at puller, to indicate that at least a single copier
// loop has been performed.
@@ -471,16 +468,15 @@ func TestDeregisterOnFailInPull(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := fsetSnapshot(t, f.fset)
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
pullWg := sync.NewWaitGroup()
pullWg.Add(1)
go func() {
f.pullerRoutine(snap, pullChan, finisherBufferChan)
f.pullerRoutine(pullChan, finisherBufferChan)
pullWg.Done()
}()
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
defer func() {
// Unblock copier and puller
go func() {
@@ -495,7 +491,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
close(finisherChan)
}()
f.handleFile(file, snap, copyChan)
f.handleFile(file, copyChan)
// Receive at finisher, we should error out as puller has nowhere to pull
// from.
@@ -558,7 +554,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
func TestIssue3164(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
ignDir := filepath.Join("issue3164", "oktodelete")
subDir := filepath.Join(ignDir, "foobar")
@@ -577,7 +573,7 @@ func TestIssue3164(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(file, fsetSnapshot(t, f.fset), dbUpdateChan, make(chan string))
f.deleteDir(file, dbUpdateChan, make(chan string))
if _, err := ffs.Stat("issue3164"); !fs.IsNotExist(err) {
t.Fatal(err)
@@ -648,7 +644,7 @@ func TestDiffEmpty(t *testing.T) {
func TestDeleteIgnorePerms(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
f.IgnorePerms = true
name := "deleteIgnorePerms"
@@ -692,9 +688,6 @@ func TestCopyOwner(t *testing.T) {
f.folder.FolderConfiguration = newFolderConfiguration(m.cfg, f.ID, f.Label, config.FilesystemTypeFake, "/TestCopyOwner")
f.folder.FolderConfiguration.CopyOwnershipFromParent = true
f.fset = newFileSet(t, f.ID, m.db)
f.mtimefs = f.Filesystem(f.fset)
// Create a parent dir with a certain owner/group.
f.mtimefs.Mkdir("foo", 0o755)
@@ -712,7 +705,7 @@ func TestCopyOwner(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string)
defer close(dbUpdateChan)
f.handleDir(dir, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.handleDir(dir, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // empty the channel for later
case toScan := <-scanChan:
@@ -742,17 +735,16 @@ func TestCopyOwner(t *testing.T) {
// but it's the way data is passed around. When the database update
// comes the finisher is done.
snap := fsetSnapshot(t, f.fset)
finisherChan := make(chan *sharedPullerState)
copierChan, copyWg := startCopier(f, nil, finisherChan)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, nil)
go f.finisherRoutine(finisherChan, dbUpdateChan, nil)
defer func() {
close(copierChan)
copyWg.Wait()
close(finisherChan)
}()
f.handleFile(file, snap, copierChan)
f.handleFile(file, copierChan)
<-dbUpdateChan
info, err = f.mtimefs.Lstat("foo/bar/baz")
@@ -771,7 +763,7 @@ func TestCopyOwner(t *testing.T) {
SymlinkTarget: []byte("over the rainbow"),
}
f.handleSymlink(symlink, snap, dbUpdateChan, scanChan)
f.handleSymlink(symlink, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan:
case toScan := <-scanChan:
@@ -792,7 +784,7 @@ func TestCopyOwner(t *testing.T) {
func TestSRConflictReplaceFileByDir(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
@@ -810,7 +802,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleDir(file, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.handleDir(file, dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@@ -824,7 +816,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
func TestSRConflictReplaceFileByLink(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
@@ -843,7 +835,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleSymlink(file, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.handleSymlink(file, dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@@ -857,7 +849,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
func TestDeleteBehindSymlink(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
link := "link"
linkFile := filepath.Join(link, "file")
@@ -873,7 +865,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
fi.Version = fi.Version.Update(device1.Short())
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteFile(fi, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.deleteFile(fi, dbUpdateChan, scanChan)
select {
case f := <-scanChan:
t.Fatalf("Received %v on scanChan", f)
@@ -903,7 +895,7 @@ func TestPullCtxCancel(t *testing.T) {
var cancel context.CancelFunc
f.ctx, cancel = context.WithCancel(context.Background())
go f.pullerRoutine(fsetSnapshot(t, f.fset), pullChan, finisherChan)
go f.pullerRoutine(pullChan, finisherChan)
defer close(pullChan)
emptyState := func() pullBlockState {
@@ -938,7 +930,7 @@ func TestPullCtxCancel(t *testing.T) {
func TestPullDeleteUnscannedDir(t *testing.T) {
_, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
dir := "foobar"
must(t, ffs.MkdirAll(dir, 0o777))
@@ -949,7 +941,7 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(fi, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
f.deleteDir(fi, dbUpdateChan, scanChan)
if _, err := ffs.Stat(dir); fs.IsNotExist(err) {
t.Error("directory has been deleted")
@@ -967,7 +959,7 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
func TestPullCaseOnlyPerformFinish(t *testing.T) {
m, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
contents := []byte("contents")
@@ -976,16 +968,17 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) {
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
it, errFn := m.LocalFiles(f.ID, protocol.LocalDeviceID)
for i := range it {
if hasCur {
t.Fatal("got more than one file")
}
cur = i
hasCur = true
return true
})
}
if err := errFn(); err != nil {
t.Fatal(err)
}
if !hasCur {
t.Fatal("file is missing")
}
@@ -999,7 +992,7 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) {
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
err := f.performFinish(remote, cur, hasCur, temp, snap, dbUpdateChan, scanChan)
err := f.performFinish(remote, cur, hasCur, temp, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // boring case sensitive filesystem
@@ -1029,7 +1022,7 @@ func TestPullCaseOnlySymlink(t *testing.T) {
func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
m, f, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := f.Filesystem(nil)
ffs := f.Filesystem()
name := "foo"
if dir {
@@ -1041,16 +1034,17 @@ func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileInfo) bool {
it, errFn := m.LocalFiles(f.ID, protocol.LocalDeviceID)
for i := range it {
if hasCur {
t.Fatal("got more than one file")
}
cur = i
hasCur = true
return true
})
}
if err := errFn(); err != nil {
t.Fatal(err)
}
if !hasCur {
t.Fatal("file is missing")
}
@@ -1063,9 +1057,9 @@ func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
remote.Name = strings.ToUpper(cur.Name)
if dir {
f.handleDir(remote, snap, dbUpdateChan, scanChan)
f.handleDir(remote, dbUpdateChan, scanChan)
} else {
f.handleSymlink(remote, snap, dbUpdateChan, scanChan)
f.handleSymlink(remote, dbUpdateChan, scanChan)
}
select {
@@ -1100,7 +1094,7 @@ func TestPullTempFileCaseConflict(t *testing.T) {
fd.Close()
}
f.handleFile(file, fsetSnapshot(t, f.fset), copyChan)
f.handleFile(file, copyChan)
cs := <-copyChan
if _, err := cs.tempFile(); err != nil {
@@ -1142,9 +1136,7 @@ func TestPullCaseOnlyRename(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 2)
scanChan := make(chan string, 2)
snap := fsetSnapshot(t, f.fset)
defer snap.Release()
if err := f.renameFile(cur, deleted, confl, snap, dbUpdateChan, scanChan); err != nil {
if err := f.renameFile(cur, deleted, confl, dbUpdateChan, scanChan); err != nil {
t.Error(err)
}
}
@@ -1219,9 +1211,7 @@ func TestPullDeleteCaseConflict(t *testing.T) {
t.Error("Missing db update for file")
}
snap := fsetSnapshot(t, f.fset)
defer snap.Release()
f.deleteDir(fi, snap, dbUpdateChan, scanChan)
f.deleteDir(fi, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan:
default:
@@ -1249,7 +1239,7 @@ func TestPullDeleteIgnoreChildDir(t *testing.T) {
scanChan := make(chan string, 2)
err := f.deleteDirOnDisk(parent, fsetSnapshot(t, f.fset), scanChan)
err := f.deleteDirOnDisk(parent, scanChan)
if err == nil {
t.Error("no error")
}