felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore(db): use one SQLite database per folder (#10042)

Browse Source 
This changes the database structure to use one database per folder, with
a small main database to coordinate. Reverts the prior change to buffer
all files in memory when pulling, meaning there is now a phase where the
WAL file will grow significantly, at least for initial sync of folders
with many directories.

---------

Co-authored-by: bt90 <btom1990@googlemail.com>
This commit is contained in:
Jakob Borg
2025-04-06 14:30:43 +02:00
committed by GitHub
co-authored by bt90
parent 7d51b1b620
commit cf1cf85ce6
32 changed files with 1696 additions and 1219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/sqlite/db_local_test.go
+21 -20
View File
@@ -9,8 +9,6 @@ package sqlite
import (
"testing"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/itererr"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -52,7 +50,7 @@ func TestBlocks(t *testing.T) {
// Search for blocks
vals, err := itererr.Collect(db.AllLocalBlocksWithHash([]byte{1, 2, 3}))
vals, err := db.AllLocalBlocksWithHash([]byte{1, 2, 3})
if err != nil {
t.Fatal(err)
}
@@ -66,27 +64,23 @@ func TestBlocks(t *testing.T) {
// Get FileInfos for those blocks
found := 0
it, errFn := db.AllLocalFilesWithBlocksHashAnyFolder(vals[0].BlocklistHash)
for folder, fileInfo := range it {
if folder != folderID {
t.Fatal("should be same folder")
}
if fileInfo.Name != "file1" {
t.Fatal("should be file1")
}
found++
}
if err := errFn(); err != nil {
res, err := db.AllLocalFilesWithBlocksHashAnyFolder(vals[0].BlocklistHash)
if err != nil {
t.Fatal(err)
}
if found != 1 {
if len(res) != 1 {
t.Fatal("should return one folder")
}
if len(res[folderID]) != 1 {
t.Fatal("should find one file")
}
if res[folderID][0].Name != "file1" {
t.Fatal("should be file1")
}
// Get the other blocks
vals, err = itererr.Collect(db.AllLocalBlocksWithHash([]byte{3, 4, 5}))
vals, err = db.AllLocalBlocksWithHash([]byte{3, 4, 5})
if err != nil {
t.Fatal(err)
}
@@ -125,7 +119,10 @@ func TestBlocksDeleted(t *testing.T) {
// We should find one entry for the block hash
search := file.Blocks[0].Hash
es := mustCollect[db.BlockMapEntry](t)(sdb.AllLocalBlocksWithHash(search))
es, err := sdb.AllLocalBlocksWithHash(search)
if err != nil {
t.Fatal(err)
}
if len(es) != 1 {
t.Fatal("expected one hit")
}
@@ -137,13 +134,17 @@ func TestBlocksDeleted(t *testing.T) {
}
// Searching for the old hash should yield no hits
if hits := mustCollect[db.BlockMapEntry](t)(sdb.AllLocalBlocksWithHash(search)); len(hits) != 0 {
if hits, err := sdb.AllLocalBlocksWithHash(search); err != nil {
t.Fatal(err)
} else if len(hits) != 0 {
t.Log(hits)
t.Error("expected no hits")
}
// Searching for the new hash should yield one hits
if hits := mustCollect[db.BlockMapEntry](t)(sdb.AllLocalBlocksWithHash(file.Blocks[0].Hash)); len(hits) != 1 {
if hits, err := sdb.AllLocalBlocksWithHash(file.Blocks[0].Hash); err != nil {
t.Fatal(err)
} else if len(hits) != 1 {
t.Log(hits)
t.Error("expected one hit")
}