felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/db: Refactor to use global list by version (fixes #6372) (#6638)

Browse Source 
Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei
2020-05-30 09:50:23 +02:00
committed by GitHub
co-authored by Jakob Borg
parent 1eea076f5c
commit 1f8e6c55f6
21 changed files with 1540 additions and 504 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/folder.go
+3 -3
View File
@@ -303,7 +303,7 @@ func (f *folder) pull() (success bool) {
// If there is nothing to do, don't even enter sync-waiting state.
abort := true
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
abort = false
return false
})
@@ -499,7 +499,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
for _, sub := range subDirs {
var iterError error
snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi protocol.FileIntf) bool {
select {
case <-f.ctx.Done():
return false
@@ -634,7 +634,7 @@ func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file proto
found := false
nf := protocol.FileInfo{}
snap.WithBlocksHash(file.BlocksHash, func(ifi db.FileIntf) bool {
snap.WithBlocksHash(file.BlocksHash, func(ifi protocol.FileIntf) bool {
fi := ifi.(protocol.FileInfo)
select {
lib/model/folder_recvonly.go
+1 -1
View File
@@ -87,7 +87,7 @@ func (f *receiveOnlyFolder) revert() {
batchSizeBytes := 0
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
lib/model/folder_sendonly.go
+2 -2
View File
@@ -52,7 +52,7 @@ func (f *sendOnlyFolder) pull() bool {
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.updateLocalsFromPulling(batch)
batch = batch[:0]
@@ -110,7 +110,7 @@ func (f *sendOnlyFolder) override() {
batchSizeBytes := 0
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.updateLocalsFromScanning(batch)
lib/model/folder_sendrecv.go
+1 -1
View File
@@ -305,7 +305,7 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
// Regular files to pull goes into the file queue, everything else
// (directories, symlinks and deletes) goes into the "process directly"
// pile.
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
select {
case <-f.ctx.Done():
return false
lib/model/model.go
+3 -3
View File
@@ -851,7 +851,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
}
rest = make([]db.FileInfoTruncated, 0, perpage)
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
@@ -1936,7 +1936,7 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
snap := s.fset.Snapshot()
defer snap.Release()
previousWasDelete := false
snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
snap.WithHaveSequence(s.prevSequence+1, func(fi protocol.FileIntf) bool {
// This is to make sure that renames (which is an add followed by a delete) land in the same batch.
// Even if the batch is full, we allow a last delete to slip in, we do this by making sure that
// the batch ends with a non-delete, or that the last item in the batch is already a delete
@@ -2248,7 +2248,7 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
snap := files.Snapshot()
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
// Don't include the prefix itself.
lib/model/model_test.go
+11 -11
View File
@@ -2426,7 +2426,7 @@ func TestIssue3496(t *testing.T) {
m.fmut.RUnlock()
var localFiles []protocol.FileInfo
snap := fs.Snapshot()
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
localFiles = append(localFiles, i.(protocol.FileInfo))
return true
})
@@ -3556,7 +3556,7 @@ func TestRenameSequenceOrder(t *testing.T) {
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
count++
return true
})
@@ -3588,7 +3588,7 @@ func TestRenameSequenceOrder(t *testing.T) {
var firstExpectedSequence int64
var secondExpectedSequence int64
failed := false
snap.WithHaveSequence(0, func(i db.FileIntf) bool {
snap.WithHaveSequence(0, func(i protocol.FileIntf) bool {
t.Log(i)
if i.FileName() == "17" {
firstExpectedSequence = i.SequenceNo() + 1
@@ -3621,7 +3621,7 @@ func TestRenameSameFile(t *testing.T) {
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
count++
return true
})
@@ -3644,7 +3644,7 @@ func TestRenameSameFile(t *testing.T) {
prevSeq := int64(0)
seen := false
snap.WithHaveSequence(0, func(i db.FileIntf) bool {
snap.WithHaveSequence(0, func(i protocol.FileIntf) bool {
if i.SequenceNo() <= prevSeq {
t.Fatalf("non-increasing sequences: %d <= %d", i.SequenceNo(), prevSeq)
}
@@ -3683,7 +3683,7 @@ func TestRenameEmptyFile(t *testing.T) {
}
count := 0
snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool {
snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool {
count++
return true
})
@@ -3693,7 +3693,7 @@ func TestRenameEmptyFile(t *testing.T) {
}
count = 0
snap.WithBlocksHash(file.BlocksHash, func(_ db.FileIntf) bool {
snap.WithBlocksHash(file.BlocksHash, func(_ protocol.FileIntf) bool {
count++
return true
})
@@ -3712,7 +3712,7 @@ func TestRenameEmptyFile(t *testing.T) {
defer snap.Release()
count = 0
snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool {
snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool {
count++
return true
})
@@ -3722,7 +3722,7 @@ func TestRenameEmptyFile(t *testing.T) {
}
count = 0
snap.WithBlocksHash(file.BlocksHash, func(i db.FileIntf) bool {
snap.WithBlocksHash(file.BlocksHash, func(i protocol.FileIntf) bool {
count++
if i.FileName() != "new-file" {
t.Fatalf("unexpected file name %s, expected new-file", i.FileName())
@@ -3757,7 +3757,7 @@ func TestBlockListMap(t *testing.T) {
}
var paths []string
snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool {
paths = append(paths, fi.FileName())
return true
})
@@ -3790,7 +3790,7 @@ func TestBlockListMap(t *testing.T) {
defer snap.Release()
paths = paths[:0]
snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool {
paths = append(paths, fi.FileName())
return true
})