felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Reorder sequences for better rename detection (#6574)

Browse Source
This commit is contained in:
Audrius Butkevicius
2020-05-11 20:15:11 +02:00
committed by GitHub
parent aac3750298
commit decb967969
9 changed files with 409 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/folder.go
+49
View File
@@ -464,6 +464,13 @@ func (f *folder) scanSubdirs(subDirs []string) error {
batch.append(res.File)
changes++
if f.localFlags&protocol.FlagLocalReceiveOnly == 0 {
if nf, ok := f.findRename(snap, mtimefs, res.File); ok {
batch.append(nf)
changes++
}
}
}
if err := batch.flush(); err != nil {
@@ -615,6 +622,48 @@ func (f *folder) scanSubdirs(subDirs []string) error {
return nil
}
func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file protocol.FileInfo) (protocol.FileInfo, bool) {
found := false
nf := protocol.FileInfo{}
snap.WithBlocksHash(file.BlocksHash, func(ifi db.FileIntf) bool {
fi := ifi.(protocol.FileInfo)
select {
case <-f.ctx.Done():
return false
default:
}
if fi.ShouldConflict() {
return true
}
if f.ignores.Match(fi.Name).IsIgnored() {
return true
}
// Only check the size.
// No point checking block equality, as that uses BlocksHash comparison if that is set (which it will be).
// No point checking BlocksHash comparison as WithBlocksHash already does that.
if file.Size != fi.Size {
return true
}
if !osutil.IsDeleted(mtimefs, fi.Name) {
return true
}
nf = fi
nf.SetDeleted(f.shortID)
nf.LocalFlags = f.localFlags
found = true
return false
})
return nf, found
}
func (f *folder) scanTimerFired() {
err := f.scanSubdirs(nil)
lib/model/folder_sendrecv.go
+30 -29
View File
@@ -355,7 +355,7 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
if ok && !df.IsDeleted() && !df.IsSymlink() && !df.IsDirectory() && !df.IsInvalid() {
fileDeletions[file.Name] = file
// Put files into buckets per first hash
key := string(df.Blocks[0].Hash)
key := string(df.BlocksHash)
buckets[key] = append(buckets[key], df)
} else {
f.deleteFileWithCurrent(file, df, ok, dbUpdateChan, scanChan)
@@ -458,30 +458,28 @@ nextFile:
// Check our list of files to be removed for a match, in which case
// we can just do a rename instead.
key := string(fi.Blocks[0].Hash)
key := string(fi.BlocksHash)
for i, candidate := range buckets[key] {
if candidate.BlocksEqual(fi) {
// Remove the candidate from the bucket
lidx := len(buckets[key]) - 1
buckets[key][i] = buckets[key][lidx]
buckets[key] = buckets[key][:lidx]
// Remove the candidate from the bucket
lidx := len(buckets[key]) - 1
buckets[key][i] = buckets[key][lidx]
buckets[key] = buckets[key][:lidx]
// candidate is our current state of the file, where as the
// desired state with the delete bit set is in the deletion
// map.
desired := fileDeletions[candidate.Name]
if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil {
// Failed to rename, try to handle files as separate
// deletions and updates.
break
}
// Remove the pending deletion (as we performed it by renaming)
delete(fileDeletions, candidate.Name)
f.queue.Done(fileName)
continue nextFile
// candidate is our current state of the file, where as the
// desired state with the delete bit set is in the deletion
// map.
desired := fileDeletions[candidate.Name]
if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil {
l.Debugln("rename shortcut for %s failed: %S", fi.Name, err.Error())
// Failed to rename, try next one.
continue
}
// Remove the pending deletion (as we performed it by renaming)
delete(fileDeletions, candidate.Name)
f.queue.Done(fileName)
continue nextFile
}
devices := snap.Availability(fileName)
@@ -1181,6 +1179,16 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
protocol.BufferPool.Put(buf)
}()
folderFilesystems := make(map[string]fs.Filesystem)
// Hope that it's usually in the same folder, so start with that one.
folders := []string{f.folderID}
for folder, cfg := range f.model.cfg.Folders() {
folderFilesystems[folder] = cfg.Filesystem()
if folder != f.folderID {
folders = append(folders, folder)
}
}
for state := range in {
if err := f.CheckAvailableSpace(state.file.Size); err != nil {
state.fail(err)
@@ -1198,13 +1206,6 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
f.model.progressEmitter.Register(state.sharedPullerState)
folderFilesystems := make(map[string]fs.Filesystem)
var folders []string
for folder, cfg := range f.model.cfg.Folders() {
folderFilesystems[folder] = cfg.Filesystem()
folders = append(folders, folder)
}
var file fs.File
var weakHashFinder *weakhash.Finder
lib/model/model.go
+15 -3
View File
@@ -1926,9 +1926,15 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
var f protocol.FileInfo
snap := s.fset.Snapshot()
defer snap.Release()
previousWasDelete := false
snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
if err = batch.flushIfFull(); err != nil {
return false
// This is to make sure that renames (which is an add followed by a delete) land in the same batch.
// Even if the batch is full, we allow a last delete to slip in, we do this by making sure that
// the batch ends with a non-delete, or that the last item in the batch is already a delete
if batch.full() && (!fi.IsDeleted() || previousWasDelete) {
if err = batch.flush(); err != nil {
return false
}
}
if shouldDebug() {
@@ -1964,6 +1970,8 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
}
f.LocalFlags = 0 // never sent externally
previousWasDelete = f.IsDeleted()
batch.append(f)
return true
})
@@ -2607,8 +2615,12 @@ func (b *fileInfoBatch) append(f protocol.FileInfo) {
b.size += f.ProtoSize()
}
func (b *fileInfoBatch) full() bool {
return len(b.infos) >= maxBatchSizeFiles || b.size >= maxBatchSizeBytes
}
func (b *fileInfoBatch) flushIfFull() error {
if len(b.infos) >= maxBatchSizeFiles || b.size >= maxBatchSizeBytes {
if b.full() {
return b.flush()
}
return nil
lib/model/model_test.go
+152
View File
@@ -18,6 +18,7 @@ import (
"path/filepath"
"runtime"
"runtime/pprof"
"sort"
"strconv"
"strings"
"sync"
@@ -3537,3 +3538,154 @@ func TestFolderAPIErrors(t *testing.T) {
}
}
}
func TestRenameSequenceOrder(t *testing.T) {
wcfg, fcfg := tmpDefaultWrapper()
m := setupModel(wcfg)
defer cleanupModel(m)
numFiles := 20
ffs := fcfg.Filesystem()
for i := 0; i < numFiles; i++ {
v := fmt.Sprintf("%d", i)
must(t, writeFile(ffs, v, []byte(v), 0644))
}
m.ScanFolders()
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != numFiles {
t.Errorf("Unexpected count: %d != %d", count, numFiles)
}
// Modify all the files, other than the ones we expect to rename
for i := 0; i < numFiles; i++ {
if i == 3 || i == 17 || i == 16 || i == 4 {
continue
}
v := fmt.Sprintf("%d", i)
must(t, writeFile(ffs, v, []byte(v+"-new"), 0644))
}
// Rename
must(t, ffs.Rename("3", "17"))
must(t, ffs.Rename("16", "4"))
// Scan
m.ScanFolders()
// Verify sequence of a appearing is followed by c disappearing.
snap = dbSnapshot(t, m, "default")
defer snap.Release()
var firstExpectedSequence int64
var secondExpectedSequence int64
failed := false
snap.WithHaveSequence(0, func(i db.FileIntf) bool {
t.Log(i)
if i.FileName() == "17" {
firstExpectedSequence = i.SequenceNo() + 1
}
if i.FileName() == "4" {
secondExpectedSequence = i.SequenceNo() + 1
}
if i.FileName() == "3" {
failed = i.SequenceNo() != firstExpectedSequence || failed
}
if i.FileName() == "16" {
failed = i.SequenceNo() != secondExpectedSequence || failed
}
return true
})
if failed {
t.Fail()
}
}
func TestBlockListMap(t *testing.T) {
wcfg, fcfg := tmpDefaultWrapper()
m := setupModel(wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem()
must(t, writeFile(ffs, "one", []byte("content"), 0644))
must(t, writeFile(ffs, "two", []byte("content"), 0644))
must(t, writeFile(ffs, "three", []byte("content"), 0644))
must(t, writeFile(ffs, "four", []byte("content"), 0644))
must(t, writeFile(ffs, "five", []byte("content"), 0644))
m.ScanFolders()
snap := dbSnapshot(t, m, "default")
defer snap.Release()
fi, ok := snap.Get(protocol.LocalDeviceID, "one")
if !ok {
t.Error("failed to find existing file")
}
var paths []string
snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
paths = append(paths, fi.FileName())
return true
})
snap.Release()
expected := []string{"one", "two", "three", "four", "five"}
if !equalStringsInAnyOrder(paths, expected) {
t.Errorf("expected %q got %q", expected, paths)
}
// Fudge the files around
// Remove
must(t, ffs.Remove("one"))
// Modify
must(t, ffs.Remove("two"))
must(t, writeFile(ffs, "two", []byte("mew-content"), 0644))
// Rename
must(t, ffs.Rename("three", "new-three"))
// Change type
must(t, ffs.Remove("four"))
must(t, ffs.Mkdir("four", 0644))
m.ScanFolders()
// Check we're left with 2 of the 5
snap = dbSnapshot(t, m, "default")
defer snap.Release()
paths = paths[:0]
snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
paths = append(paths, fi.FileName())
return true
})
snap.Release()
expected = []string{"new-three", "five"}
if !equalStringsInAnyOrder(paths, expected) {
t.Errorf("expected %q got %q", expected, paths)
}
}
func equalStringsInAnyOrder(a, b []string) bool {
if len(a) != len(b) {
return false
}
sort.Strings(a)
sort.Strings(b)
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}