chore(model): don't check existing file twice in rename detection (#10833)
This is related to and lessens issue:
https://github.com/syncthing/syncthing/issues/10777
@SoongVilda found that a recent change to track already deleted or
existing files only in the scope of a batch, also stopped tracking
already existing files. Thus the osutil.IsDeleted check can be executed
many times if lots of identical files get picked up in a scan:
https://github.com/syncthing/syncthing/issues/10777#issuecomment-4991039201
The change in question:
chore(model): more efficient tracking of renames during scan (#10653)
2721b7b522
---------
Signed-off-by: Simon Frei <freisim93@gmail.com>
This commit is contained in:
+15
-14
@@ -582,17 +582,17 @@ func (f *folder) scanSubdirs(ctx context.Context, subDirs []string) error {
|
|||||||
const maxToRemove = 1000
|
const maxToRemove = 1000
|
||||||
|
|
||||||
type scanBatch struct {
|
type scanBatch struct {
|
||||||
f *folder
|
f *folder
|
||||||
updateBatch *FileInfoBatch
|
updateBatch *FileInfoBatch
|
||||||
toRemove []string
|
toRemove []string
|
||||||
deleted map[string]struct{}
|
skipInFindRename map[string]struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *folder) newScanBatch() *scanBatch {
|
func (f *folder) newScanBatch() *scanBatch {
|
||||||
b := &scanBatch{
|
b := &scanBatch{
|
||||||
f: f,
|
f: f,
|
||||||
toRemove: make([]string, 0, maxToRemove),
|
toRemove: make([]string, 0, maxToRemove),
|
||||||
deleted: make(map[string]struct{}),
|
skipInFindRename: make(map[string]struct{}),
|
||||||
}
|
}
|
||||||
b.updateBatch = NewFileInfoBatch(func(fs []protocol.FileInfo) error {
|
b.updateBatch = NewFileInfoBatch(func(fs []protocol.FileInfo) error {
|
||||||
if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
|
if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
|
||||||
@@ -600,7 +600,7 @@ func (f *folder) newScanBatch() *scanBatch {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.f.updateLocalsFromScanning(fs)
|
b.f.updateLocalsFromScanning(fs)
|
||||||
clear(b.deleted)
|
clear(b.skipInFindRename)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
return b
|
return b
|
||||||
@@ -636,12 +636,12 @@ func (b *scanBatch) FlushIfFull() error {
|
|||||||
return b.updateBatch.FlushIfFull()
|
return b.updateBatch.FlushIfFull()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *scanBatch) markDeleted(name string) {
|
func (b *scanBatch) markSkipInFindRename(name string) {
|
||||||
b.deleted[name] = struct{}{}
|
b.skipInFindRename[name] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *scanBatch) hasDeleted(name string) bool {
|
func (b *scanBatch) shouldSkipInFindRenames(name string) bool {
|
||||||
_, ok := b.deleted[name]
|
_, ok := b.skipInFindRename[name]
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -756,7 +756,6 @@ func (f *folder) scanSubdirsChangedAndNew(ctx context.Context, subDirs []string,
|
|||||||
return 0, err
|
return 0, err
|
||||||
} else if ok {
|
} else if ok {
|
||||||
changes++
|
changes++
|
||||||
batch.markDeleted(nf.Name)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -950,7 +949,7 @@ loop:
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if batch.hasDeleted(fi.Name) {
|
if batch.shouldSkipInFindRenames(fi.Name) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -970,6 +969,7 @@ loop:
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !osutil.IsDeleted(f.mtimefs, fi.Name) {
|
if !osutil.IsDeleted(f.mtimefs, fi.Name) {
|
||||||
|
batch.markSkipInFindRename(fi.Name)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -981,6 +981,7 @@ loop:
|
|||||||
nf.SetDeleted(f.shortID)
|
nf.SetDeleted(f.shortID)
|
||||||
nf.LocalFlags = f.localFlags
|
nf.LocalFlags = f.localFlags
|
||||||
found = true
|
found = true
|
||||||
|
batch.markSkipInFindRename(fi.Name)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user