felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: remove weak hashing which does not pull its weight (#10005)

Browse Source 
We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 13:21:10 +01:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/folder_sendrecv.go
+40 -116
View File
@@ -31,7 +31,6 @@ import (
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
)
var (
@@ -1136,12 +1135,12 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
func (f *sendReceiveFolder) reuseBlocks(blocks []protocol.BlockInfo, reused []int, file protocol.FileInfo, tempName string) ([]protocol.BlockInfo, []int) {
// Check for an old temporary file which might have some blocks we could
// reuse.
tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false)
tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil)
if err != nil {
var caseErr *fs.ErrCaseConflict
if errors.As(err, &caseErr) {
if rerr := f.mtimefs.Rename(caseErr.Real, tempName); rerr == nil {
tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false)
tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil)
}
}
}
@@ -1310,8 +1309,6 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
f.model.progressEmitter.Register(state.sharedPullerState)
}
weakHashFinder, file := f.initWeakHashFinder(state)
blocks:
for _, block := range state.blocks {
select {
@@ -1336,75 +1333,49 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
buf = protocol.BufferPool.Upgrade(buf, int(block.Size))
var found bool
if f.Type != config.FolderTypeReceiveEncrypted {
found, err = weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
if f.verifyBuffer(buf, block) != nil {
return true
}
err = f.limitedWriteAt(dstFd, buf, block.Offset)
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
}
if offset == block.Offset {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromOriginShifted(block.Size)
}
return false
})
found := f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
ffs := folderFilesystems[folder]
fd, err := ffs.Open(path)
if err != nil {
l.Debugln("weak hasher iter", err)
return false
}
}
defer fd.Close()
if !found {
found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
ffs := folderFilesystems[folder]
fd, err := ffs.Open(path)
if err != nil {
srcOffset := int64(state.file.BlockSize()) * int64(index)
_, err = fd.ReadAt(buf, srcOffset)
if err != nil {
return false
}
// Hash is not SHA256 as it's an encrypted hash token. In that
// case we can't verify the block integrity so we'll take it on
// trust. (The other side can and will verify.)
if f.Type != config.FolderTypeReceiveEncrypted {
if err := f.verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
}
defer fd.Close()
}
srcOffset := int64(state.file.BlockSize()) * int64(index)
_, err = fd.ReadAt(buf, srcOffset)
if err != nil {
return false
}
// Hash is not SHA256 as it's an encrypted hash token. In that
// case we can't verify the block integrity so we'll take it on
// trust. (The other side can and will verify.)
if f.Type != config.FolderTypeReceiveEncrypted {
if err := f.verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
}
}
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
err = f.withLimiter(func() error {
dstFd.mut.Lock()
defer dstFd.mut.Unlock()
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
})
} else {
err = f.limitedWriteAt(dstFd, buf, block.Offset)
}
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
}
if path == state.file.Name {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromElsewhere(block.Size)
}
return true
})
}
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
err = f.withLimiter(func() error {
dstFd.mut.Lock()
defer dstFd.mut.Unlock()
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
})
} else {
err = f.limitedWriteAt(dstFd, buf, block.Offset)
}
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
}
if path == state.file.Name {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromElsewhere(block.Size)
}
return true
})
if state.failed() != nil {
break
@@ -1421,58 +1392,11 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
state.copyDone(block)
}
}
if file != nil {
// os.File used to return invalid argument if nil.
// fs.File panics as it's an interface.
file.Close()
}
out <- state.sharedPullerState
}
}
func (f *sendReceiveFolder) initWeakHashFinder(state copyBlocksState) (*weakhash.Finder, fs.File) {
if f.Type == config.FolderTypeReceiveEncrypted {
l.Debugln("not weak hashing due to folder type", f.Type)
return nil, nil
}
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
}
if blocksPercentChanged < f.WeakHashThresholdPct {
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
return nil, nil
}
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
}
}
if len(hashesToFind) == 0 {
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
return nil, nil
}
file, err := f.mtimefs.Open(state.file.Name)
if err != nil {
l.Debugln("weak hasher", err)
return nil, nil
}
weakHashFinder, err := weakhash.NewFinder(f.ctx, file, state.file.BlockSize(), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
return nil, file
}
return weakHashFinder, file
}
func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) error {
if len(buf) != int(block.Size) {
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
@@ -1574,7 +1498,7 @@ loop:
activity.using(selected)
var buf []byte
blockNo := int(state.block.Offset / int64(state.file.BlockSize()))
buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
activity.done(selected)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, selected.ID.Short(), "returned error:", lastError)