felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: remove weak hashing which does not pull its weight (#10005)

Browse Source 
We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 13:21:10 +01:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/model.go
+8 -14
View File
@@ -118,7 +118,7 @@ type Model interface {
GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error)
RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error)
RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
}
type model struct {
@@ -606,8 +606,6 @@ func (m *model) UsageReportingStats(report *contract.Report, version int, previe
report.BlockStats.Pulled = v
case "copyOrigin":
report.BlockStats.CopyOrigin = v
case "copyOriginShifted":
report.BlockStats.CopyOriginShifted = v
case "copyElsewhere":
report.BlockStats.CopyElsewhere = v
}
@@ -2036,7 +2034,7 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
return nil, protocol.ErrNoSuchFile
}
_, err := readOffsetIntoBuf(folderFs, tempFn, req.Offset, res.data)
if err == nil && scanner.Validate(res.data, req.Hash, req.WeakHash) {
if err == nil && scanner.Validate(res.data, req.Hash) {
return res, nil
}
// Fall through to reading from a non-temp file, just in case the temp
@@ -2065,8 +2063,8 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
return nil, protocol.ErrGeneric
}
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash, req.WeakHash) {
m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash)
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash) {
m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash)
l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID.Short(), req.Folder, req.Name, req.Offset, req.Size)
return nil, protocol.ErrNoSuchFile
}
@@ -2092,7 +2090,7 @@ func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requ
return res
}
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32) {
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte) {
cf, ok, err := m.CurrentFolderFile(folder, name)
if err != nil {
l.Debugf("%v recheckFile: %s: %q / %q: current file error: %v", m, deviceID, folder, name, err)
@@ -2121,10 +2119,6 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, off
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
return
}
if weakHash != 0 && block.WeakHash != weakHash {
l.Debugf("%v recheckFile: %s: %q / %q i=%d: weak hash mismatch %v != %v", m, deviceID, folder, name, blockIndex, block.WeakHash, weakHash)
return
}
// The hashes provided part of the request match what we expect to find according
// to what we have in the database, yet the content we've read off the filesystem doesn't
@@ -2462,14 +2456,14 @@ func (m *model) deviceDidCloseRLocked(deviceID protocol.DeviceID, duration time.
}
}
func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
conn, connOK := m.requestConnectionForDevice(deviceID)
if !connOK {
return nil, fmt.Errorf("requestGlobal: no connection to device: %s", deviceID.Short())
}
l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, WeakHash: weakHash, FromTemporary: fromTemporary})
l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, fromTemporary)
return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, FromTemporary: fromTemporary})
}
// requestConnectionForDevice returns a connection to the given device, to