chore: remove weak hashing which does not pull its weight (#10005)

We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 13:21:10 +01:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions
+7 -26
View File
@@ -10,8 +10,6 @@ import (
"bytes"
"context"
"crypto/sha256"
"hash"
"hash/adler32"
"io"
"github.com/syncthing/syncthing/lib/protocol"
@@ -24,7 +22,7 @@ type Counter interface {
}
// Blocks returns the blockwise hash of the reader.
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
if counter == nil {
counter = &noopCounter{}
}
@@ -32,15 +30,6 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
hf := sha256.New()
const hashLength = sha256.Size
var weakHf hash.Hash32 = noopHash{}
var multiHf io.Writer = hf
if useWeakHashes {
// Use an actual weak hash function, make the multiHf
// write to both hash functions.
weakHf = adler32.New()
multiHf = io.MultiWriter(hf, weakHf)
}
var blocks []protocol.BlockInfo
var hashes, thisHash []byte
@@ -70,7 +59,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
}
lr.N = int64(blocksize)
n, err := io.CopyBuffer(multiHf, lr, buf)
n, err := io.CopyBuffer(hf, lr, buf)
if err != nil {
return nil, err
}
@@ -87,17 +76,15 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
b := protocol.BlockInfo{
Size: int(n),
Offset: offset,
Hash: thisHash,
WeakHash: weakHf.Sum32(),
Size: int(n),
Offset: offset,
Hash: thisHash,
}
blocks = append(blocks, b)
offset += n
hf.Reset()
weakHf.Reset()
}
if len(blocks) == 0 {
@@ -112,14 +99,8 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
return blocks, nil
}
// Validate quickly validates buf against the 32-bit weakHash, if not zero,
// else against the cryptohash hash, if len(hash)>0. It is satisfied if
// either hash matches or neither hash is given.
func Validate(buf, hash []byte, weakHash uint32) bool {
if weakHash != 0 && adler32.Checksum(buf) == weakHash {
return true
}
// Validate validates the hash, if len(hash)>0.
func Validate(buf, hash []byte) bool {
if len(hash) > 0 {
hbuf := sha256.Sum256(buf)
return bytes.Equal(hbuf[:], hash)