chore: remove weak hashing which does not pull its weight (#10005)
We've had weak/rolling hashing in the code for quite a while. It was a popular request for a while, based on the belief that rsync does this and we should too. However, the benefit is quite small; we save on average about 0.8% of transferred blocks over the population as a whole: <img width="974" alt="Screenshot 2025-03-28 at 17 09 02" src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2" /> This would be fine if the cost was comparably low, however the downside of attempting rolling hash matching is that we (by default) do a complete file read on the destination in order to look for matches before we starting pulling blocks for the file. For any larger file this means a sometimes long, I/O-intensive pause before the file starts syncing, for usually no benefit. I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
+7
-26
@@ -10,8 +10,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"hash/adler32"
|
||||
"io"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
@@ -24,7 +22,7 @@ type Counter interface {
|
||||
}
|
||||
|
||||
// Blocks returns the blockwise hash of the reader.
|
||||
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
|
||||
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
|
||||
if counter == nil {
|
||||
counter = &noopCounter{}
|
||||
}
|
||||
@@ -32,15 +30,6 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
hf := sha256.New()
|
||||
const hashLength = sha256.Size
|
||||
|
||||
var weakHf hash.Hash32 = noopHash{}
|
||||
var multiHf io.Writer = hf
|
||||
if useWeakHashes {
|
||||
// Use an actual weak hash function, make the multiHf
|
||||
// write to both hash functions.
|
||||
weakHf = adler32.New()
|
||||
multiHf = io.MultiWriter(hf, weakHf)
|
||||
}
|
||||
|
||||
var blocks []protocol.BlockInfo
|
||||
var hashes, thisHash []byte
|
||||
|
||||
@@ -70,7 +59,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
}
|
||||
|
||||
lr.N = int64(blocksize)
|
||||
n, err := io.CopyBuffer(multiHf, lr, buf)
|
||||
n, err := io.CopyBuffer(hf, lr, buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -87,17 +76,15 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
|
||||
|
||||
b := protocol.BlockInfo{
|
||||
Size: int(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
WeakHash: weakHf.Sum32(),
|
||||
Size: int(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
}
|
||||
|
||||
blocks = append(blocks, b)
|
||||
offset += n
|
||||
|
||||
hf.Reset()
|
||||
weakHf.Reset()
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
@@ -112,14 +99,8 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// Validate quickly validates buf against the 32-bit weakHash, if not zero,
|
||||
// else against the cryptohash hash, if len(hash)>0. It is satisfied if
|
||||
// either hash matches or neither hash is given.
|
||||
func Validate(buf, hash []byte, weakHash uint32) bool {
|
||||
if weakHash != 0 && adler32.Checksum(buf) == weakHash {
|
||||
return true
|
||||
}
|
||||
|
||||
// Validate validates the hash, if len(hash)>0.
|
||||
func Validate(buf, hash []byte) bool {
|
||||
if len(hash) > 0 {
|
||||
hbuf := sha256.Sum256(buf)
|
||||
return bytes.Equal(hbuf[:], hash)
|
||||
|
||||
Reference in New Issue
Block a user