chore: remove weak hashing which does not pull its weight (#10005)
We've had weak/rolling hashing in the code for quite a while. It was a popular request for a while, based on the belief that rsync does this and we should too. However, the benefit is quite small; we save on average about 0.8% of transferred blocks over the population as a whole: <img width="974" alt="Screenshot 2025-03-28 at 17 09 02" src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2" /> This would be fine if the cost was comparably low, however the downside of attempting rolling hash matching is that we (by default) do a complete file read on the destination in order to look for matches before we starting pulling blocks for the file. For any larger file this means a sometimes long, I/O-intensive pause before the file starts syncing, for usually no benefit. I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
@@ -544,32 +544,29 @@ func (f *FileInfo) setNoContent() {
|
||||
}
|
||||
|
||||
type BlockInfo struct {
|
||||
Hash []byte
|
||||
Offset int64
|
||||
Size int
|
||||
WeakHash uint32
|
||||
Hash []byte
|
||||
Offset int64
|
||||
Size int
|
||||
}
|
||||
|
||||
func (b BlockInfo) ToWire() *bep.BlockInfo {
|
||||
return &bep.BlockInfo{
|
||||
Hash: b.Hash,
|
||||
Offset: b.Offset,
|
||||
Size: int32(b.Size),
|
||||
WeakHash: b.WeakHash,
|
||||
Hash: b.Hash,
|
||||
Offset: b.Offset,
|
||||
Size: int32(b.Size),
|
||||
}
|
||||
}
|
||||
|
||||
func BlockInfoFromWire(w *bep.BlockInfo) BlockInfo {
|
||||
return BlockInfo{
|
||||
Hash: w.Hash,
|
||||
Offset: w.Offset,
|
||||
Size: int(w.Size),
|
||||
WeakHash: w.WeakHash,
|
||||
Hash: w.Hash,
|
||||
Offset: w.Offset,
|
||||
Size: int(w.Size),
|
||||
}
|
||||
}
|
||||
|
||||
func (b BlockInfo) String() string {
|
||||
return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash)
|
||||
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
|
||||
}
|
||||
|
||||
// For each block size, the hash of a block of all zeroes
|
||||
@@ -596,7 +593,6 @@ func BlocksHash(bs []BlockInfo) []byte {
|
||||
h := sha256.New()
|
||||
for _, b := range bs {
|
||||
_, _ = h.Write(b.Hash)
|
||||
_ = binary.Write(h, binary.BigEndian, b.WeakHash)
|
||||
}
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@ type Request struct {
|
||||
Size int
|
||||
Hash []byte
|
||||
FromTemporary bool
|
||||
WeakHash uint32
|
||||
BlockNo int
|
||||
}
|
||||
|
||||
@@ -38,7 +37,6 @@ func (r *Request) toWire() *bep.Request {
|
||||
Size: int32(r.Size),
|
||||
Hash: r.Hash,
|
||||
FromTemporary: r.FromTemporary,
|
||||
WeakHash: r.WeakHash,
|
||||
BlockNo: int32(r.BlockNo),
|
||||
}
|
||||
}
|
||||
@@ -52,7 +50,6 @@ func requestFromWire(w *bep.Request) *Request {
|
||||
Size: int(w.Size),
|
||||
Hash: w.Hash,
|
||||
FromTemporary: w.FromTemporary,
|
||||
WeakHash: w.WeakHash,
|
||||
BlockNo: int(w.BlockNo),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@ type TestModel struct {
|
||||
offset int64
|
||||
size int32
|
||||
hash []byte
|
||||
weakHash uint32
|
||||
fromTemporary bool
|
||||
indexFn func(string, []FileInfo)
|
||||
ccFn func(*ClusterConfig)
|
||||
@@ -48,7 +47,6 @@ func (t *TestModel) Request(_ Connection, req *Request) (RequestResponse, error)
|
||||
t.offset = req.Offset
|
||||
t.size = int32(req.Size)
|
||||
t.hash = req.Hash
|
||||
t.weakHash = req.WeakHash
|
||||
t.fromTemporary = req.FromTemporary
|
||||
buf := make([]byte, len(t.data))
|
||||
copy(buf, t.data)
|
||||
|
||||
Reference in New Issue
Block a user