chore: remove weak hashing which does not pull its weight (#10005)

We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 13:21:10 +01:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions
+3 -9
View File
@@ -436,11 +436,8 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
}
f := checkReceived(<-received)
if f.Blocks[0].WeakHash != 103547413 {
t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
}
res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash})
res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash})
if err != nil {
t.Fatal(err)
}
@@ -454,17 +451,14 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
writeFile(t, tfs, "foo", payload)
_, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash})
_, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash})
if err == nil {
t.Fatalf("expected failure")
}
select {
case fs := <-received:
f := checkReceived(fs)
if f.Blocks[0].WeakHash != 41943361 {
t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
}
checkReceived(fs)
case <-time.After(time.Second):
t.Fatalf("timed out")
}