chore: remove weak hashing which does not pull its weight (#10005)
We've had weak/rolling hashing in the code for quite a while. It was a popular request for a while, based on the belief that rsync does this and we should too. However, the benefit is quite small; we save on average about 0.8% of transferred blocks over the population as a whole: <img width="974" alt="Screenshot 2025-03-28 at 17 09 02" src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2" /> This would be fine if the cost was comparably low, however the downside of attempting rolling hash matching is that we (by default) do a complete file read on the destination in order to look for matches before we starting pulling blocks for the file. For any larger file this means a sometimes long, I/O-intensive pause before the file starts syncing, for usually no benefit. I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
@@ -16,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
// HashFile hashes the files and returns a list of blocks representing the file.
|
||||
func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
|
||||
func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter) ([]protocol.BlockInfo, error) {
|
||||
fd, err := fs.Open(path)
|
||||
if err != nil {
|
||||
l.Debugln("open:", err)
|
||||
@@ -36,7 +36,7 @@ func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path strin
|
||||
|
||||
// Hash the file. This may take a while for large files.
|
||||
|
||||
blocks, err := Blocks(ctx, fd, blockSize, size, counter, useWeakHashes)
|
||||
blocks, err := Blocks(ctx, fd, blockSize, size, counter)
|
||||
if err != nil {
|
||||
l.Debugln("blocks:", err)
|
||||
return nil, err
|
||||
@@ -108,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
|
||||
panic("Bug. Asked to hash a directory or a deleted file.")
|
||||
}
|
||||
|
||||
blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter, true)
|
||||
blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter)
|
||||
if err != nil {
|
||||
handleError(ctx, "hashing", f.Name, err, ph.outbox)
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user