felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: remove weak hashing which does not pull its weight (#10005)

Browse Source 
We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 13:21:10 +01:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/ur/usage_report.go
+6 -11
View File
@@ -116,8 +116,8 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
report.TotMiB = int(totBytes / 1024 / 1024)
report.FolderMaxMiB = int(maxBytes / 1024 / 1024)
report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false)
report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond)
report.HashPerf = report.SHA256Perf
report.MemorySize = int(memorySize() / 1024 / 1024)
report.NumCPU = runtime.NumCPU()
@@ -252,11 +252,6 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
if cfg.DisableTempIndexes {
report.FolderUsesV3.DisableTempIndexes++
}
if cfg.WeakHashThresholdPct < 0 {
report.FolderUsesV3.AlwaysWeakHash++
} else if cfg.WeakHashThresholdPct != 25 {
report.FolderUsesV3.CustomWeakHashThreshold++
}
if cfg.FSWatcherEnabled {
report.FolderUsesV3.FsWatcherEnabled++
}
@@ -443,7 +438,7 @@ var (
)
// CpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWeakHash bool) float64 {
func CpuBench(ctx context.Context, iterations int, duration time.Duration) float64 {
blocksResultMut.Lock()
defer blocksResultMut.Unlock()
@@ -454,7 +449,7 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
var perf float64
for i := 0; i < iterations; i++ {
if v := cpuBenchOnce(ctx, duration, useWeakHash, bs); v > perf {
if v := cpuBenchOnce(ctx, duration, bs); v > perf {
perf = v
}
}
@@ -467,13 +462,13 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
return perf
}
func cpuBenchOnce(ctx context.Context, duration time.Duration, useWeakHash bool, bs []byte) float64 {
func cpuBenchOnce(ctx context.Context, duration time.Duration, bs []byte) float64 {
t0 := time.Now()
b := 0
var err error
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil)
if err != nil {
return 0 // Context done
}