felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

chore: remove weak hashing which does not pull its weight (#10005)

Browse Source 
We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 13:21:10 +01:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/ur/contract/contract.go
+6 -12
View File
@@ -111,8 +111,6 @@ type Report struct {
ConflictsOther int `json:"conflictsOther,omitempty" metric:"folder_feature{feature=ConflictsOther},summary" since:"3"`
DisableSparseFiles int `json:"disableSparseFiles,omitempty" metric:"folder_feature{feature=DisableSparseFiles},summary" since:"3"`
DisableTempIndexes int `json:"disableTempIndexes,omitempty" metric:"folder_feature{feature=DisableTempIndexes},summary" since:"3"`
AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" metric:"folder_feature{feature=AlwaysWeakhash},summary" since:"3"`
CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" metric:"folder_feature{feature=CustomWeakhashThreshold},summary" since:"3"`
FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" metric:"folder_feature{feature=FSWatcherEnabled},summary" since:"3"`
PullOrder map[string]int `json:"pullOrder,omitempty" metric:"folder_pull_order,summaryVec:order" since:"3"`
FilesystemType map[string]int `json:"filesystemType,omitempty" metric:"folder_file_system_type,summaryVec:type" since:"3"`
@@ -152,13 +150,12 @@ type Report struct {
} `json:"guiStats,omitempty" since:"3"`
BlockStats struct {
Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"`
Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"`
Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"`
Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"`
CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"`
CopyOriginShifted int `json:"copyOriginShifted,omitempty" metric:"blocks_processed{source=copy_origin_shifted},gauge" since:"3"`
CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"`
Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"`
Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"`
Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"`
Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"`
CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"`
CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"`
} `json:"blockStats,omitempty" since:"3"`
TransportStats map[string]int `json:"transportStats,omitempty" since:"3"`
@@ -175,9 +172,6 @@ type Report struct {
Stars int `json:"stars,omitempty" metric:"folder_ignore_lines{kind=stars},summary" since:"3"`
} `json:"ignoreStats,omitempty" since:"3"`
// V3 fields added late in the RC
WeakHashEnabled bool `json:"weakHashEnabled,omitempty" metric:"-" since:"3"` // Deprecated and not provided client-side anymore
// Added in post processing
Received time.Time `json:"received,omitempty"`
Date string `json:"date,omitempty"`
lib/ur/usage_report.go
+6 -11
View File
@@ -116,8 +116,8 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
report.TotMiB = int(totBytes / 1024 / 1024)
report.FolderMaxMiB = int(maxBytes / 1024 / 1024)
report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false)
report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond)
report.HashPerf = report.SHA256Perf
report.MemorySize = int(memorySize() / 1024 / 1024)
report.NumCPU = runtime.NumCPU()
@@ -252,11 +252,6 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
if cfg.DisableTempIndexes {
report.FolderUsesV3.DisableTempIndexes++
}
if cfg.WeakHashThresholdPct < 0 {
report.FolderUsesV3.AlwaysWeakHash++
} else if cfg.WeakHashThresholdPct != 25 {
report.FolderUsesV3.CustomWeakHashThreshold++
}
if cfg.FSWatcherEnabled {
report.FolderUsesV3.FsWatcherEnabled++
}
@@ -443,7 +438,7 @@ var (
)
// CpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWeakHash bool) float64 {
func CpuBench(ctx context.Context, iterations int, duration time.Duration) float64 {
blocksResultMut.Lock()
defer blocksResultMut.Unlock()
@@ -454,7 +449,7 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
var perf float64
for i := 0; i < iterations; i++ {
if v := cpuBenchOnce(ctx, duration, useWeakHash, bs); v > perf {
if v := cpuBenchOnce(ctx, duration, bs); v > perf {
perf = v
}
}
@@ -467,13 +462,13 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
return perf
}
func cpuBenchOnce(ctx context.Context, duration time.Duration, useWeakHash bool, bs []byte) float64 {
func cpuBenchOnce(ctx context.Context, duration time.Duration, bs []byte) float64 {
t0 := time.Now()
b := 0
var err error
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil)
if err != nil {
return 0 // Context done
}