felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/sha256: Remove it (#9643)

Browse Source 
### Purpose

Remove the `lib/sha256` package, because it's no longer necessary. Go's
standard library now has the same performance and is on par with
`sha256-simd` since [Since Go
1.21](https://github.com/golang/go/commit/1a64574f42b95594cf9c8a12e9ca13d75585429c).
Therefore using `sha256-simd` has no benefits anymore.

ARM already has optimized sha256 assembly code since
https://github.com/golang/go/commit/7b8a7f8272fd1941a199af1adb334bd9996e8909,
`sha256-simd` published their results before that optimized assembly was
implemented,
https://github.com/minio/sha256-simd/commit/f941fedda826b68a196de2e0a9183e273ec0cb91.
The assembly looks very similar and the benchmarks in the Go commit
match that of `sha256-simd`.

This patch removes all of the related code of `lib/sha256` and makes
`crypto/sha256` the 'default'.

Benchmark of `sha256-simd` and `crypto/sha256`:
<details>

```
cpu: AMD Ryzen 5 3600X 6-Core Processor
                │  simd.txt   │               go.txt                │
                │   sec/op    │    sec/op     vs base               │
Hash/8Bytes-12    63.25n ± 1%    73.38n ± 1%  +16.02% (p=0.002 n=6)
Hash/64Bytes-12   98.73n ± 1%   105.30n ± 1%   +6.65% (p=0.002 n=6)
Hash/1K-12        567.2n ± 1%    572.8n ± 1%   +0.99% (p=0.002 n=6)
Hash/8K-12        4.062µ ± 1%    4.062µ ± 1%        ~ (p=0.396 n=6)
Hash/1M-12        512.1µ ± 0%    510.6µ ± 1%        ~ (p=0.485 n=6)
Hash/5M-12        2.556m ± 1%    2.564m ± 0%        ~ (p=0.093 n=6)
Hash/10M-12       5.112m ± 0%    5.127m ± 0%        ~ (p=0.093 n=6)
geomean           13.82µ         14.27µ        +3.28%

                │   simd.txt   │               go.txt                │
                │     B/s      │     B/s       vs base               │
Hash/8Bytes-12    120.6Mi ± 1%   104.0Mi ± 1%  -13.81% (p=0.002 n=6)
Hash/64Bytes-12   618.2Mi ± 1%   579.8Mi ± 1%   -6.22% (p=0.002 n=6)
Hash/1K-12        1.682Gi ± 1%   1.665Gi ± 1%   -0.98% (p=0.002 n=6)
Hash/8K-12        1.878Gi ± 1%   1.878Gi ± 1%        ~ (p=0.310 n=6)
Hash/1M-12        1.907Gi ± 0%   1.913Gi ± 1%        ~ (p=0.485 n=6)
Hash/5M-12        1.911Gi ± 1%   1.904Gi ± 0%        ~ (p=0.093 n=6)
Hash/10M-12       1.910Gi ± 0%   1.905Gi ± 0%        ~ (p=0.093 n=6)
geomean           1.066Gi        1.032Gi        -3.18%
```

</details>


### Testing

Compiled and tested on Linux.

### Documentation

https://github.com/syncthing/docs/pull/874
This commit is contained in:
Gusted
2024-08-10 12:58:20 +01:00
committed by GitHub
parent 19693734a3
commit 356c5055ad
26 changed files with 17 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/build/build.go
-1
View File
@@ -40,7 +40,6 @@ var (
envTags = []string{
"STGUIASSETS",
"STHASHING",
"STNORESTART",
"STNOUPGRADE",
}
lib/db/lowlevel.go
+1 -1
View File
@@ -9,6 +9,7 @@ package db
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"errors"
"fmt"
@@ -22,7 +23,6 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
lib/fs/tempname.go
+1 -1
View File
@@ -7,12 +7,12 @@
package fs
import (
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/sha256"
)
const (
lib/ignore/ignore.go
+1 -1
View File
@@ -9,6 +9,7 @@ package ignore
import (
"bufio"
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io"
@@ -21,7 +22,6 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore/ignoreresult"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
)
lib/model/folder_sendrecv.go
+1 -1
View File
@@ -9,6 +9,7 @@ package model
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@@ -28,7 +29,6 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
lib/protocol/bep_extensions.go
+1 -1
View File
@@ -4,6 +4,7 @@ package protocol
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
@@ -12,7 +13,6 @@ import (
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
)
const (
lib/protocol/deviceid.go
+1 -2
View File
@@ -4,13 +4,12 @@ package protocol
import (
"bytes"
"crypto/sha256"
"encoding/base32"
"encoding/binary"
"errors"
"fmt"
"strings"
"github.com/syncthing/syncthing/lib/sha256"
)
const (
lib/protocol/encryption.go
+1 -1
View File
@@ -8,6 +8,7 @@ package protocol
import (
"context"
"crypto/sha256"
"encoding/base32"
"encoding/binary"
"errors"
@@ -20,7 +21,6 @@ import (
lru "github.com/hashicorp/golang-lru/v2"
"github.com/miscreant/miscreant.go"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/crypto/chacha20poly1305"
"golang.org/x/crypto/hkdf"
"golang.org/x/crypto/scrypt"
lib/scanner/blocks.go
+1 -1
View File
@@ -9,12 +9,12 @@ package scanner
import (
"bytes"
"context"
"crypto/sha256"
"hash"
"hash/adler32"
"io"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
)
var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}
lib/scanner/blocks_test.go
+1 -1
View File
@@ -10,6 +10,7 @@ import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"fmt"
origAdler32 "hash/adler32"
mrand "math/rand"
@@ -18,7 +19,6 @@ import (
rollingAdler32 "github.com/chmduquesne/rollinghash/adler32"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
)
var blocksTestData = []struct {
lib/scanner/walk_test.go
+1 -1
View File
@@ -9,6 +9,7 @@ package scanner
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@@ -26,7 +27,6 @@ import (
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/text/unicode/norm"
)
lib/sha256/sha256.go
-157
View File
@@ -1,157 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package sha256
import (
cryptoSha256 "crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"math/rand"
"os"
"time"
minioSha256 "github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/logger"
)
var l = logger.DefaultLogger.NewFacility("sha256", "SHA256 hashing package")
const (
benchmarkingIterations = 3
benchmarkingDuration = 150 * time.Millisecond
defaultImpl = "crypto/sha256"
minioImpl = "minio/sha256-simd"
Size = cryptoSha256.Size
)
// May be switched out for another implementation
var (
New = cryptoSha256.New
Sum256 = cryptoSha256.Sum256
)
var (
selectedImpl = defaultImpl
cryptoPerf float64
minioPerf float64
)
func SelectAlgo() {
switch os.Getenv("STHASHING") {
case "":
// When unset, probe for the fastest implementation.
benchmark()
if minioPerf > cryptoPerf {
selectMinio()
}
case "minio":
// When set to "minio", use that.
selectMinio()
default:
// When set to anything else, such as "standard", use the default Go
// implementation. Make sure not to touch the minio
// implementation as it may be disabled for incompatibility reasons.
}
verifyCorrectness()
}
// Report prints a line with the measured hash performance rates for the
// selected and alternate implementation.
func Report() {
var otherImpl string
var selectedRate, otherRate float64
switch selectedImpl {
case defaultImpl:
selectedRate = cryptoPerf
otherRate = minioPerf
otherImpl = minioImpl
case minioImpl:
selectedRate = minioPerf
otherRate = cryptoPerf
otherImpl = defaultImpl
}
if selectedRate == 0 {
return
}
l.Infof("Single thread SHA256 performance is %s using %s (%s using %s).", formatRate(selectedRate), selectedImpl, formatRate(otherRate), otherImpl)
}
func selectMinio() {
New = minioSha256.New
Sum256 = minioSha256.Sum256
selectedImpl = minioImpl
}
func benchmark() {
// Interleave the tests to achieve some sort of fairness if the CPU is
// just in the process of spinning up to full speed.
for i := 0; i < benchmarkingIterations; i++ {
if perf := cpuBenchOnce(benchmarkingDuration, cryptoSha256.New); perf > cryptoPerf {
cryptoPerf = perf
}
if perf := cpuBenchOnce(benchmarkingDuration, minioSha256.New); perf > minioPerf {
minioPerf = perf
}
}
}
func cpuBenchOnce(duration time.Duration, newFn func() hash.Hash) float64 {
chunkSize := 100 * 1 << 10
h := newFn()
bs := make([]byte, chunkSize)
r := rand.New(rand.NewSource(time.Now().UnixNano()))
r.Read(bs)
t0 := time.Now()
b := 0
for time.Since(t0) < duration {
h.Write(bs)
b += chunkSize
}
h.Sum(nil)
d := time.Since(t0)
return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
}
func formatRate(rate float64) string {
decimals := 0
if rate < 1 {
decimals = 2
} else if rate < 10 {
decimals = 1
}
return fmt.Sprintf("%.*f MB/s", decimals, rate)
}
func verifyCorrectness() {
// The currently selected algo should in fact perform a SHA256 calculation.
// $ echo "Syncthing Magic Testing Value" | openssl dgst -sha256 -hex
correct := "87f6cfd24131724c6ec43495594c5c22abc7d2b86bcc134bc6f10b7ec3dda4ee"
input := "Syncthing Magic Testing Value\n"
h := New()
h.Write([]byte(input))
sum := hex.EncodeToString(h.Sum(nil))
if sum != correct {
panic("sha256 is broken")
}
arr := Sum256([]byte(input))
sum = hex.EncodeToString(arr[:])
if sum != correct {
panic("sha256 is broken")
}
}
lib/signature/signature.go
+1 -1
View File
@@ -11,6 +11,7 @@ package signature
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/sha256"
"crypto/x509"
"encoding/asn1"
"encoding/pem"
@@ -20,7 +21,6 @@ import (
"math/big"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
)
// GenerateKeys returns a new key pair, with the private and public key
lib/syncthing/syncthing.go
-6
View File
@@ -37,7 +37,6 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
@@ -153,11 +152,6 @@ func (a *App) startup() error {
l.SetPrefix(fmt.Sprintf("[%s] ", a.myID.String()[:5]))
l.Infoln("My ID:", a.myID)
// Select SHA256 implementation and report. Affected by the
// STHASHING environment variable.
sha256.SelectAlgo()
sha256.Report()
// Emit the Starting event, now that we know who we are.
a.evLogger.Log(events.Starting, map[string]string{