felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/protocol: Cache expensive key operations (fixes #8599) (#8820)

Browse Source 
This adds a cache to the expensive key generation operations. It's fixes
size LRU/MRU stuff to keep memory usage bounded under absurd conditions.

Also closes #8600.
This commit is contained in:
Jakob Borg
2023-03-12 20:06:59 +01:00
committed by GitHub
parent 3ffe859fe8
commit 466b56ded1
12 changed files with 212 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/protocol/encryption.go
+89 -26
View File
@@ -17,6 +17,7 @@ import (
"sync"
"github.com/gogo/protobuf/proto"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/miscreant/miscreant.go"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
@@ -34,6 +35,8 @@ const (
maxPathComponent = 200 // characters
encryptedDirExtension = ".syncthing-enc" // for top level dirs
miscreantAlgo = "AES-SIV"
folderKeyCacheEntries = 1000
fileKeyCacheEntries = 5000
)
// The encryptedModel sits between the encrypted device and the model. It
@@ -42,12 +45,21 @@ const (
type encryptedModel struct {
model Model
folderKeys *folderKeyRegistry
keyGen *KeyGenerator
}
func newEncryptedModel(model Model, folderKeys *folderKeyRegistry, keyGen *KeyGenerator) encryptedModel {
return encryptedModel{
model: model,
folderKeys: folderKeys,
keyGen: keyGen,
}
}
func (e encryptedModel) Index(deviceID DeviceID, folder string, files []FileInfo) error {
if folderKey, ok := e.folderKeys.get(folder); ok {
// incoming index data to be decrypted
if err := decryptFileInfos(files, folderKey); err != nil {
if err := decryptFileInfos(e.keyGen, files, folderKey); err != nil {
return err
}
}
@@ -57,7 +69,7 @@ func (e encryptedModel) Index(deviceID DeviceID, folder string, files []FileInfo
func (e encryptedModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) error {
if folderKey, ok := e.folderKeys.get(folder); ok {
// incoming index data to be decrypted
if err := decryptFileInfos(files, folderKey); err != nil {
if err := decryptFileInfos(e.keyGen, files, folderKey); err != nil {
return err
}
}
@@ -86,7 +98,7 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo,
// Decrypt the block hash.
fileKey := FileKey(realName, folderKey)
fileKey := e.keyGen.FileKey(realName, folderKey)
var additional [8]byte
binary.BigEndian.PutUint64(additional[:], uint64(realOffset))
realHash, err := decryptDeterministic(hash, fileKey, additional[:])
@@ -145,6 +157,16 @@ type encryptedConnection struct {
ConnectionInfo
conn *rawConnection
folderKeys *folderKeyRegistry
keyGen *KeyGenerator
}
func newEncryptedConnection(ci ConnectionInfo, conn *rawConnection, folderKeys *folderKeyRegistry, keyGen *KeyGenerator) encryptedConnection {
return encryptedConnection{
ConnectionInfo: ci,
conn: conn,
folderKeys: folderKeys,
keyGen: keyGen,
}
}
func (e encryptedConnection) Start() {
@@ -161,14 +183,14 @@ func (e encryptedConnection) ID() DeviceID {
func (e encryptedConnection) Index(ctx context.Context, folder string, files []FileInfo) error {
if folderKey, ok := e.folderKeys.get(folder); ok {
encryptFileInfos(files, folderKey)
encryptFileInfos(e.keyGen, files, folderKey)
}
return e.conn.Index(ctx, folder, files)
}
func (e encryptedConnection) IndexUpdate(ctx context.Context, folder string, files []FileInfo) error {
if folderKey, ok := e.folderKeys.get(folder); ok {
encryptFileInfos(files, folderKey)
encryptFileInfos(e.keyGen, files, folderKey)
}
return e.conn.IndexUpdate(ctx, folder, files)
}
@@ -200,7 +222,7 @@ func (e encryptedConnection) Request(ctx context.Context, folder string, name st
// Return the decrypted block (or an error if it fails decryption)
fileKey := FileKey(name, folderKey)
fileKey := e.keyGen.FileKey(name, folderKey)
bs, err = DecryptBytes(bs, fileKey)
if err != nil {
return nil, err
@@ -232,16 +254,16 @@ func (e encryptedConnection) Statistics() Statistics {
return e.conn.Statistics()
}
func encryptFileInfos(files []FileInfo, folderKey *[keySize]byte) {
func encryptFileInfos(keyGen *KeyGenerator, files []FileInfo, folderKey *[keySize]byte) {
for i, fi := range files {
files[i] = encryptFileInfo(fi, folderKey)
files[i] = encryptFileInfo(keyGen, fi, folderKey)
}
}
// encryptFileInfo encrypts a FileInfo and wraps it into a new fake FileInfo
// with an encrypted name.
func encryptFileInfo(fi FileInfo, folderKey *[keySize]byte) FileInfo {
fileKey := FileKey(fi.Name, folderKey)
func encryptFileInfo(keyGen *KeyGenerator, fi FileInfo, folderKey *[keySize]byte) FileInfo {
fileKey := keyGen.FileKey(fi.Name, folderKey)
// The entire FileInfo is encrypted with a random nonce, and concatenated
// with that nonce.
@@ -319,7 +341,7 @@ func encryptFileInfo(fi FileInfo, folderKey *[keySize]byte) FileInfo {
enc := FileInfo{
Name: encryptName(fi.Name, folderKey),
Type: typ,
Permissions: 0644,
Permissions: 0o644,
ModifiedS: 1234567890, // Sat Feb 14 00:31:30 CET 2009
Deleted: fi.Deleted,
RawInvalid: fi.IsInvalid(),
@@ -336,9 +358,9 @@ func encryptFileInfo(fi FileInfo, folderKey *[keySize]byte) FileInfo {
return enc
}
func decryptFileInfos(files []FileInfo, folderKey *[keySize]byte) error {
func decryptFileInfos(keyGen *KeyGenerator, files []FileInfo, folderKey *[keySize]byte) error {
for i, fi := range files {
decFI, err := DecryptFileInfo(fi, folderKey)
decFI, err := DecryptFileInfo(keyGen, fi, folderKey)
if err != nil {
return err
}
@@ -349,13 +371,13 @@ func decryptFileInfos(files []FileInfo, folderKey *[keySize]byte) error {
// DecryptFileInfo extracts the encrypted portion of a FileInfo, decrypts it
// and returns that.
func DecryptFileInfo(fi FileInfo, folderKey *[keySize]byte) (FileInfo, error) {
func DecryptFileInfo(keyGen *KeyGenerator, fi FileInfo, folderKey *[keySize]byte) (FileInfo, error) {
realName, err := decryptName(fi.Name, folderKey)
if err != nil {
return FileInfo{}, err
}
fileKey := FileKey(realName, folderKey)
fileKey := keyGen.FileKey(realName, folderKey)
dec, err := DecryptBytes(fi.Encrypted, fileKey)
if err != nil {
return FileInfo{}, err
@@ -476,10 +498,10 @@ func randomNonce() *[nonceSize]byte {
// keysFromPasswords converts a set of folder ID to password into a set of
// folder ID to encryption key, using our key derivation function.
func keysFromPasswords(passwords map[string]string) map[string]*[keySize]byte {
func keysFromPasswords(keyGen *KeyGenerator, passwords map[string]string) map[string]*[keySize]byte {
res := make(map[string]*[keySize]byte, len(passwords))
for folder, password := range passwords {
res[folder] = KeyFromPassword(folder, password)
res[folder] = keyGen.KeyFromPassword(folder, password)
}
return res
}
@@ -488,9 +510,35 @@ func knownBytes(folderID string) []byte {
return []byte("syncthing" + folderID)
}
type KeyGenerator struct {
mut sync.Mutex
folderKeys *lru.TwoQueueCache[folderKeyCacheKey, *[keySize]byte]
fileKeys *lru.TwoQueueCache[fileKeyCacheKey, *[keySize]byte]
}
func NewKeyGenerator() *KeyGenerator {
folderKeys, _ := lru.New2Q[folderKeyCacheKey, *[keySize]byte](folderKeyCacheEntries)
fileKeys, _ := lru.New2Q[fileKeyCacheKey, *[keySize]byte](fileKeyCacheEntries)
return &KeyGenerator{
folderKeys: folderKeys,
fileKeys: fileKeys,
}
}
type folderKeyCacheKey struct {
folderID string
password string
}
// KeyFromPassword uses key derivation to generate a stronger key from a
// probably weak password.
func KeyFromPassword(folderID, password string) *[keySize]byte {
func (g *KeyGenerator) KeyFromPassword(folderID, password string) *[keySize]byte {
cacheKey := folderKeyCacheKey{folderID, password}
g.mut.Lock()
defer g.mut.Unlock()
if key, ok := g.folderKeys.Get(cacheKey); ok {
return key
}
bs, err := scrypt.Key([]byte(password), knownBytes(folderID), 32768, 8, 1, keySize)
if err != nil {
panic("key derivation failure: " + err.Error())
@@ -500,23 +548,36 @@ func KeyFromPassword(folderID, password string) *[keySize]byte {
}
var key [keySize]byte
copy(key[:], bs)
g.folderKeys.Add(cacheKey, &key)
return &key
}
var hkdfSalt = []byte("syncthing")
func FileKey(filename string, folderKey *[keySize]byte) *[keySize]byte {
type fileKeyCacheKey struct {
file string
key [keySize]byte
}
func (g *KeyGenerator) FileKey(filename string, folderKey *[keySize]byte) *[keySize]byte {
g.mut.Lock()
defer g.mut.Unlock()
cacheKey := fileKeyCacheKey{filename, *folderKey}
if key, ok := g.fileKeys.Get(cacheKey); ok {
return key
}
kdf := hkdf.New(sha256.New, append(folderKey[:], filename...), hkdfSalt, nil)
var fileKey [keySize]byte
n, err := io.ReadFull(kdf, fileKey[:])
if err != nil || n != keySize {
panic("hkdf failure")
}
g.fileKeys.Add(cacheKey, &fileKey)
return &fileKey
}
func PasswordToken(folderID, password string) []byte {
return encryptDeterministic(knownBytes(folderID), KeyFromPassword(folderID, password), nil)
func PasswordToken(keyGen *KeyGenerator, folderID, password string) []byte {
return encryptDeterministic(knownBytes(folderID), keyGen.KeyFromPassword(folderID, password), nil)
}
// slashify inserts slashes (and file extension) in the string to create an
@@ -593,13 +654,15 @@ func IsEncryptedParent(pathComponents []string) bool {
}
type folderKeyRegistry struct {
keys map[string]*[keySize]byte // folder ID -> key
mut sync.RWMutex
keyGen *KeyGenerator
keys map[string]*[keySize]byte // folder ID -> key
mut sync.RWMutex
}
func newFolderKeyRegistry(passwords map[string]string) *folderKeyRegistry {
func newFolderKeyRegistry(keyGen *KeyGenerator, passwords map[string]string) *folderKeyRegistry {
return &folderKeyRegistry{
keys: keysFromPasswords(passwords),
keyGen: keyGen,
keys: keysFromPasswords(keyGen, passwords),
}
}
@@ -612,6 +675,6 @@ func (r *folderKeyRegistry) get(folder string) (*[keySize]byte, bool) {
func (r *folderKeyRegistry) setPasswords(passwords map[string]string) {
r.mut.Lock()
r.keys = keysFromPasswords(passwords)
r.keys = keysFromPasswords(r.keyGen, passwords)
r.mut.Unlock()
}