From 3b7a57d108807f9c6f5ad07a18ca9c231eaf38a6 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 2 Feb 2021 20:15:14 +0100 Subject: [PATCH] lib/protocol: Hide repeated data blocks in a given file (#7319) --- lib/protocol/encryption.go | 49 ++++++++++++++++++++++++--------- lib/protocol/encryption_test.go | 21 ++++++++++++-- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/lib/protocol/encryption.go b/lib/protocol/encryption.go index 597ec09ef..e4612cd04 100644 --- a/lib/protocol/encryption.go +++ b/lib/protocol/encryption.go @@ -11,7 +11,9 @@ import ( "crypto/rand" "crypto/sha256" "encoding/base32" + "encoding/binary" "errors" + "fmt" "io" "strings" "time" @@ -73,7 +75,7 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo, realName, err := decryptName(name, folderKey) if err != nil { - return nil, err + return nil, fmt.Errorf("decrypting name: %w", err) } realSize := size - blockOverhead realOffset := offset - int64(blockNo*blockOverhead) @@ -82,10 +84,23 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo, return nil, errors.New("short request") } - // Perform that request and grab the data. Explicitly zero out the - // hashes which are meaningless. + // Decrypt the block hash. - resp, err := e.model.Request(deviceID, folder, realName, blockNo, realSize, realOffset, nil, 0, false) + fileKey := FileKey(realName, folderKey) + var additional [8]byte + binary.BigEndian.PutUint64(additional[:], uint64(realOffset)) + realHash, err := decryptDeterministic(hash, fileKey, additional[:]) + if err != nil { + // "Legacy", no offset additional data? + realHash, err = decryptDeterministic(hash, fileKey, nil) + } + if err != nil { + return nil, fmt.Errorf("decrypting block hash: %w", err) + } + + // Perform that request and grab the data. + + resp, err := e.model.Request(deviceID, folder, realName, blockNo, realSize, realOffset, realHash, 0, false) if err != nil { return nil, err } @@ -102,7 +117,6 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo, } data = nd } - fileKey := FileKey(realName, folderKey) enc := encryptBytes(data, fileKey) resp.Close() return rawResponse{enc}, nil @@ -266,10 +280,19 @@ func encryptFileInfo(fi FileInfo, folderKey *[keySize]byte) FileInfo { b.Size = minPaddedSize } size := b.Size + blockOverhead + + // The offset goes into the encrypted block hash as additional data, + // essentially mixing in with the nonce. This means a block hash + // remains stable for the same data at the same offset, but doesn't + // reveal the existence of identical data blocks at other offsets. + var additional [8]byte + binary.BigEndian.PutUint64(additional[:], uint64(b.Offset)) + hash := encryptDeterministic(b.Hash, fileKey, additional[:]) + blocks[i] = BlockInfo{ + Hash: hash, Offset: offset, Size: size, - Hash: encryptDeterministic(b.Hash, fileKey), } offset += int64(size) } @@ -337,7 +360,7 @@ func DecryptFileInfo(fi FileInfo, folderKey *[keySize]byte) (FileInfo, error) { // result is always the same for any given string) and encodes it in a // filesystem-friendly manner. func encryptName(name string, key *[keySize]byte) string { - enc := encryptDeterministic([]byte(name), key) + enc := encryptDeterministic([]byte(name), key, nil) b32enc := base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(enc) return slashify(b32enc) } @@ -349,7 +372,7 @@ func decryptName(name string, key *[keySize]byte) (string, error) { if err != nil { return "", err } - dec, err := decryptDeterministic(bs, key) + dec, err := decryptDeterministic(bs, key, nil) if err != nil { return "", err } @@ -364,21 +387,21 @@ func encryptBytes(data []byte, key *[keySize]byte) []byte { } // encryptDeterministic encrypts bytes using AES-SIV -func encryptDeterministic(data []byte, key *[keySize]byte) []byte { +func encryptDeterministic(data []byte, key *[keySize]byte, additionalData []byte) []byte { aead, err := miscreant.NewAEAD(miscreantAlgo, key[:], 0) if err != nil { panic("cipher failure: " + err.Error()) } - return aead.Seal(nil, nil, data, nil) + return aead.Seal(nil, nil, data, additionalData) } // decryptDeterministic decrypts bytes using AES-SIV -func decryptDeterministic(data []byte, key *[keySize]byte) ([]byte, error) { +func decryptDeterministic(data []byte, key *[keySize]byte, additionalData []byte) ([]byte, error) { aead, err := miscreant.NewAEAD(miscreantAlgo, key[:], 0) if err != nil { panic("cipher failure: " + err.Error()) } - return aead.Open(nil, nil, data, nil) + return aead.Open(nil, nil, data, additionalData) } func encrypt(data []byte, nonce *[nonceSize]byte, key *[keySize]byte) []byte { @@ -471,7 +494,7 @@ func FileKey(filename string, folderKey *[keySize]byte) *[keySize]byte { } func PasswordToken(folderID, password string) []byte { - return encryptDeterministic(knownBytes(folderID), KeyFromPassword(folderID, password)) + return encryptDeterministic(knownBytes(folderID), KeyFromPassword(folderID, password), nil) } // slashify inserts slashes (and file extension) in the string to create an diff --git a/lib/protocol/encryption_test.go b/lib/protocol/encryption_test.go index 22c7830cf..f9c9d94ea 100644 --- a/lib/protocol/encryption_test.go +++ b/lib/protocol/encryption_test.go @@ -82,13 +82,30 @@ func TestEnDecryptFileInfo(t *testing.T) { ModifiedS: 8080, Blocks: []BlockInfo{ { - Size: 45, - Hash: []byte{1, 2, 3}, + Offset: 0, + Size: 45, + Hash: []byte{1, 2, 3}, + }, + { + Offset: 45, + Size: 45, + Hash: []byte{1, 2, 3}, }, }, } enc := encryptFileInfo(fi, &key) + if bytes.Equal(enc.Blocks[0].Hash, enc.Blocks[1].Hash) { + t.Error("block hashes should not repeat when on different offsets") + } + again := encryptFileInfo(fi, &key) + if !bytes.Equal(enc.Blocks[0].Hash, again.Blocks[0].Hash) { + t.Error("block hashes should remain stable (0)") + } + if !bytes.Equal(enc.Blocks[1].Hash, again.Blocks[1].Hash) { + t.Error("block hashes should remain stable (1)") + } + dec, err := DecryptFileInfo(enc, &key) if err != nil { t.Error(err)