lib/protocol: Hide repeated data blocks in a given file (#7319)

lib/protocol/encryption.go

@@ -11,7 +11,9 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base32"
+	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 	"strings"
 	"time"
@@ -73,7 +75,7 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo,
 
 	realName, err := decryptName(name, folderKey)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("decrypting name: %w", err)
 	}
 	realSize := size - blockOverhead
 	realOffset := offset - int64(blockNo*blockOverhead)
@@ -82,10 +84,23 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo,
 		return nil, errors.New("short request")
 	}
 
-	// Perform that request and grab the data. Explicitly zero out the
+	// Decrypt the block hash.
-	// hashes which are meaningless.
 
-	resp, err := e.model.Request(deviceID, folder, realName, blockNo, realSize, realOffset, nil, 0, false)
+	fileKey := FileKey(realName, folderKey)
+	var additional [8]byte
+	binary.BigEndian.PutUint64(additional[:], uint64(realOffset))
+	realHash, err := decryptDeterministic(hash, fileKey, additional[:])
+	if err != nil {
+		// "Legacy", no offset additional data?
+		realHash, err = decryptDeterministic(hash, fileKey, nil)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("decrypting block hash: %w", err)
+	}
+
+	// Perform that request and grab the data.
+
+	resp, err := e.model.Request(deviceID, folder, realName, blockNo, realSize, realOffset, realHash, 0, false)
 	if err != nil {
 		return nil, err
 	}
@@ -102,7 +117,6 @@ func (e encryptedModel) Request(deviceID DeviceID, folder, name string, blockNo,
 		}
 		data = nd
 	}
-	fileKey := FileKey(realName, folderKey)
 	enc := encryptBytes(data, fileKey)
 	resp.Close()
 	return rawResponse{enc}, nil
@@ -266,10 +280,19 @@ func encryptFileInfo(fi FileInfo, folderKey *[keySize]byte) FileInfo {
 			b.Size = minPaddedSize
 		}
 		size := b.Size + blockOverhead
+
+		// The offset goes into the encrypted block hash as additional data,
+		// essentially mixing in with the nonce. This means a block hash
+		// remains stable for the same data at the same offset, but doesn't
+		// reveal the existence of identical data blocks at other offsets.
+		var additional [8]byte
+		binary.BigEndian.PutUint64(additional[:], uint64(b.Offset))
+		hash := encryptDeterministic(b.Hash, fileKey, additional[:])
+
 		blocks[i] = BlockInfo{
+			Hash:   hash,
 			Offset: offset,
 			Size:   size,
-			Hash:   encryptDeterministic(b.Hash, fileKey),
 		}
 		offset += int64(size)
 	}
@@ -337,7 +360,7 @@ func DecryptFileInfo(fi FileInfo, folderKey *[keySize]byte) (FileInfo, error) {
 // result is always the same for any given string) and encodes it in a
 // filesystem-friendly manner.
 func encryptName(name string, key *[keySize]byte) string {
-	enc := encryptDeterministic([]byte(name), key)
+	enc := encryptDeterministic([]byte(name), key, nil)
 	b32enc := base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(enc)
 	return slashify(b32enc)
 }
@@ -349,7 +372,7 @@ func decryptName(name string, key *[keySize]byte) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	dec, err := decryptDeterministic(bs, key)
+	dec, err := decryptDeterministic(bs, key, nil)
 	if err != nil {
 		return "", err
 	}
@@ -364,21 +387,21 @@ func encryptBytes(data []byte, key *[keySize]byte) []byte {
 }
 
 // encryptDeterministic encrypts bytes using AES-SIV
-func encryptDeterministic(data []byte, key *[keySize]byte) []byte {
+func encryptDeterministic(data []byte, key *[keySize]byte, additionalData []byte) []byte {
 	aead, err := miscreant.NewAEAD(miscreantAlgo, key[:], 0)
 	if err != nil {
 		panic("cipher failure: " + err.Error())
 	}
-	return aead.Seal(nil, nil, data, nil)
+	return aead.Seal(nil, nil, data, additionalData)
 }
 
 // decryptDeterministic decrypts bytes using AES-SIV
-func decryptDeterministic(data []byte, key *[keySize]byte) ([]byte, error) {
+func decryptDeterministic(data []byte, key *[keySize]byte, additionalData []byte) ([]byte, error) {
 	aead, err := miscreant.NewAEAD(miscreantAlgo, key[:], 0)
 	if err != nil {
 		panic("cipher failure: " + err.Error())
 	}
-	return aead.Open(nil, nil, data, nil)
+	return aead.Open(nil, nil, data, additionalData)
 }
 
 func encrypt(data []byte, nonce *[nonceSize]byte, key *[keySize]byte) []byte {
@@ -471,7 +494,7 @@ func FileKey(filename string, folderKey *[keySize]byte) *[keySize]byte {
 }
 
 func PasswordToken(folderID, password string) []byte {
-	return encryptDeterministic(knownBytes(folderID), KeyFromPassword(folderID, password))
+	return encryptDeterministic(knownBytes(folderID), KeyFromPassword(folderID, password), nil)
 }
 
 // slashify inserts slashes (and file extension) in the string to create an

lib/protocol/encryption_test.go

@@ -82,13 +82,30 @@ func TestEnDecryptFileInfo(t *testing.T) {
 		ModifiedS:   8080,
 		Blocks: []BlockInfo{
 			{
-				Size: 45,
+				Offset: 0,
-				Hash: []byte{1, 2, 3},
+				Size:   45,
+				Hash:   []byte{1, 2, 3},
+			},
+			{
+				Offset: 45,
+				Size:   45,
+				Hash:   []byte{1, 2, 3},
 			},
 		},
 	}
 
 	enc := encryptFileInfo(fi, &key)
+	if bytes.Equal(enc.Blocks[0].Hash, enc.Blocks[1].Hash) {
+		t.Error("block hashes should not repeat when on different offsets")
+	}
+	again := encryptFileInfo(fi, &key)
+	if !bytes.Equal(enc.Blocks[0].Hash, again.Blocks[0].Hash) {
+		t.Error("block hashes should remain stable (0)")
+	}
+	if !bytes.Equal(enc.Blocks[1].Hash, again.Blocks[1].Hash) {
+		t.Error("block hashes should remain stable (1)")
+	}
+
 	dec, err := DecryptFileInfo(enc, &key)
 	if err != nil {
 		t.Error(err)