felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

fix: improve conflict detection by tracking previous file hash (fixes #10349) (#10351)

Browse Source 
This adds a new field to the file information we keep, the "previous
blocks hash". This is the hash of the file contents as it was in its
previous incarnation. That is, every scan that updates the blocks hash
will move the current hash to the "previous" field.

This enables an addition to the conflict detection algorithm: if the
file to be synced is in conflict with the current file on disk
(version-counter wise), but it indicates that it was based on the
precise contents we have (new.prevBlocksHash == current.blocksHash),
then it's not really a conflict.

Signed-off-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Jakob Borg
2025-09-13 16:16:28 +02:00
committed by GitHub
parent 1b0eaa093a
commit d037681fd1
8 changed files with 447 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/api/api.go
+17 -16
View File
@@ -1772,22 +1772,23 @@ func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
func fileIntfJSONMap(f protocol.FileInfo) map[string]interface{} {
out := map[string]interface{}{
"name": f.FileName(),
"type": f.FileType().String(),
"size": f.FileSize(),
"deleted": f.IsDeleted(),
"invalid": f.IsInvalid(),
"ignored": f.IsIgnored(),
"mustRescan": f.MustRescan(),
"noPermissions": !f.HasPermissionBits(),
"modified": f.ModTime(),
"modifiedBy": f.FileModifiedBy().String(),
"sequence": f.SequenceNo(),
"version": jsonVersionVector(f.FileVersion()),
"localFlags": f.FileLocalFlags(),
"platform": f.PlatformData(),
"inodeChange": f.InodeChangeTime(),
"blocksHash": f.FileBlocksHash(),
"name": f.FileName(),
"type": f.FileType().String(),
"size": f.FileSize(),
"deleted": f.IsDeleted(),
"invalid": f.IsInvalid(),
"ignored": f.IsIgnored(),
"mustRescan": f.MustRescan(),
"noPermissions": !f.HasPermissionBits(),
"modified": f.ModTime(),
"modifiedBy": f.FileModifiedBy().String(),
"sequence": f.SequenceNo(),
"version": jsonVersionVector(f.FileVersion()),
"localFlags": f.FileLocalFlags(),
"platform": f.PlatformData(),
"inodeChange": f.InodeChangeTime(),
"blocksHash": f.FileBlocksHash(),
"previousBlocksHash": f.PreviousBlocksHash,
}
if f.HasPermissionBits() {
out["permissions"] = fmt.Sprintf("%#o", f.FilePermissions())
lib/model/folder_sendrecv.go
+4 -20
View File
@@ -602,7 +602,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
}
// Remove it to replace with the dir.
if !curFile.IsSymlink() && f.inConflict(curFile.Version, file.Version) {
if !curFile.IsSymlink() && file.InConflictWith(curFile) {
// The new file has been changed in conflict with the existing one. We
// should file it away as a conflict instead of just removing or
// archiving.
@@ -800,7 +800,7 @@ func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, s
}
// Remove it to replace with the symlink. This also handles the
// "change symlink type" path.
if !curFile.IsDirectory() && !curFile.IsSymlink() && f.inConflict(curFile.Version, file.Version) {
if !curFile.IsDirectory() && !curFile.IsSymlink() && file.InConflictWith(curFile) {
// The new file has been changed in conflict with the existing one. We
// should file it away as a conflict instead of just removing or
// archiving.
@@ -923,7 +923,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
}
switch {
case f.inConflict(cur.Version, file.Version) && !cur.IsSymlink():
case file.InConflictWith(cur) && !cur.IsSymlink():
// If the delete constitutes winning a conflict, we move the file to
// a conflict copy instead of doing the delete
err = f.inWritableDir(func(name string) error {
@@ -1652,7 +1652,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
return fmt.Errorf("checking existing file: %w", err)
}
if !curFile.IsDirectory() && !curFile.IsSymlink() && f.inConflict(curFile.Version, file.Version) {
if !curFile.IsDirectory() && !curFile.IsSymlink() && file.InConflictWith(curFile) {
// The new file has been changed in conflict with the existing one. We
// should file it away as a conflict instead of just removing or
// archiving.
@@ -1847,22 +1847,6 @@ func (f *sendReceiveFolder) pullScannerRoutine(scanChan <-chan string) {
}
}
func (f *sendReceiveFolder) inConflict(current, replacement protocol.Vector) bool {
if current.Concurrent(replacement) {
// Obvious case
return true
}
if replacement.Counter(f.shortID) > current.Counter(f.shortID) {
// The replacement file contains a higher version for ourselves than
// what we have. This isn't supposed to be possible, since it's only
// we who can increment that counter. We take it as a sign that
// something is wrong (our index may have been corrupted or removed)
// and flag it as a conflict.
return true
}
return false
}
func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan chan<- string) error {
if isConflict(name) {
f.sl.Info("Conflict on existing conflict copy; not copying again", slogutil.FilePath(name))
lib/protocol/bep_fileinfo.go
+77 -61
View File
@@ -113,17 +113,18 @@ const (
)
type FileInfo struct {
Name string
Size int64
ModifiedS int64
ModifiedBy ShortID
Version Vector
Sequence int64
Blocks []BlockInfo
SymlinkTarget []byte
BlocksHash []byte
Encrypted []byte
Platform PlatformData
Name string
Size int64
ModifiedS int64
ModifiedBy ShortID
Version Vector
Sequence int64
Blocks []BlockInfo
SymlinkTarget []byte
BlocksHash []byte
PreviousBlocksHash []byte
Encrypted []byte
Platform PlatformData
Type FileInfoType
Permissions uint32
@@ -138,10 +139,6 @@ type FileInfo struct {
// the protocol.
LocalFlags FlagLocal
// The version_hash is an implementation detail and not part of the wire
// format.
VersionHash []byte
// The time when the inode was last changed (i.e., permissions, xattrs
// etc changed). This is host-local, not sent over the wire.
InodeChangeNs int64
@@ -165,34 +162,54 @@ func (f *FileInfo) ToWire(withInternalFields bool) *bep.FileInfo {
blocks[j] = b.ToWire()
}
w := &bep.FileInfo{
Name: f.Name,
Size: f.Size,
ModifiedS: f.ModifiedS,
ModifiedBy: uint64(f.ModifiedBy),
Version: f.Version.ToWire(),
Sequence: f.Sequence,
Blocks: blocks,
SymlinkTarget: f.SymlinkTarget,
BlocksHash: f.BlocksHash,
Encrypted: f.Encrypted,
Type: f.Type,
Permissions: f.Permissions,
ModifiedNs: f.ModifiedNs,
BlockSize: f.RawBlockSize,
Platform: f.Platform.toWire(),
Deleted: f.Deleted,
Invalid: f.IsInvalid(),
NoPermissions: f.NoPermissions,
Name: f.Name,
Size: f.Size,
ModifiedS: f.ModifiedS,
ModifiedBy: uint64(f.ModifiedBy),
Version: f.Version.ToWire(),
Sequence: f.Sequence,
Blocks: blocks,
SymlinkTarget: f.SymlinkTarget,
BlocksHash: f.BlocksHash,
PreviousBlocksHash: f.PreviousBlocksHash,
Encrypted: f.Encrypted,
Type: f.Type,
Permissions: f.Permissions,
ModifiedNs: f.ModifiedNs,
BlockSize: f.RawBlockSize,
Platform: f.Platform.toWire(),
Deleted: f.Deleted,
Invalid: f.IsInvalid(),
NoPermissions: f.NoPermissions,
}
if withInternalFields {
w.LocalFlags = uint32(f.LocalFlags)
w.VersionHash = f.VersionHash
w.InodeChangeNs = f.InodeChangeNs
w.EncryptionTrailerSize = int32(f.EncryptionTrailerSize)
}
return w
}
func (f *FileInfo) InConflictWith(previous FileInfo) bool {
if f.Version.GreaterEqual(previous.Version) {
// If the new file is strictly greater in the ordering than the
// existing file, it is not a conflict. If any counter has moved
// backwards, or different counters have increased independently,
// then the file is not greater but concurrent and we don't take
// this branch.
return false
}
if len(f.PreviousBlocksHash) == 0 || len(f.BlocksHash) == 0 {
// Don't have data to make a content determination, or the type has
// changed (file to directory, etc). Consider it a conflict.
return true
}
// If the new file is based on the old contents we have, it's not really
// a conflict.
return !bytes.Equal(f.PreviousBlocksHash, previous.BlocksHash)
}
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f *FileInfo) WinsConflict(other FileInfo) bool {
@@ -259,6 +276,7 @@ type FileInfoWithoutBlocks interface {
// GetBlocks() []*bep.BlockInfo // not included
GetSymlinkTarget() []byte
GetBlocksHash() []byte
GetPreviousBlocksHash() []byte
GetEncrypted() []byte
GetType() FileInfoType
GetPermissions() uint32
@@ -266,7 +284,6 @@ type FileInfoWithoutBlocks interface {
GetBlockSize() int32
GetPlatform() *bep.PlatformData
GetLocalFlags() uint32
GetVersionHash() []byte
GetInodeChangeNs() int64
GetEncryptionTrailerSize() int32
GetDeleted() bool
@@ -280,31 +297,31 @@ func fileInfoFromWireWithBlocks(w FileInfoWithoutBlocks, blocks []BlockInfo) Fil
localFlags = FlagLocalRemoteInvalid
}
return FileInfo{
Name: w.GetName(),
Size: w.GetSize(),
ModifiedS: w.GetModifiedS(),
ModifiedBy: ShortID(w.GetModifiedBy()),
Version: VectorFromWire(w.GetVersion()),
Sequence: w.GetSequence(),
Blocks: blocks,
SymlinkTarget: w.GetSymlinkTarget(),
BlocksHash: w.GetBlocksHash(),
Encrypted: w.GetEncrypted(),
Type: w.GetType(),
Permissions: w.GetPermissions(),
ModifiedNs: w.GetModifiedNs(),
RawBlockSize: w.GetBlockSize(),
Platform: platformDataFromWire(w.GetPlatform()),
Deleted: w.GetDeleted(),
LocalFlags: localFlags,
NoPermissions: w.GetNoPermissions(),
Name: w.GetName(),
Size: w.GetSize(),
ModifiedS: w.GetModifiedS(),
ModifiedBy: ShortID(w.GetModifiedBy()),
Version: VectorFromWire(w.GetVersion()),
Sequence: w.GetSequence(),
Blocks: blocks,
SymlinkTarget: w.GetSymlinkTarget(),
BlocksHash: w.GetBlocksHash(),
PreviousBlocksHash: w.GetPreviousBlocksHash(),
Encrypted: w.GetEncrypted(),
Type: w.GetType(),
Permissions: w.GetPermissions(),
ModifiedNs: w.GetModifiedNs(),
RawBlockSize: w.GetBlockSize(),
Platform: platformDataFromWire(w.GetPlatform()),
Deleted: w.GetDeleted(),
LocalFlags: localFlags,
NoPermissions: w.GetNoPermissions(),
}
}
func FileInfoFromDB(w *bep.FileInfo) FileInfo {
f := FileInfoFromWire(w)
f.LocalFlags = FlagLocal(w.LocalFlags)
f.VersionHash = w.VersionHash
f.InodeChangeNs = w.InodeChangeNs
f.EncryptionTrailerSize = int(w.EncryptionTrailerSize)
return f
@@ -313,7 +330,6 @@ func FileInfoFromDB(w *bep.FileInfo) FileInfo {
func FileInfoFromDBTruncated(w FileInfoWithoutBlocks) FileInfo {
f := fileInfoFromWireWithBlocks(w, nil)
f.LocalFlags = FlagLocal(w.GetLocalFlags())
f.VersionHash = w.GetVersionHash()
f.InodeChangeNs = w.GetInodeChangeNs()
f.EncryptionTrailerSize = int(w.GetEncryptionTrailerSize())
f.truncated = true
@@ -323,14 +339,14 @@ func FileInfoFromDBTruncated(w FileInfoWithoutBlocks) FileInfo {
func (f FileInfo) String() string {
switch f.Type {
case FileInfoTypeDirectory:
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.Platform, f.InodeChangeTime())
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.Platform, f.InodeChangeTime())
case FileInfoTypeFile:
return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d, NumBlocks:%d, BlocksHash:%x, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Size, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.RawBlockSize, len(f.Blocks), f.BlocksHash, f.Platform, f.InodeChangeTime())
return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d, NumBlocks:%d, BlocksHash:%x, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.RawBlockSize, len(f.Blocks), f.BlocksHash, f.Platform, f.InodeChangeTime())
case FileInfoTypeSymlink, FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile:
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Type, f.Sequence, f.Version, f.VersionHash, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.SymlinkTarget, f.Platform, f.InodeChangeTime())
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Type, f.Sequence, f.Version, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.SymlinkTarget, f.Platform, f.InodeChangeTime())
default:
panic("mystery file type detected")
}
lib/scanner/walk.go
+1
View File
@@ -655,6 +655,7 @@ func (w *walker) updateFileInfo(dst, src protocol.FileInfo) protocol.FileInfo {
dst.Version = src.Version.Update(w.ShortID)
dst.ModifiedBy = w.ShortID
dst.LocalFlags = w.LocalFlags
dst.PreviousBlocksHash = src.BlocksHash
// Copy OS data from src to dst, unless it was already set on dst.
dst.Platform.MergeWith(&src.Platform)