felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

fix: track invalid files in LocalFlags to fix global count (#10170)

Browse Source 
Move the "invalid" bit to a local flag, making it easier to track in counts etc.
This commit is contained in:
Simon Frei
2025-06-13 07:33:31 +02:00
committed by GitHub
parent cb7cea93a2
commit 7b319111d3
19 changed files with 132 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/fakeconns_test.go
+1 -1
View File
@@ -74,7 +74,7 @@ func (f *fakeConnection) DownloadProgress(_ context.Context, dp *protocol.Downlo
})
}
func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags uint32) {
func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags protocol.FlagLocal) {
blockSize := protocol.BlockSize(int64(len(data)))
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil)
lib/model/folder.go
+1 -1
View File
@@ -44,7 +44,7 @@ type folder struct {
*stats.FolderStatisticsReference
ioLimiter *semaphore.Semaphore
localFlags uint32
localFlags protocol.FlagLocal
model *model
shortID protocol.ShortID
lib/model/indexhandler.go
-2
View File
@@ -481,8 +481,6 @@ func (s *indexHandler) logSequenceAnomaly(msg string, extra map[string]any) {
}
func prepareFileInfoForIndex(f protocol.FileInfo) protocol.FileInfo {
// Mark the file as invalid if any of the local bad stuff flags are set.
f.RawInvalid = f.IsInvalid()
// If the file is marked LocalReceive (i.e., changed locally on a
// receive only folder) we do not want it to ever become the
// globally best version, invalid or not.
lib/model/model_test.go
+2 -2
View File
@@ -3609,7 +3609,7 @@ func TestIssue6961(t *testing.T) {
// Remote, valid and existing file
must(t, m.Index(conn1, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, Version: version, Sequence: 1}}}))
// Remote, invalid (receive-only) and existing file
must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, RawInvalid: true, Sequence: 1}}}))
must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, LocalFlags: protocol.FlagLocalRemoteInvalid, Sequence: 1}}}))
// Create a local file
if fd, err := tfs.OpenFile(name, fs.OptCreate, 0o666); err != nil {
t.Fatal(err)
@@ -3635,7 +3635,7 @@ func TestIssue6961(t *testing.T) {
m.ScanFolders()
// Drop the remote index, add some other file.
must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: "bar", RawInvalid: true, Sequence: 1}}}))
must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: "bar", LocalFlags: protocol.FlagLocalRemoteInvalid, Sequence: 1}}}))
// Pause and unpause folder to create new db.FileSet and thus recalculate everything
pauseFolder(t, wcfg, fcfg.ID, true)
lib/protocol/bep_fileinfo.go
+35 -24
View File
@@ -17,26 +17,33 @@ import (
"github.com/syncthing/syncthing/lib/build"
)
type FlagLocal uint32
// FileInfo.LocalFlags flags
const (
FlagLocalUnsupported = 1 << 0 // 1: The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored = 1 << 1 // 2: Matches local ignore patterns
FlagLocalMustRescan = 1 << 2 // 4: Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly = 1 << 3 // 8: Change detected on receive only folder
FlagLocalGlobal = 1 << 4 // 16: This is the global file version
FlagLocalNeeded = 1 << 5 // 32: We need this file
FlagLocalUnsupported FlagLocal = 1 << 0 // 1: The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored FlagLocal = 1 << 1 // 2: Matches local ignore patterns
FlagLocalMustRescan FlagLocal = 1 << 2 // 4: Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly FlagLocal = 1 << 3 // 8: Change detected on receive only folder
FlagLocalGlobal FlagLocal = 1 << 4 // 16: This is the global file version
FlagLocalNeeded FlagLocal = 1 << 5 // 32: We need this file
FlagLocalRemoteInvalid FlagLocal = 1 << 6 // 64: The remote marked this as invalid
// Flags that should result in the Invalid bit on outgoing updates
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly
// Flags that should result in the Invalid bit on outgoing updates (or had it on ingoing ones)
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalRemoteInvalid
// Flags that should result in a file being in conflict with its
// successor, due to us not having an up to date picture of its state on
// disk.
LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly
LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalGlobal | FlagLocalNeeded
LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalGlobal | FlagLocalNeeded | FlagLocalRemoteInvalid
)
func (f FlagLocal) IsInvalid() bool {
return f&LocalInvalidFlags != 0
}
// BlockSizes is the list of valid block sizes, from min to max
var BlockSizes []int
@@ -82,7 +89,9 @@ type FileInfo struct {
// host only. It is not part of the protocol, doesn't get sent or
// received (we make sure to zero it), nonetheless we need it on our
// struct and to be able to serialize it to/from the database.
LocalFlags uint32
// It does carry the info to decide if the file is invalid, which is part of
// the protocol.
LocalFlags FlagLocal
// The version_hash is an implementation detail and not part of the wire
// format.
@@ -97,7 +106,6 @@ type FileInfo struct {
EncryptionTrailerSize int
Deleted bool
RawInvalid bool
NoPermissions bool
truncated bool // was created from a truncated file info without blocks
@@ -128,11 +136,11 @@ func (f *FileInfo) ToWire(withInternalFields bool) *bep.FileInfo {
BlockSize: f.RawBlockSize,
Platform: f.Platform.toWire(),
Deleted: f.Deleted,
Invalid: f.RawInvalid,
Invalid: f.IsInvalid(),
NoPermissions: f.NoPermissions,
}
if withInternalFields {
w.LocalFlags = f.LocalFlags
w.LocalFlags = uint32(f.LocalFlags)
w.VersionHash = f.VersionHash
w.InodeChangeNs = f.InodeChangeNs
w.EncryptionTrailerSize = int32(f.EncryptionTrailerSize)
@@ -207,6 +215,10 @@ type FileInfoWithoutBlocks interface {
}
func fileInfoFromWireWithBlocks(w FileInfoWithoutBlocks, blocks []BlockInfo) FileInfo {
var localFlags FlagLocal
if w.GetInvalid() {
localFlags = FlagLocalRemoteInvalid
}
return FileInfo{
Name: w.GetName(),
Size: w.GetSize(),
@@ -224,14 +236,14 @@ func fileInfoFromWireWithBlocks(w FileInfoWithoutBlocks, blocks []BlockInfo) Fil
RawBlockSize: w.GetBlockSize(),
Platform: platformDataFromWire(w.GetPlatform()),
Deleted: w.GetDeleted(),
RawInvalid: w.GetInvalid(),
LocalFlags: localFlags,
NoPermissions: w.GetNoPermissions(),
}
}
func FileInfoFromDB(w *bep.FileInfo) FileInfo {
f := FileInfoFromWire(w)
f.LocalFlags = w.LocalFlags
f.LocalFlags = FlagLocal(w.LocalFlags)
f.VersionHash = w.VersionHash
f.InodeChangeNs = w.InodeChangeNs
f.EncryptionTrailerSize = int(w.EncryptionTrailerSize)
@@ -240,7 +252,7 @@ func FileInfoFromDB(w *bep.FileInfo) FileInfo {
func FileInfoFromDBTruncated(w FileInfoWithoutBlocks) FileInfo {
f := fileInfoFromWireWithBlocks(w, nil)
f.LocalFlags = w.GetLocalFlags()
f.LocalFlags = FlagLocal(w.GetLocalFlags())
f.VersionHash = w.GetVersionHash()
f.InodeChangeNs = w.GetInodeChangeNs()
f.EncryptionTrailerSize = int(w.GetEncryptionTrailerSize())
@@ -252,13 +264,13 @@ func (f FileInfo) String() string {
switch f.Type {
case FileInfoTypeDirectory:
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.Platform, f.InodeChangeTime())
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.Platform, f.InodeChangeTime())
case FileInfoTypeFile:
return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d, NumBlocks:%d, BlocksHash:%x, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Size, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.RawBlockSize, len(f.Blocks), f.BlocksHash, f.Platform, f.InodeChangeTime())
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Size, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.RawBlockSize, len(f.Blocks), f.BlocksHash, f.Platform, f.InodeChangeTime())
case FileInfoTypeSymlink, FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile:
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Type, f.Sequence, f.Version, f.VersionHash, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.SymlinkTarget, f.Platform, f.InodeChangeTime())
f.Name, f.Type, f.Sequence, f.Version, f.VersionHash, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.SymlinkTarget, f.Platform, f.InodeChangeTime())
default:
panic("mystery file type detected")
}
@@ -269,7 +281,7 @@ func (f FileInfo) IsDeleted() bool {
}
func (f FileInfo) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0
return f.LocalFlags.IsInvalid()
}
func (f FileInfo) IsUnsupported() bool {
@@ -342,7 +354,7 @@ func (f FileInfo) FileName() string {
return f.Name
}
func (f FileInfo) FileLocalFlags() uint32 {
func (f FileInfo) FileLocalFlags() FlagLocal {
return f.LocalFlags
}
@@ -386,7 +398,7 @@ type FileInfoComparison struct {
ModTimeWindow time.Duration
IgnorePerms bool
IgnoreBlocks bool
IgnoreFlags uint32
IgnoreFlags FlagLocal
IgnoreOwnership bool
IgnoreXattrs bool
}
@@ -533,8 +545,7 @@ func (f *FileInfo) SetDeleted(by ShortID) {
f.setNoContent()
}
func (f *FileInfo) setLocalFlags(flags uint32) {
f.RawInvalid = false
func (f *FileInfo) setLocalFlags(flags FlagLocal) {
f.LocalFlags = flags
f.setNoContent()
}
lib/protocol/bep_fileinfo_test.go
+6 -6
View File
@@ -45,7 +45,7 @@ func TestIsEquivalent(t *testing.T) {
b FileInfo
ignPerms *bool // nil means should not matter, we'll test both variants
ignBlocks *bool
ignFlags uint32
ignFlags FlagLocal
eq bool
}
cases := []testCase{
@@ -75,8 +75,8 @@ func TestIsEquivalent(t *testing.T) {
eq: false,
},
{
a: FileInfo{RawInvalid: false},
b: FileInfo{RawInvalid: true},
a: FileInfo{LocalFlags: 0},
b: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
eq: false,
},
{
@@ -100,8 +100,8 @@ func TestIsEquivalent(t *testing.T) {
eq: false,
},
{
a: FileInfo{RawInvalid: true},
b: FileInfo{RawInvalid: true},
a: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
b: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
eq: true,
},
{
@@ -110,7 +110,7 @@ func TestIsEquivalent(t *testing.T) {
eq: true,
},
{
a: FileInfo{RawInvalid: true},
a: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
b: FileInfo{LocalFlags: FlagLocalUnsupported},
eq: true,
},
lib/protocol/conflict_test.go
+1 -1
View File
@@ -15,7 +15,7 @@ func TestWinsConflict(t *testing.T) {
// The first should always win over the second
{{ModifiedS: 42}, {ModifiedS: 41}},
{{ModifiedS: 41}, {ModifiedS: 42, Deleted: true}},
{{Deleted: true}, {ModifiedS: 10, RawInvalid: true}},
{{Deleted: true}, {ModifiedS: 10, LocalFlags: FlagLocalRemoteInvalid}},
{{ModifiedS: 41, Version: Vector{Counters: []Counter{{ID: 42, Value: 2}, {ID: 43, Value: 1}}}}, {ModifiedS: 41, Version: Vector{Counters: []Counter{{ID: 42, Value: 1}, {ID: 43, Value: 2}}}}},
}
lib/protocol/encryption.go
+3 -1
View File
@@ -357,11 +357,13 @@ func encryptFileInfo(keyGen *KeyGenerator, fi FileInfo, folderKey *[keySize]byte
Permissions: 0o644,
ModifiedS: 1234567890, // Sat Feb 14 00:31:30 CET 2009
Deleted: fi.Deleted,
RawInvalid: fi.IsInvalid(),
Version: version,
Sequence: fi.Sequence,
Encrypted: encryptedFI,
}
if fi.IsInvalid() {
enc.LocalFlags = FlagLocalRemoteInvalid
}
if typ == FileInfoTypeFile {
enc.Size = offset // new total file size
enc.Blocks = blocks
lib/scanner/walk.go
+1 -1
View File
@@ -54,7 +54,7 @@ type Config struct {
// events are emitted. Negative number means disabled.
ProgressTickIntervalS int
// Local flags to set on scanned files
LocalFlags uint32
LocalFlags protocol.FlagLocal
// Modification time is to be considered unchanged if the difference is lower.
ModTimeWindow time.Duration
// Event logger to which the scan progress events are sent
lib/scanner/walk_test.go
+1 -1
View File
@@ -567,7 +567,7 @@ func TestScanOwnershipWindows(t *testing.T) {
}
}
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo {
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags protocol.FlagLocal) []protocol.FileInfo {
cfg, cancel := testConfig()
defer cancel()
cfg.Filesystem = fs