felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

fix: track invalid files in LocalFlags to fix global count (#10170)

Browse Source 
Move the "invalid" bit to a local flag, making it easier to track in counts etc.
This commit is contained in:
Simon Frei
2025-06-13 07:33:31 +02:00
committed by GitHub
parent cb7cea93a2
commit 7b319111d3
19 changed files with 132 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/counts.go
+3 -3
View File
@@ -19,9 +19,9 @@ type Counts struct {
Symlinks int Symlinks int
Deleted int Deleted int
Bytes int64 Bytes int64
Sequence int64 // zero for the global state Sequence int64 // zero for the global state
DeviceID protocol.DeviceID // device ID for remote devices, or special values for local/global DeviceID protocol.DeviceID // device ID for remote devices, or special values for local/global
LocalFlags uint32 // the local flag for this count bucket LocalFlags protocol.FlagLocal // the local flag for this count bucket
} }
func (c Counts) Add(other Counts) Counts { func (c Counts) Add(other Counts) Counts {
internal/db/interface.go
+1 -1
View File
@@ -100,7 +100,7 @@ type FileMetadata struct {
Sequence int64 Sequence int64
ModNanos int64 ModNanos int64
Size int64 Size int64
LocalFlags int64 LocalFlags protocol.FlagLocal
Type protocol.FileInfoType Type protocol.FileInfoType
Deleted bool Deleted bool
Invalid bool Invalid bool
internal/db/sqlite/basedb.go
+1
View File
@@ -103,6 +103,7 @@ func openBase(path string, maxConns int, pragmas, schemaScripts, migrationScript
"FlagLocalReceiveOnly": protocol.FlagLocalReceiveOnly, "FlagLocalReceiveOnly": protocol.FlagLocalReceiveOnly,
"FlagLocalGlobal": protocol.FlagLocalGlobal, "FlagLocalGlobal": protocol.FlagLocalGlobal,
"FlagLocalNeeded": protocol.FlagLocalNeeded, "FlagLocalNeeded": protocol.FlagLocalNeeded,
"LocalInvalidFlags": protocol.LocalInvalidFlags,
"SyncthingVersion": build.LongVersion, "SyncthingVersion": build.LongVersion,
} }
internal/db/sqlite/db_global_test.go
+51
View File
@@ -268,6 +268,57 @@ func TestDontNeedIgnored(t *testing.T) {
} }
} }
func TestDontNeedRemoteInvalid(t *testing.T) {
t.Parallel()
db, err := OpenTemp()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := db.Close(); err != nil {
t.Fatal(err)
}
})
// A remote file with the invalid bit set
files := []protocol.FileInfo{
genFile("test1", 1, 103),
}
files[0].LocalFlags = protocol.FlagLocalRemoteInvalid
err = db.Update(folderID, protocol.DeviceID{42}, files)
if err != nil {
t.Fatal(err)
}
// It's not part of the global size
s, err := db.CountGlobal(folderID)
if err != nil {
t.Fatal(err)
}
if s.Bytes != 0 || s.Files != 0 {
t.Log(s)
t.Error("bad global")
}
// We don't need it
s, err = db.CountNeed(folderID, protocol.LocalDeviceID)
if err != nil {
t.Fatal(err)
}
if s.Bytes != 0 || s.Files != 0 {
t.Log(s)
t.Error("bad need")
}
// It shouldn't show up in the need list
names := mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0))
if len(names) != 0 {
t.Log(names)
t.Error("need no files")
}
}
func TestRemoteDontNeedLocalIgnored(t *testing.T) { func TestRemoteDontNeedLocalIgnored(t *testing.T) {
t.Parallel() t.Parallel()
internal/db/sqlite/folderdb_counts.go
+4 -7
View File
@@ -39,13 +39,10 @@ func (s *folderDB) CountNeed(device protocol.DeviceID) (db.Counts, error) {
} }
func (s *folderDB) CountGlobal() (db.Counts, error) { func (s *folderDB) CountGlobal() (db.Counts, error) {
// Exclude ignored and receive-only changed files from the global count
// (legacy expectation? it's a bit weird since those files can in fact
// be global and you can get them with GetGlobal etc.)
var res []countsRow var res []countsRow
err := s.stmt(` err := s.stmt(`
SELECT s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s SELECT s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s
WHERE s.local_flags & {{.FlagLocalGlobal}} != 0 AND s.local_flags & {{or .FlagLocalReceiveOnly .FlagLocalIgnored}} = 0 WHERE s.local_flags & {{.FlagLocalGlobal}} != 0 AND s.local_flags & {{.LocalInvalidFlags}} = 0
`).Select(&res) `).Select(&res)
if err != nil { if err != nil {
return db.Counts{}, wrap(err) return db.Counts{}, wrap(err)
@@ -84,7 +81,7 @@ func (s *folderDB) needSizeRemote(device protocol.DeviceID) (db.Counts, error) {
// See neededGlobalFilesRemote for commentary as that is the same query without summing // See neededGlobalFilesRemote for commentary as that is the same query without summing
if err := s.stmt(` if err := s.stmt(`
SELECT g.type, count(*) as count, sum(g.size) as size, g.local_flags, g.deleted FROM files g SELECT g.type, count(*) as count, sum(g.size) as size, g.local_flags, g.deleted FROM files g
WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND NOT g.deleted AND NOT g.invalid AND NOT EXISTS ( WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND NOT g.deleted AND g.local_flags & {{.LocalInvalidFlags}} = 0 AND NOT EXISTS (
SELECT 1 FROM FILES f SELECT 1 FROM FILES f
INNER JOIN devices d ON d.idx = f.device_idx INNER JOIN devices d ON d.idx = f.device_idx
WHERE f.name = g.name AND f.version = g.version AND d.device_id = ? WHERE f.name = g.name AND f.version = g.version AND d.device_id = ?
@@ -94,10 +91,10 @@ func (s *folderDB) needSizeRemote(device protocol.DeviceID) (db.Counts, error) {
UNION ALL UNION ALL
SELECT g.type, count(*) as count, sum(g.size) as size, g.local_flags, g.deleted FROM files g SELECT g.type, count(*) as count, sum(g.size) as size, g.local_flags, g.deleted FROM files g
WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND g.deleted AND NOT g.invalid AND EXISTS ( WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND g.deleted AND g.local_flags & {{.LocalInvalidFlags}} = 0 AND EXISTS (
SELECT 1 FROM FILES f SELECT 1 FROM FILES f
INNER JOIN devices d ON d.idx = f.device_idx INNER JOIN devices d ON d.idx = f.device_idx
WHERE f.name = g.name AND d.device_id = ? AND NOT f.deleted AND NOT f.invalid WHERE f.name = g.name AND d.device_id = ? AND NOT f.deleted AND f.local_flags & {{.LocalInvalidFlags}} = 0
) )
GROUP BY g.type, g.local_flags, g.deleted GROUP BY g.type, g.local_flags, g.deleted
`).Select(&res, device.String(), `).Select(&res, device.String(),
internal/db/sqlite/folderdb_global.go
+5 -5
View File
@@ -74,7 +74,7 @@ func (s *folderDB) GetGlobalAvailability(file string) ([]protocol.DeviceID, erro
func (s *folderDB) AllGlobalFiles() (iter.Seq[db.FileMetadata], func() error) { func (s *folderDB) AllGlobalFiles() (iter.Seq[db.FileMetadata], func() error) {
it, errFn := iterStructs[db.FileMetadata](s.stmt(` it, errFn := iterStructs[db.FileMetadata](s.stmt(`
SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.local_flags as localflags FROM files f
WHERE f.local_flags & {{.FlagLocalGlobal}} != 0 WHERE f.local_flags & {{.FlagLocalGlobal}} != 0
ORDER BY f.name ORDER BY f.name
`).Queryx()) `).Queryx())
@@ -93,7 +93,7 @@ func (s *folderDB) AllGlobalFilesPrefix(prefix string) (iter.Seq[db.FileMetadata
end := prefixEnd(prefix) end := prefixEnd(prefix)
it, errFn := iterStructs[db.FileMetadata](s.stmt(` it, errFn := iterStructs[db.FileMetadata](s.stmt(`
SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.local_flags as localflags FROM files f
WHERE f.name >= ? AND f.name < ? AND f.local_flags & {{.FlagLocalGlobal}} != 0 WHERE f.name >= ? AND f.name < ? AND f.local_flags & {{.FlagLocalGlobal}} != 0
ORDER BY f.name ORDER BY f.name
`).Queryx(prefix, end)) `).Queryx(prefix, end))
@@ -158,7 +158,7 @@ func (s *folderDB) neededGlobalFilesRemote(device protocol.DeviceID, selectOpts
SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi
INNER JOIN files g on fi.sequence = g.sequence INNER JOIN files g on fi.sequence = g.sequence
LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash
WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND NOT g.deleted AND NOT g.invalid AND NOT EXISTS ( WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND NOT g.deleted AND g.local_flags & {{.LocalInvalidFlags}} = 0 AND NOT EXISTS (
SELECT 1 FROM FILES f SELECT 1 FROM FILES f
INNER JOIN devices d ON d.idx = f.device_idx INNER JOIN devices d ON d.idx = f.device_idx
WHERE f.name = g.name AND f.version = g.version AND d.device_id = ? WHERE f.name = g.name AND f.version = g.version AND d.device_id = ?
@@ -169,10 +169,10 @@ func (s *folderDB) neededGlobalFilesRemote(device protocol.DeviceID, selectOpts
SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi
INNER JOIN files g on fi.sequence = g.sequence INNER JOIN files g on fi.sequence = g.sequence
LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash
WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND g.deleted AND NOT g.invalid AND EXISTS ( WHERE g.local_flags & {{.FlagLocalGlobal}} != 0 AND g.deleted AND g.local_flags & {{.LocalInvalidFlags}} = 0 AND EXISTS (
SELECT 1 FROM FILES f SELECT 1 FROM FILES f
INNER JOIN devices d ON d.idx = f.device_idx INNER JOIN devices d ON d.idx = f.device_idx
WHERE f.name = g.name AND d.device_id = ? AND NOT f.deleted AND NOT f.invalid WHERE f.name = g.name AND d.device_id = ? AND NOT f.deleted AND f.local_flags & {{.LocalInvalidFlags}} = 0
) )
`+selectOpts).Queryx( `+selectOpts).Queryx(
device.String(), device.String(),
internal/db/sqlite/folderdb_local.go
+1 -1
View File
@@ -89,7 +89,7 @@ func (s *folderDB) AllLocalFilesWithPrefix(device protocol.DeviceID, prefix stri
func (s *folderDB) AllLocalFilesWithBlocksHash(h []byte) (iter.Seq[db.FileMetadata], func() error) { func (s *folderDB) AllLocalFilesWithBlocksHash(h []byte) (iter.Seq[db.FileMetadata], func() error) {
return iterStructs[db.FileMetadata](s.stmt(` return iterStructs[db.FileMetadata](s.stmt(`
SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.local_flags as localflags FROM files f
WHERE f.device_idx = {{.LocalDeviceIdx}} AND f.blocklist_hash = ? WHERE f.device_idx = {{.LocalDeviceIdx}} AND f.blocklist_hash = ?
`).Queryx(h)) `).Queryx(h))
} }
internal/db/sqlite/folderdb_update.go
+15 -12
View File
@@ -46,8 +46,8 @@ func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) erro
//nolint:sqlclosecheck //nolint:sqlclosecheck
insertFileStmt, err := txp.Preparex(` insertFileStmt, err := txp.Preparex(`
INSERT OR REPLACE INTO files (device_idx, remote_sequence, name, type, modified, size, version, deleted, invalid, local_flags, blocklist_hash) INSERT OR REPLACE INTO files (device_idx, remote_sequence, name, type, modified, size, version, deleted, local_flags, blocklist_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
RETURNING sequence RETURNING sequence
`) `)
if err != nil { if err != nil {
@@ -101,7 +101,7 @@ func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) erro
remoteSeq = &f.Sequence remoteSeq = &f.Sequence
} }
var localSeq int64 var localSeq int64
if err := insertFileStmt.Get(&localSeq, deviceIdx, remoteSeq, f.Name, f.Type, f.ModTime().UnixNano(), f.Size, f.Version.String(), f.IsDeleted(), f.IsInvalid(), f.LocalFlags, blockshash); err != nil { if err := insertFileStmt.Get(&localSeq, deviceIdx, remoteSeq, f.Name, f.Type, f.ModTime().UnixNano(), f.Size, f.Version.String(), f.IsDeleted(), f.LocalFlags, blockshash); err != nil {
return wrap(err, "insert file") return wrap(err, "insert file")
} }
@@ -329,7 +329,7 @@ func (s *folderDB) recalcGlobalForFolder(txp *txPreparedStmts) error {
func (s *folderDB) recalcGlobalForFile(txp *txPreparedStmts, file string) error { func (s *folderDB) recalcGlobalForFile(txp *txPreparedStmts, file string) error {
//nolint:sqlclosecheck //nolint:sqlclosecheck
selStmt, err := txp.Preparex(` selStmt, err := txp.Preparex(`
SELECT name, device_idx, sequence, modified, version, deleted, invalid, local_flags FROM files SELECT name, device_idx, sequence, modified, version, deleted, local_flags FROM files
WHERE name = ? WHERE name = ?
`) `)
if err != nil { if err != nil {
@@ -350,7 +350,7 @@ func (s *folderDB) recalcGlobalForFile(txp *txPreparedStmts, file string) error
// The global version is the first one in the list that is not invalid, // The global version is the first one in the list that is not invalid,
// or just the first one in the list if all are invalid. // or just the first one in the list if all are invalid.
var global fileRow var global fileRow
globIdx := slices.IndexFunc(es, func(e fileRow) bool { return !e.Invalid }) globIdx := slices.IndexFunc(es, func(e fileRow) bool { return !e.IsInvalid() })
if globIdx < 0 { if globIdx < 0 {
globIdx = 0 globIdx = 0
} }
@@ -368,7 +368,7 @@ func (s *folderDB) recalcGlobalForFile(txp *txPreparedStmts, file string) error
// Set the global flag on the global entry. Set the need flag if the // Set the global flag on the global entry. Set the need flag if the
// local device needs this file, unless it's invalid. // local device needs this file, unless it's invalid.
global.LocalFlags |= protocol.FlagLocalGlobal global.LocalFlags |= protocol.FlagLocalGlobal
if hasLocal || global.Invalid { if hasLocal || global.IsInvalid() {
global.LocalFlags &= ^protocol.FlagLocalNeeded global.LocalFlags &= ^protocol.FlagLocalNeeded
} else { } else {
global.LocalFlags |= protocol.FlagLocalNeeded global.LocalFlags |= protocol.FlagLocalNeeded
@@ -426,9 +426,8 @@ type fileRow struct {
Sequence int64 Sequence int64
Modified int64 Modified int64
Size int64 Size int64
LocalFlags int64 `db:"local_flags"` LocalFlags protocol.FlagLocal `db:"local_flags"`
Deleted bool Deleted bool
Invalid bool
} }
func (e fileRow) Compare(other fileRow) int { func (e fileRow) Compare(other fileRow) int {
@@ -436,8 +435,8 @@ func (e fileRow) Compare(other fileRow) int {
vc := e.Version.Compare(other.Version.Vector) vc := e.Version.Compare(other.Version.Vector)
switch vc { switch vc {
case protocol.Equal: case protocol.Equal:
if e.Invalid != other.Invalid { if e.IsInvalid() != other.IsInvalid() {
if e.Invalid { if e.IsInvalid() {
return 1 return 1
} }
return -1 return -1
@@ -453,8 +452,8 @@ func (e fileRow) Compare(other fileRow) int {
case protocol.Lesser: // we are older case protocol.Lesser: // we are older
return 1 return 1
case protocol.ConcurrentGreater, protocol.ConcurrentLesser: // there is a conflict case protocol.ConcurrentGreater, protocol.ConcurrentLesser: // there is a conflict
if e.Invalid != other.Invalid { if e.IsInvalid() != other.IsInvalid() {
if e.Invalid { // we are invalid, we lose if e.IsInvalid() { // we are invalid, we lose
return 1 return 1
} }
return -1 // they are invalid, we win return -1 // they are invalid, we win
@@ -477,6 +476,10 @@ func (e fileRow) Compare(other fileRow) int {
} }
} }
func (e fileRow) IsInvalid() bool {
return e.LocalFlags.IsInvalid()
}
func (s *folderDB) periodicCheckpointLocked(fs []protocol.FileInfo) { func (s *folderDB) periodicCheckpointLocked(fs []protocol.FileInfo) {
// Induce periodic checkpoints. We add points for each file and block, // Induce periodic checkpoints. We add points for each file and block,
// and checkpoint when we've written more than a threshold of points. // and checkpoint when we've written more than a threshold of points.
internal/db/sqlite/sql/schema/folder/20-files.sql
-1
View File
@@ -31,7 +31,6 @@ CREATE TABLE IF NOT EXISTS files (
size INTEGER NOT NULL, size INTEGER NOT NULL,
version TEXT NOT NULL COLLATE BINARY, version TEXT NOT NULL COLLATE BINARY,
deleted INTEGER NOT NULL, -- boolean deleted INTEGER NOT NULL, -- boolean
invalid INTEGER NOT NULL, -- boolean
local_flags INTEGER NOT NULL, local_flags INTEGER NOT NULL,
blocklist_hash BLOB, -- null when there are no blocks blocklist_hash BLOB, -- null when there are no blocks
FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE
lib/model/fakeconns_test.go
+1 -1
View File
@@ -74,7 +74,7 @@ func (f *fakeConnection) DownloadProgress(_ context.Context, dp *protocol.Downlo
}) })
} }
func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags uint32) { func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags protocol.FlagLocal) {
blockSize := protocol.BlockSize(int64(len(data))) blockSize := protocol.BlockSize(int64(len(data)))
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil) blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil)
lib/model/folder.go
+1 -1
View File
@@ -44,7 +44,7 @@ type folder struct {
*stats.FolderStatisticsReference *stats.FolderStatisticsReference
ioLimiter *semaphore.Semaphore ioLimiter *semaphore.Semaphore
localFlags uint32 localFlags protocol.FlagLocal
model *model model *model
shortID protocol.ShortID shortID protocol.ShortID
lib/model/indexhandler.go
-2
View File
@@ -481,8 +481,6 @@ func (s *indexHandler) logSequenceAnomaly(msg string, extra map[string]any) {
} }
func prepareFileInfoForIndex(f protocol.FileInfo) protocol.FileInfo { func prepareFileInfoForIndex(f protocol.FileInfo) protocol.FileInfo {
// Mark the file as invalid if any of the local bad stuff flags are set.
f.RawInvalid = f.IsInvalid()
// If the file is marked LocalReceive (i.e., changed locally on a // If the file is marked LocalReceive (i.e., changed locally on a
// receive only folder) we do not want it to ever become the // receive only folder) we do not want it to ever become the
// globally best version, invalid or not. // globally best version, invalid or not.
lib/model/model_test.go
+2 -2
View File
@@ -3609,7 +3609,7 @@ func TestIssue6961(t *testing.T) {
// Remote, valid and existing file // Remote, valid and existing file
must(t, m.Index(conn1, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, Version: version, Sequence: 1}}})) must(t, m.Index(conn1, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, Version: version, Sequence: 1}}}))
// Remote, invalid (receive-only) and existing file // Remote, invalid (receive-only) and existing file
must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, RawInvalid: true, Sequence: 1}}})) must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: name, LocalFlags: protocol.FlagLocalRemoteInvalid, Sequence: 1}}}))
// Create a local file // Create a local file
if fd, err := tfs.OpenFile(name, fs.OptCreate, 0o666); err != nil { if fd, err := tfs.OpenFile(name, fs.OptCreate, 0o666); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -3635,7 +3635,7 @@ func TestIssue6961(t *testing.T) {
m.ScanFolders() m.ScanFolders()
// Drop the remote index, add some other file. // Drop the remote index, add some other file.
must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: "bar", RawInvalid: true, Sequence: 1}}})) must(t, m.Index(conn2, &protocol.Index{Folder: fcfg.ID, Files: []protocol.FileInfo{{Name: "bar", LocalFlags: protocol.FlagLocalRemoteInvalid, Sequence: 1}}}))
// Pause and unpause folder to create new db.FileSet and thus recalculate everything // Pause and unpause folder to create new db.FileSet and thus recalculate everything
pauseFolder(t, wcfg, fcfg.ID, true) pauseFolder(t, wcfg, fcfg.ID, true)
lib/protocol/bep_fileinfo.go
+35 -24
View File
@@ -17,26 +17,33 @@ import (
"github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/build"
) )
type FlagLocal uint32
// FileInfo.LocalFlags flags // FileInfo.LocalFlags flags
const ( const (
FlagLocalUnsupported = 1 << 0 // 1: The kind is unsupported, e.g. symlinks on Windows FlagLocalUnsupported FlagLocal = 1 << 0 // 1: The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored = 1 << 1 // 2: Matches local ignore patterns FlagLocalIgnored FlagLocal = 1 << 1 // 2: Matches local ignore patterns
FlagLocalMustRescan = 1 << 2 // 4: Doesn't match content on disk, must be rechecked fully FlagLocalMustRescan FlagLocal = 1 << 2 // 4: Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly = 1 << 3 // 8: Change detected on receive only folder FlagLocalReceiveOnly FlagLocal = 1 << 3 // 8: Change detected on receive only folder
FlagLocalGlobal = 1 << 4 // 16: This is the global file version FlagLocalGlobal FlagLocal = 1 << 4 // 16: This is the global file version
FlagLocalNeeded = 1 << 5 // 32: We need this file FlagLocalNeeded FlagLocal = 1 << 5 // 32: We need this file
FlagLocalRemoteInvalid FlagLocal = 1 << 6 // 64: The remote marked this as invalid
// Flags that should result in the Invalid bit on outgoing updates // Flags that should result in the Invalid bit on outgoing updates (or had it on ingoing ones)
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalRemoteInvalid
// Flags that should result in a file being in conflict with its // Flags that should result in a file being in conflict with its
// successor, due to us not having an up to date picture of its state on // successor, due to us not having an up to date picture of its state on
// disk. // disk.
LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly
LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalGlobal | FlagLocalNeeded LocalAllFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly | FlagLocalGlobal | FlagLocalNeeded | FlagLocalRemoteInvalid
) )
func (f FlagLocal) IsInvalid() bool {
return f&LocalInvalidFlags != 0
}
// BlockSizes is the list of valid block sizes, from min to max // BlockSizes is the list of valid block sizes, from min to max
var BlockSizes []int var BlockSizes []int
@@ -82,7 +89,9 @@ type FileInfo struct {
// host only. It is not part of the protocol, doesn't get sent or // host only. It is not part of the protocol, doesn't get sent or
// received (we make sure to zero it), nonetheless we need it on our // received (we make sure to zero it), nonetheless we need it on our
// struct and to be able to serialize it to/from the database. // struct and to be able to serialize it to/from the database.
LocalFlags uint32 // It does carry the info to decide if the file is invalid, which is part of
// the protocol.
LocalFlags FlagLocal
// The version_hash is an implementation detail and not part of the wire // The version_hash is an implementation detail and not part of the wire
// format. // format.
@@ -97,7 +106,6 @@ type FileInfo struct {
EncryptionTrailerSize int EncryptionTrailerSize int
Deleted bool Deleted bool
RawInvalid bool
NoPermissions bool NoPermissions bool
truncated bool // was created from a truncated file info without blocks truncated bool // was created from a truncated file info without blocks
@@ -128,11 +136,11 @@ func (f *FileInfo) ToWire(withInternalFields bool) *bep.FileInfo {
BlockSize: f.RawBlockSize, BlockSize: f.RawBlockSize,
Platform: f.Platform.toWire(), Platform: f.Platform.toWire(),
Deleted: f.Deleted, Deleted: f.Deleted,
Invalid: f.RawInvalid, Invalid: f.IsInvalid(),
NoPermissions: f.NoPermissions, NoPermissions: f.NoPermissions,
} }
if withInternalFields { if withInternalFields {
w.LocalFlags = f.LocalFlags w.LocalFlags = uint32(f.LocalFlags)
w.VersionHash = f.VersionHash w.VersionHash = f.VersionHash
w.InodeChangeNs = f.InodeChangeNs w.InodeChangeNs = f.InodeChangeNs
w.EncryptionTrailerSize = int32(f.EncryptionTrailerSize) w.EncryptionTrailerSize = int32(f.EncryptionTrailerSize)
@@ -207,6 +215,10 @@ type FileInfoWithoutBlocks interface {
} }
func fileInfoFromWireWithBlocks(w FileInfoWithoutBlocks, blocks []BlockInfo) FileInfo { func fileInfoFromWireWithBlocks(w FileInfoWithoutBlocks, blocks []BlockInfo) FileInfo {
var localFlags FlagLocal
if w.GetInvalid() {
localFlags = FlagLocalRemoteInvalid
}
return FileInfo{ return FileInfo{
Name: w.GetName(), Name: w.GetName(),
Size: w.GetSize(), Size: w.GetSize(),
@@ -224,14 +236,14 @@ func fileInfoFromWireWithBlocks(w FileInfoWithoutBlocks, blocks []BlockInfo) Fil
RawBlockSize: w.GetBlockSize(), RawBlockSize: w.GetBlockSize(),
Platform: platformDataFromWire(w.GetPlatform()), Platform: platformDataFromWire(w.GetPlatform()),
Deleted: w.GetDeleted(), Deleted: w.GetDeleted(),
RawInvalid: w.GetInvalid(), LocalFlags: localFlags,
NoPermissions: w.GetNoPermissions(), NoPermissions: w.GetNoPermissions(),
} }
} }
func FileInfoFromDB(w *bep.FileInfo) FileInfo { func FileInfoFromDB(w *bep.FileInfo) FileInfo {
f := FileInfoFromWire(w) f := FileInfoFromWire(w)
f.LocalFlags = w.LocalFlags f.LocalFlags = FlagLocal(w.LocalFlags)
f.VersionHash = w.VersionHash f.VersionHash = w.VersionHash
f.InodeChangeNs = w.InodeChangeNs f.InodeChangeNs = w.InodeChangeNs
f.EncryptionTrailerSize = int(w.EncryptionTrailerSize) f.EncryptionTrailerSize = int(w.EncryptionTrailerSize)
@@ -240,7 +252,7 @@ func FileInfoFromDB(w *bep.FileInfo) FileInfo {
func FileInfoFromDBTruncated(w FileInfoWithoutBlocks) FileInfo { func FileInfoFromDBTruncated(w FileInfoWithoutBlocks) FileInfo {
f := fileInfoFromWireWithBlocks(w, nil) f := fileInfoFromWireWithBlocks(w, nil)
f.LocalFlags = w.GetLocalFlags() f.LocalFlags = FlagLocal(w.GetLocalFlags())
f.VersionHash = w.GetVersionHash() f.VersionHash = w.GetVersionHash()
f.InodeChangeNs = w.GetInodeChangeNs() f.InodeChangeNs = w.GetInodeChangeNs()
f.EncryptionTrailerSize = int(w.GetEncryptionTrailerSize()) f.EncryptionTrailerSize = int(w.GetEncryptionTrailerSize())
@@ -252,13 +264,13 @@ func (f FileInfo) String() string {
switch f.Type { switch f.Type {
case FileInfoTypeDirectory: case FileInfoTypeDirectory:
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, Platform:%v, InodeChangeTime:%v}", return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.Platform, f.InodeChangeTime()) f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.Platform, f.InodeChangeTime())
case FileInfoTypeFile: case FileInfoTypeFile:
return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d, NumBlocks:%d, BlocksHash:%x, Platform:%v, InodeChangeTime:%v}", return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, VersionHash:%x, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d, NumBlocks:%d, BlocksHash:%x, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Size, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.RawBlockSize, len(f.Blocks), f.BlocksHash, f.Platform, f.InodeChangeTime()) f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.VersionHash, f.Size, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.RawBlockSize, len(f.Blocks), f.BlocksHash, f.Platform, f.InodeChangeTime())
case FileInfoTypeSymlink, FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile: case FileInfoTypeSymlink, FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile:
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q, Platform:%v, InodeChangeTime:%v}", return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, VersionHash:%x, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q, Platform:%v, InodeChangeTime:%v}",
f.Name, f.Type, f.Sequence, f.Version, f.VersionHash, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.SymlinkTarget, f.Platform, f.InodeChangeTime()) f.Name, f.Type, f.Sequence, f.Version, f.VersionHash, f.Deleted, f.IsInvalid(), f.LocalFlags, f.NoPermissions, f.SymlinkTarget, f.Platform, f.InodeChangeTime())
default: default:
panic("mystery file type detected") panic("mystery file type detected")
} }
@@ -269,7 +281,7 @@ func (f FileInfo) IsDeleted() bool {
} }
func (f FileInfo) IsInvalid() bool { func (f FileInfo) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0 return f.LocalFlags.IsInvalid()
} }
func (f FileInfo) IsUnsupported() bool { func (f FileInfo) IsUnsupported() bool {
@@ -342,7 +354,7 @@ func (f FileInfo) FileName() string {
return f.Name return f.Name
} }
func (f FileInfo) FileLocalFlags() uint32 { func (f FileInfo) FileLocalFlags() FlagLocal {
return f.LocalFlags return f.LocalFlags
} }
@@ -386,7 +398,7 @@ type FileInfoComparison struct {
ModTimeWindow time.Duration ModTimeWindow time.Duration
IgnorePerms bool IgnorePerms bool
IgnoreBlocks bool IgnoreBlocks bool
IgnoreFlags uint32 IgnoreFlags FlagLocal
IgnoreOwnership bool IgnoreOwnership bool
IgnoreXattrs bool IgnoreXattrs bool
} }
@@ -533,8 +545,7 @@ func (f *FileInfo) SetDeleted(by ShortID) {
f.setNoContent() f.setNoContent()
} }
func (f *FileInfo) setLocalFlags(flags uint32) { func (f *FileInfo) setLocalFlags(flags FlagLocal) {
f.RawInvalid = false
f.LocalFlags = flags f.LocalFlags = flags
f.setNoContent() f.setNoContent()
} }
lib/protocol/bep_fileinfo_test.go
+6 -6
View File
@@ -45,7 +45,7 @@ func TestIsEquivalent(t *testing.T) {
b FileInfo b FileInfo
ignPerms *bool // nil means should not matter, we'll test both variants ignPerms *bool // nil means should not matter, we'll test both variants
ignBlocks *bool ignBlocks *bool
ignFlags uint32 ignFlags FlagLocal
eq bool eq bool
} }
cases := []testCase{ cases := []testCase{
@@ -75,8 +75,8 @@ func TestIsEquivalent(t *testing.T) {
eq: false, eq: false,
}, },
{ {
a: FileInfo{RawInvalid: false}, a: FileInfo{LocalFlags: 0},
b: FileInfo{RawInvalid: true}, b: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
eq: false, eq: false,
}, },
{ {
@@ -100,8 +100,8 @@ func TestIsEquivalent(t *testing.T) {
eq: false, eq: false,
}, },
{ {
a: FileInfo{RawInvalid: true}, a: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
b: FileInfo{RawInvalid: true}, b: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
eq: true, eq: true,
}, },
{ {
@@ -110,7 +110,7 @@ func TestIsEquivalent(t *testing.T) {
eq: true, eq: true,
}, },
{ {
a: FileInfo{RawInvalid: true}, a: FileInfo{LocalFlags: FlagLocalRemoteInvalid},
b: FileInfo{LocalFlags: FlagLocalUnsupported}, b: FileInfo{LocalFlags: FlagLocalUnsupported},
eq: true, eq: true,
}, },
lib/protocol/conflict_test.go
+1 -1
View File
@@ -15,7 +15,7 @@ func TestWinsConflict(t *testing.T) {
// The first should always win over the second // The first should always win over the second
{{ModifiedS: 42}, {ModifiedS: 41}}, {{ModifiedS: 42}, {ModifiedS: 41}},
{{ModifiedS: 41}, {ModifiedS: 42, Deleted: true}}, {{ModifiedS: 41}, {ModifiedS: 42, Deleted: true}},
{{Deleted: true}, {ModifiedS: 10, RawInvalid: true}}, {{Deleted: true}, {ModifiedS: 10, LocalFlags: FlagLocalRemoteInvalid}},
{{ModifiedS: 41, Version: Vector{Counters: []Counter{{ID: 42, Value: 2}, {ID: 43, Value: 1}}}}, {ModifiedS: 41, Version: Vector{Counters: []Counter{{ID: 42, Value: 1}, {ID: 43, Value: 2}}}}}, {{ModifiedS: 41, Version: Vector{Counters: []Counter{{ID: 42, Value: 2}, {ID: 43, Value: 1}}}}, {ModifiedS: 41, Version: Vector{Counters: []Counter{{ID: 42, Value: 1}, {ID: 43, Value: 2}}}}},
} }
lib/protocol/encryption.go
+3 -1
View File
@@ -357,11 +357,13 @@ func encryptFileInfo(keyGen *KeyGenerator, fi FileInfo, folderKey *[keySize]byte
Permissions: 0o644, Permissions: 0o644,
ModifiedS: 1234567890, // Sat Feb 14 00:31:30 CET 2009 ModifiedS: 1234567890, // Sat Feb 14 00:31:30 CET 2009
Deleted: fi.Deleted, Deleted: fi.Deleted,
RawInvalid: fi.IsInvalid(),
Version: version, Version: version,
Sequence: fi.Sequence, Sequence: fi.Sequence,
Encrypted: encryptedFI, Encrypted: encryptedFI,
} }
if fi.IsInvalid() {
enc.LocalFlags = FlagLocalRemoteInvalid
}
if typ == FileInfoTypeFile { if typ == FileInfoTypeFile {
enc.Size = offset // new total file size enc.Size = offset // new total file size
enc.Blocks = blocks enc.Blocks = blocks
lib/scanner/walk.go
+1 -1
View File
@@ -54,7 +54,7 @@ type Config struct {
// events are emitted. Negative number means disabled. // events are emitted. Negative number means disabled.
ProgressTickIntervalS int ProgressTickIntervalS int
// Local flags to set on scanned files // Local flags to set on scanned files
LocalFlags uint32 LocalFlags protocol.FlagLocal
// Modification time is to be considered unchanged if the difference is lower. // Modification time is to be considered unchanged if the difference is lower.
ModTimeWindow time.Duration ModTimeWindow time.Duration
// Event logger to which the scan progress events are sent // Event logger to which the scan progress events are sent
lib/scanner/walk_test.go
+1 -1
View File
@@ -567,7 +567,7 @@ func TestScanOwnershipWindows(t *testing.T) {
} }
} }
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo { func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags protocol.FlagLocal) []protocol.FileInfo {
cfg, cancel := testConfig() cfg, cancel := testConfig()
defer cancel() defer cancel()
cfg.Filesystem = fs cfg.Filesystem = fs