diff --git a/lib/db/db_test.go b/lib/db/db_test.go index 3aceb4eaf..fbb6cfc8c 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -843,6 +843,81 @@ func TestFlushRecursion(t *testing.T) { } } +func TestCheckLocalNeed(t *testing.T) { + db := NewLowlevel(backend.OpenMemory()) + defer db.Close() + + folderStr := "test" + fs := NewFileSet(folderStr, fs.NewFilesystem(fs.FilesystemTypeFake, ""), db) + + // Add files such that we are in sync for a and b, and need c and d. + files := []protocol.FileInfo{ + {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, + {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, + {Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, + {Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}, + } + fs.Update(protocol.LocalDeviceID, files) + files[2].Version = files[2].Version.Update(remoteDevice0.Short()) + files[3].Version = files[2].Version.Update(remoteDevice0.Short()) + fs.Update(remoteDevice0, files) + + checkNeed := func() { + snap := fs.Snapshot() + defer snap.Release() + c := snap.NeedSize(protocol.LocalDeviceID) + if c.Files != 2 { + t.Errorf("Expected 2 needed files locally, got %v in meta", c.Files) + } + needed := make([]protocol.FileInfo, 0, 2) + snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { + needed = append(needed, fi.(protocol.FileInfo)) + return true + }) + if l := len(needed); l != 2 { + t.Errorf("Expected 2 needed files locally, got %v in db", l) + } else if needed[0].Name != "c" || needed[1].Name != "d" { + t.Errorf("Expected files c and d to be needed, got %v and %v", needed[0].Name, needed[1].Name) + } + } + + checkNeed() + + trans, err := db.newReadWriteTransaction() + if err != nil { + t.Fatal(err) + } + defer trans.close() + + // Add "b" to needed and remove "d" + folder := []byte(folderStr) + key, err := trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[1].Name)) + if err != nil { + t.Fatal(err) + } + if err = trans.Put(key, nil); err != nil { + t.Fatal(err) + } + key, err = trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[3].Name)) + if err != nil { + t.Fatal(err) + } + if err = trans.Delete(key); err != nil { + t.Fatal(err) + } + if err := trans.Commit(); err != nil { + t.Fatal(err) + } + + if repaired, err := db.checkLocalNeed(folder); err != nil { + t.Fatal(err) + } else if repaired != 2 { + t.Error("Expected 2 repaired local need items, got", repaired) + } + + checkNeed() +} + func numBlockLists(db *Lowlevel) (int, error) { it, err := db.Backend.NewPrefixIterator([]byte{KeyTypeBlockList}) if err != nil { diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index ad4eea60d..c91ec9e36 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -801,19 +801,33 @@ func (db *Lowlevel) getMetaAndCheck(folder string) *metadataTracker { db.gcMut.RLock() defer db.gcMut.RUnlock() - meta, err := db.recalcMeta(folder) - if err == nil { - var fixed int - fixed, err = db.repairSequenceGCLocked(folder, meta) - if fixed != 0 { - l.Infof("Repaired %d sequence entries in database", fixed) + var err error + defer func() { + if err != nil && !backend.IsClosed(err) { + panic(err) } + }() + + var fixed int + fixed, err = db.checkLocalNeed([]byte(folder)) + if err != nil { + return nil + } + if fixed != 0 { + l.Infof("Repaired %d local need entries for folder %v in database", fixed, folder) } - if backend.IsClosed(err) { + meta, err := db.recalcMeta(folder) + if err != nil { return nil - } else if err != nil { - panic(err) + } + + fixed, err = db.repairSequenceGCLocked(folder, meta) + if err != nil { + return nil + } + if fixed != 0 { + l.Infof("Repaired %d sequence entries for folder %v in database", fixed, folder) } return meta @@ -1035,6 +1049,86 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack return fixed, t.Commit() } +// Does not take care of metadata - if anything is repaired, the need count +// needs to be recalculated. +func (db *Lowlevel) checkLocalNeed(folder []byte) (int, error) { + repaired := 0 + + t, err := db.newReadWriteTransaction() + if err != nil { + return 0, err + } + defer t.close() + + key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil) + if err != nil { + return 0, err + } + dbi, err := t.NewPrefixIterator(key.WithoutName()) + if err != nil { + return 0, err + } + defer dbi.Release() + + var needName string + var needDone bool + next := func() { + needDone = !dbi.Next() + if !needDone { + needName = string(t.keyer.NameFromGlobalVersionKey(dbi.Key())) + } + } + next() + t.withNeedIteratingGlobal(folder, protocol.LocalDeviceID[:], true, func(fi protocol.FileIntf) bool { + f := fi.(FileInfoTruncated) + for !needDone && needName < f.Name { + repaired++ + if err = t.Delete(dbi.Key()); err != nil { + return false + } + l.Debugln("check local need: removing", needName) + next() + } + if needName == f.Name { + next() + } else { + repaired++ + key, err = t.keyer.GenerateNeedFileKey(key, folder, []byte(f.Name)) + if err != nil { + return false + } + if err = t.Put(key, nil); err != nil { + return false + } + l.Debugln("check local need: adding", f.Name) + } + return true + }) + if err != nil { + return 0, err + } + + for !needDone { + repaired++ + if err := t.Delete(dbi.Key()); err != nil { + return 0, err + } + l.Debugln("check local need: removing", needName) + next() + } + + if err := dbi.Error(); err != nil { + return 0, err + } + dbi.Release() + + if err = t.Commit(); err != nil { + return 0, err + } + + return repaired, nil +} + // unchanged checks if two files are the same and thus don't need to be updated. // Local flags or the invalid bit might change without the version // being bumped. diff --git a/lib/db/transactions.go b/lib/db/transactions.go index 33aec9176..c993fbf7f 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -429,7 +429,10 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn if bytes.Equal(device, protocol.LocalDeviceID[:]) { return t.withNeedLocal(folder, truncate, fn) } + return t.withNeedIteratingGlobal(folder, device, truncate, fn) +} +func (t *readOnlyTransaction) withNeedIteratingGlobal(folder, device []byte, truncate bool, fn Iterator) error { key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil) if err != nil { return err @@ -468,11 +471,12 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn return err } - globalDev, ok := globalFV.FirstDevice() - if !ok { - return errEmptyFileVersion + if shouldDebug() { + if globalDev, ok := globalFV.FirstDevice(); ok { + globalID, _ := protocol.DeviceIDFromBytes(globalDev) + l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalID) + } } - l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalDev) if !fn(gf) { return dbi.Error() }