lib/db, lib/model: Keep need stats in metadata (ref #5899) (#6413)

lib/db/schemaupdater.go

@@ -22,9 +22,10 @@ import (
 //   6: v0.14.50
 //   7: v0.14.53
 //   8-9: v1.4.0
+//   10: v1.6.0
 const (
-	dbVersion             = 9
-	dbMinSyncthingVersion = "v1.4.0"
+	dbVersion             = 10
+	dbMinSyncthingVersion = "v1.6.0"
 )
 
 type databaseDowngradeError struct {
@@ -85,6 +86,7 @@ func (db *schemaUpdater) updateSchema() error {
 		{6, db.updateSchema5to6},
 		{7, db.updateSchema6to7},
 		{9, db.updateSchemato9},
+		{10, db.updateSchemato10},
 	}
 
 	for _, m := range migrations {
@@ -154,8 +156,10 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			if err != nil {
 				return err
 			}
+			// Purposely pass nil file name to remove from global list,
+			// but don't touch meta and needs
 			buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil)
-			if err != nil {
+			if err != nil && err != errEntryFromGlobalMissing {
 				return err
 			}
 			if err := t.Delete(dbi.Key()); err != nil {
@@ -278,7 +282,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 			if ok {
 				v = haveFile.FileVersion()
 			}
-			if !need(f, ok, v) {
+			fv := FileVersion{
+				Version: f.FileVersion(),
+				Invalid: f.IsInvalid(),
+				Deleted: f.IsDeleted(),
+			}
+			if !need(fv, ok, v) {
 				return true
 			}
 			nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
@@ -390,7 +399,6 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 		var delErr error
 		err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
 			name := []byte(f.FileName())
-			global := f.(protocol.FileInfo)
 			gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if delErr != nil {
 				return false
@@ -413,7 +421,13 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 				// so lets not act on it.
 				return true
 			}
-			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(global, haveLocalFV, localFV.Version) {
+			globalFV := FileVersion{
+				Version: f.FileVersion(),
+				Invalid: f.IsInvalid(),
+				Deleted: f.IsDeleted(),
+			}
+
+			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) {
 				key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
 				if err != nil {
 					delErr = err
@@ -484,3 +498,73 @@ func (db *schemaUpdater) updateSchemato9(prev int) error {
 
 	return t.Commit()
 }
+
+func (db *schemaUpdater) updateSchemato10(_ int) error {
+	t, err := db.newReadWriteTransaction()
+	if err != nil {
+		return err
+	}
+	defer t.close()
+
+	var buf []byte
+
+	for _, folderStr := range db.ListFolders() {
+		folder := []byte(folderStr)
+
+		buf, err = t.keyer.GenerateGlobalVersionKey(buf, folder, nil)
+		if err != nil {
+			return err
+		}
+		buf = globalVersionKey(buf).WithoutName()
+		dbi, err := t.NewPrefixIterator(buf)
+		if err != nil {
+			return err
+		}
+		defer dbi.Release()
+
+		for dbi.Next() {
+			var vl VersionList
+			if err := vl.Unmarshal(dbi.Value()); err != nil {
+				return err
+			}
+
+			changed := false
+			name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
+
+			for i, fv := range vl.Versions {
+				buf, err = t.keyer.GenerateDeviceFileKey(buf, folder, fv.Device, name)
+				if err != nil {
+					return err
+				}
+				f, ok, err := t.getFileTrunc(buf, true)
+				if !ok {
+					return errEntryFromGlobalMissing
+				}
+				if err != nil {
+					return err
+				}
+				if f.IsDeleted() {
+					vl.Versions[i].Deleted = true
+					changed = true
+				}
+			}
+
+			if changed {
+				if err := t.Put(dbi.Key(), mustMarshal(&vl)); err != nil {
+					return err
+				}
+				if err := t.Checkpoint(); err != nil {
+					return err
+				}
+			}
+		}
+		dbi.Release()
+	}
+
+	// Trigger metadata recalc
+	if err := t.deleteKeyPrefix([]byte{KeyTypeFolderMeta}); err != nil {
+		return err
+	}
+
+	return t.Commit()
+}