lib/db: Refactor to use global list by version (fixes #6372) (#6638)

Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors.

Co-authored-by: Jakob Borg <jakob@kastelo.net>

lib/db/schemaupdater.go

@@ -7,7 +7,10 @@
 package db
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/syncthing/syncthing/lib/db/backend"
@@ -23,12 +26,14 @@ import (
 //   7: v0.14.53
 //   8-9: v1.4.0
 //   10-11: v1.6.0
-//   12: v1.7.0
+//   12-13: v1.7.0
 const (
-	dbVersion             = 12
+	dbVersion             = 13
 	dbMinSyncthingVersion = "v1.7.0"
 )
 
+var errFolderMissing = errors.New("folder present in global list but missing in keyer index")
+
 type databaseDowngradeError struct {
 	minSyncthingVersion string
 }
@@ -89,14 +94,14 @@ func (db *schemaUpdater) updateSchema() error {
 		{9, db.updateSchemaTo9},
 		{10, db.updateSchemaTo10},
 		{11, db.updateSchemaTo11},
-		{12, db.updateSchemaTo12},
+		{13, db.updateSchemaTo13},
 	}
 
 	for _, m := range migrations {
 		if prevVersion < m.schemaVersion {
 			l.Infof("Migrating database to schema version %d...", m.schemaVersion)
 			if err := m.migration(int(prevVersion)); err != nil {
-				return err
+				return fmt.Errorf("failed migrating to version %v: %w", m.schemaVersion, err)
 			}
 		}
 	}
@@ -128,8 +133,8 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 	symlinkConv := 0
 	changedFolders := make(map[string]struct{})
 	ignAdded := 0
-	meta := newMetadataTracker() // dummy metadata tracker
-	var gk, buf []byte
+	var gk []byte
+	ro := t.readOnlyTransaction
 
 	for dbi.Next() {
 		folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
@@ -155,17 +160,27 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			if _, ok := changedFolders[string(folder)]; !ok {
 				changedFolders[string(folder)] = struct{}{}
 			}
+			if err := t.Delete(dbi.Key()); err != nil {
+				return err
+			}
 			gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if err != nil {
 				return err
 			}
-			// Purposely pass nil file name to remove from global list,
-			// but don't touch meta and needs
-			buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil)
-			if err != nil && err != errEntryFromGlobalMissing {
+			fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
+			if backend.IsNotFound(err) {
+				// Shouldn't happen, but not critical.
+				continue
+			} else if err != nil {
 				return err
 			}
-			if err := t.Delete(dbi.Key()); err != nil {
+			_, _ = fl.pop(device)
+			if len(fl.Versions) == 0 {
+				err = t.Delete(gk)
+			} else {
+				err = t.Put(gk, mustMarshal(&fl))
+			}
+			if err != nil {
 				return err
 			}
 			continue
@@ -199,13 +214,41 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			if err != nil {
 				return err
 			}
-			if buf, ok, err = t.updateGlobal(gk, buf, folder, device, f, meta); err != nil {
+
+			fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
+			if err != nil && !backend.IsNotFound(err) {
 				return err
-			} else if ok {
-				if _, ok = changedFolders[string(folder)]; !ok {
-					changedFolders[string(folder)] = struct{}{}
+			}
+			i := 0
+			i = sort.Search(len(fl.Versions), func(j int) bool {
+				return fl.Versions[j].Invalid
+			})
+			for ; i < len(fl.Versions); i++ {
+				ordering := fl.Versions[i].Version.Compare(f.Version)
+				shouldInsert := ordering == protocol.Equal
+				if !shouldInsert {
+					shouldInsert, err = shouldInsertBefore(ordering, folder, fl.Versions[i].Device, true, f, ro)
+					if err != nil {
+						return err
+					}
 				}
-				ignAdded++
+				if shouldInsert {
+					nv := FileVersionDeprecated{
+						Device:  device,
+						Version: f.Version,
+						Invalid: true,
+					}
+					fl.insertAt(i, nv)
+					if err := t.Put(gk, mustMarshal(&fl)); err != nil {
+						return err
+					}
+					if _, ok := changedFolders[string(folder)]; !ok {
+						changedFolders[string(folder)] = struct{}{}
+					}
+					ignAdded++
+					break
+				}
+
 			}
 		}
 		if err := t.Checkpoint(); err != nil {
@@ -217,11 +260,6 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 		return err
 	}
 
-	for folder := range changedFolders {
-		if err := db.dropFolderMeta([]byte(folder)); err != nil {
-			return err
-		}
-	}
 	return t.Commit()
 }
 
@@ -239,7 +277,7 @@ func (db *schemaUpdater) updateSchema1to2(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f protocol.FileIntf) bool {
 			sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
 			if putErr != nil {
 				return false
@@ -274,7 +312,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := t.withGlobal(folder, nil, true, func(f FileIntf) bool {
+		err := withGlobalBefore11(folder, true, func(f protocol.FileIntf) bool {
 			name := []byte(f.FileName())
 			dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
 			if putErr != nil {
@@ -289,12 +327,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 			if ok {
 				v = haveFile.FileVersion()
 			}
-			fv := FileVersion{
+			fv := FileVersionDeprecated{
 				Version: f.FileVersion(),
 				Invalid: f.IsInvalid(),
 				Deleted: f.IsDeleted(),
 			}
-			if !need(fv, ok, v) {
+			if !needDeprecated(fv, ok, v) {
 				return true
 			}
 			nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
@@ -303,7 +341,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 			}
 			putErr = t.Put(nk, nil)
 			return putErr == nil
-		})
+		}, t.readOnlyTransaction)
 		if putErr != nil {
 			return putErr
 		}
@@ -359,7 +397,7 @@ func (db *schemaUpdater) updateSchema5to6(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var iterErr error
-		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f protocol.FileIntf) bool {
 			if !f.IsInvalid() {
 				return true
 			}
@@ -404,7 +442,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var delErr error
-		err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
+		err := withNeedLocalBefore11(folder, false, func(f protocol.FileIntf) bool {
 			name := []byte(f.FileName())
 			gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if delErr != nil {
@@ -421,20 +459,20 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 				delErr = t.Delete(key)
 				return delErr == nil
 			}
-			var fl VersionList
+			var fl VersionListDeprecated
 			err = fl.Unmarshal(svl)
 			if err != nil {
 				// This can't happen, but it's ignored everywhere else too,
 				// so lets not act on it.
 				return true
 			}
-			globalFV := FileVersion{
+			globalFV := FileVersionDeprecated{
 				Version: f.FileVersion(),
 				Invalid: f.IsInvalid(),
 				Deleted: f.IsDeleted(),
 			}
 
-			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) {
+			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !needDeprecated(globalFV, haveLocalFV, localFV.Version) {
 				key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
 				if err != nil {
 					delErr = err
@@ -443,7 +481,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 				delErr = t.Delete(key)
 			}
 			return delErr == nil
-		})
+		}, t.readOnlyTransaction)
 		if delErr != nil {
 			return delErr
 		}
@@ -480,6 +518,7 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
 	if err != nil {
 		return err
 	}
+	defer it.Release()
 	for it.Next() {
 		intf, err := t.unmarshalTrunc(it.Value(), false)
 		if backend.IsNotFound(err) {
@@ -510,6 +549,8 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
 }
 
 func (db *schemaUpdater) updateSchemaTo10(_ int) error {
+	// Rewrites global lists to include a Deleted flag.
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -533,7 +574,7 @@ func (db *schemaUpdater) updateSchemaTo10(_ int) error {
 		defer dbi.Release()
 
 		for dbi.Next() {
-			var vl VersionList
+			var vl VersionListDeprecated
 			if err := vl.Unmarshal(dbi.Value()); err != nil {
 				return err
 			}
@@ -592,7 +633,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi protocol.FileIntf) bool {
 			f := fi.(FileInfoTruncated)
 			if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() || f.IsInvalid() || f.BlocksHash == nil {
 				return true
@@ -620,7 +661,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
 	return t.Commit()
 }
 
-func (db *schemaUpdater) updateSchemaTo12(_ int) error {
+func (db *schemaUpdater) updateSchemaTo13(prev int) error {
 	// Loads and rewrites all files, to deduplicate version vectors.
 
 	t, err := db.newReadWriteTransaction()
@@ -629,9 +670,280 @@ func (db *schemaUpdater) updateSchemaTo12(_ int) error {
 	}
 	defer t.close()
 
-	if err := db.rewriteFiles(t); err != nil {
+	if prev < 12 {
+		if err := db.rewriteFiles(t); err != nil {
+			return err
+		}
+	}
+
+	if err := db.rewriteGlobals(t); err != nil {
 		return err
 	}
 
 	return t.Commit()
 }
+
+func (db *schemaUpdater) rewriteGlobals(t readWriteTransaction) error {
+	it, err := t.NewPrefixIterator([]byte{KeyTypeGlobal})
+	if err != nil {
+		return err
+	}
+	defer it.Release()
+	for it.Next() {
+		var vl VersionListDeprecated
+		if err := vl.Unmarshal(it.Value()); err != nil {
+			// If we crashed during an earlier migration, some version
+			// lists might already be in the new format: Skip those.
+			var nvl VersionList
+			if nerr := nvl.Unmarshal(it.Value()); nerr == nil {
+				continue
+			}
+			return err
+		}
+		if len(vl.Versions) == 0 {
+			if err := t.Delete(it.Key()); err != nil {
+				return err
+			}
+		}
+
+		newVl, err := convertVersionList(vl)
+		if err != nil {
+			return err
+		}
+		if err := t.Put(it.Key(), mustMarshal(&newVl)); err != nil {
+			return err
+		}
+		if err := t.Checkpoint(); err != nil {
+			return err
+		}
+	}
+	it.Release()
+	return it.Error()
+}
+
+func convertVersionList(vl VersionListDeprecated) (VersionList, error) {
+	var newVl VersionList
+	var newPos, oldPos int
+	var lastVersion protocol.Vector
+
+	for _, fv := range vl.Versions {
+		if fv.Invalid {
+			break
+		}
+		oldPos++
+		if lastVersion.Equal(fv.Version) {
+			newVl.RawVersions[newPos].Devices = append(newVl.RawVersions[newPos].Devices, fv.Device)
+			continue
+		}
+		newPos = len(newVl.RawVersions)
+		newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, false, fv.Deleted))
+		lastVersion = fv.Version
+	}
+
+	if oldPos == len(vl.Versions) {
+		return newVl, nil
+	}
+
+	if len(newVl.RawVersions) == 0 {
+		fv := vl.Versions[oldPos]
+		newVl.RawVersions = []FileVersion{newFileVersion(fv.Device, fv.Version, true, fv.Deleted)}
+	}
+	newPos = 0
+outer:
+	for _, fv := range vl.Versions[oldPos:] {
+		for _, nfv := range newVl.RawVersions[newPos:] {
+			switch nfv.Version.Compare(fv.Version) {
+			case protocol.Equal:
+				newVl.RawVersions[newPos].InvalidDevices = append(newVl.RawVersions[newPos].InvalidDevices, fv.Device)
+				lastVersion = fv.Version
+				continue outer
+			case protocol.Lesser:
+				newVl.insertAt(newPos, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
+				lastVersion = fv.Version
+				continue outer
+			case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
+				// The version is invalid, i.e. it looses anyway,
+				// no need to check/get the conflicting file.
+			}
+			newPos++
+		}
+		// Couldn't insert into any existing versions
+		newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
+		lastVersion = fv.Version
+		newPos++
+	}
+
+	return newVl, nil
+}
+
+func getGlobalVersionsByKeyBefore11(key []byte, t readOnlyTransaction) (VersionListDeprecated, error) {
+	bs, err := t.Get(key)
+	if err != nil {
+		return VersionListDeprecated{}, err
+	}
+
+	var vl VersionListDeprecated
+	if err := vl.Unmarshal(bs); err != nil {
+		return VersionListDeprecated{}, err
+	}
+
+	return vl, nil
+}
+
+func withGlobalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
+	key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var dk []byte
+	for dbi.Next() {
+		name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
+
+		var vl VersionListDeprecated
+		if err := vl.Unmarshal(dbi.Value()); err != nil {
+			return err
+		}
+
+		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
+		if err != nil {
+			return err
+		}
+
+		f, ok, err := t.getFileTrunc(dk, truncate)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+
+		if !fn(f) {
+			return nil
+		}
+	}
+	if err != nil {
+		return err
+	}
+	return dbi.Error()
+}
+
+func withNeedLocalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
+	key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key.WithoutName())
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var keyBuf []byte
+	var f protocol.FileIntf
+	var ok bool
+	for dbi.Next() {
+		keyBuf, f, ok, err = getGlobalBefore11(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate, t)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}
+
+func getGlobalBefore11(keyBuf, folder, file []byte, truncate bool, t readOnlyTransaction) ([]byte, protocol.FileIntf, bool, error) {
+	keyBuf, err := t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	bs, err := t.Get(keyBuf)
+	if backend.IsNotFound(err) {
+		return keyBuf, nil, false, nil
+	} else if err != nil {
+		return nil, nil, false, err
+	}
+	var vl VersionListDeprecated
+	if err := vl.Unmarshal(bs); err != nil {
+		return nil, nil, false, err
+	}
+	if len(vl.Versions) == 0 {
+		return nil, nil, false, nil
+	}
+	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	fi, ok, err := t.getFileTrunc(keyBuf, truncate)
+	if err != nil || !ok {
+		return keyBuf, nil, false, err
+	}
+	return keyBuf, fi, true, nil
+}
+
+func (vl *VersionListDeprecated) String() string {
+	var b bytes.Buffer
+	var id protocol.DeviceID
+	b.WriteString("{")
+	for i, v := range vl.Versions {
+		if i > 0 {
+			b.WriteString(", ")
+		}
+		copy(id[:], v.Device)
+		fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
+	}
+	b.WriteString("}")
+	return b.String()
+}
+
+func (vl *VersionListDeprecated) pop(device []byte) (FileVersionDeprecated, int) {
+	for i, v := range vl.Versions {
+		if bytes.Equal(v.Device, device) {
+			vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
+			return v, i
+		}
+	}
+	return FileVersionDeprecated{}, -1
+}
+
+func (vl *VersionListDeprecated) Get(device []byte) (FileVersionDeprecated, bool) {
+	for _, v := range vl.Versions {
+		if bytes.Equal(v.Device, device) {
+			return v, true
+		}
+	}
+
+	return FileVersionDeprecated{}, false
+}
+
+func (vl *VersionListDeprecated) insertAt(i int, v FileVersionDeprecated) {
+	vl.Versions = append(vl.Versions, FileVersionDeprecated{})
+	copy(vl.Versions[i+1:], vl.Versions[i:])
+	vl.Versions[i] = v
+}
+
+func needDeprecated(global FileVersionDeprecated, haveLocal bool, localVersion protocol.Vector) bool {
+	// We never need an invalid file.
+	if global.Invalid {
+		return false
+	}
+	// We don't need a deleted file if we don't have it.
+	if global.Deleted && !haveLocal {
+		return false
+	}
+	// We don't need the global file if we already have the same version.
+	if haveLocal && localVersion.GreaterEqual(global.Version) {
+		return false
+	}
+	return true
+}