all: Transactionalize db.FileSet (fixes #5952) (#6239)

lib/db/benchmark_test.go

@@ -145,10 +145,12 @@ func BenchmarkNeedHalf(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		benchS.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap := benchS.Snapshot()
+		snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(secondHalf) {
 			b.Errorf("wrong length %d != %d", count, len(secondHalf))
 		}
@@ -167,10 +169,12 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		fset.WithNeed(remoteDevice0, func(fi db.FileIntf) bool {
+		snap := fset.Snapshot()
+		snap.WithNeed(remoteDevice0, func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(secondHalf) {
 			b.Errorf("wrong length %d != %d", count, len(secondHalf))
 		}
@@ -186,10 +190,12 @@ func BenchmarkHave(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		benchS.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap := benchS.Snapshot()
+		snap.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(firstHalf) {
 			b.Errorf("wrong length %d != %d", count, len(firstHalf))
 		}
@@ -205,10 +211,12 @@ func BenchmarkGlobal(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		benchS.WithGlobal(func(fi db.FileIntf) bool {
+		snap := benchS.Snapshot()
+		snap.WithGlobal(func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(files) {
 			b.Errorf("wrong length %d != %d", count, len(files))
 		}
@@ -224,10 +232,12 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		benchS.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap := benchS.Snapshot()
+		snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(secondHalf) {
 			b.Errorf("wrong length %d != %d", count, len(secondHalf))
 		}
@@ -243,10 +253,12 @@ func BenchmarkHaveTruncated(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		benchS.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap := benchS.Snapshot()
+		snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(firstHalf) {
 			b.Errorf("wrong length %d != %d", count, len(firstHalf))
 		}
@@ -262,10 +274,12 @@ func BenchmarkGlobalTruncated(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		count := 0
-		benchS.WithGlobalTruncated(func(fi db.FileIntf) bool {
+		snap := benchS.Snapshot()
+		snap.WithGlobalTruncated(func(fi db.FileIntf) bool {
 			count++
 			return true
 		})
+		snap.Release()
 		if count != len(files) {
 			b.Errorf("wrong length %d != %d", count, len(files))
 		}

lib/db/db_test.go

@@ -72,7 +72,9 @@ func TestIgnoredFiles(t *testing.T) {
 	// Local files should have the "ignored" bit in addition to just being
 	// generally invalid if we want to look at the simulation of that bit.
 
-	fi, ok := fs.Get(protocol.LocalDeviceID, "foo")
+	snap := fs.Snapshot()
+	defer snap.Release()
+	fi, ok := snap.Get(protocol.LocalDeviceID, "foo")
 	if !ok {
 		t.Fatal("foo should exist")
 	}
@@ -83,7 +85,7 @@ func TestIgnoredFiles(t *testing.T) {
 		t.Error("foo should be ignored")
 	}
 
-	fi, ok = fs.Get(protocol.LocalDeviceID, "bar")
+	fi, ok = snap.Get(protocol.LocalDeviceID, "bar")
 	if !ok {
 		t.Fatal("bar should exist")
 	}
@@ -97,7 +99,7 @@ func TestIgnoredFiles(t *testing.T) {
 	// Remote files have the invalid bit as usual, and the IsInvalid() method
 	// should pick this up too.
 
-	fi, ok = fs.Get(protocol.DeviceID{42}, "baz")
+	fi, ok = snap.Get(protocol.DeviceID{42}, "baz")
 	if !ok {
 		t.Fatal("baz should exist")
 	}
@@ -108,7 +110,7 @@ func TestIgnoredFiles(t *testing.T) {
 		t.Error("baz should be invalid")
 	}
 
-	fi, ok = fs.Get(protocol.DeviceID{42}, "quux")
+	fi, ok = snap.Get(protocol.DeviceID{42}, "quux")
 	if !ok {
 		t.Fatal("quux should exist")
 	}
@@ -167,7 +169,11 @@ func TestUpdate0to3(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, ok, err := db.getFileDirty(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil {
+	trans, err := db.newReadOnlyTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok, err := trans.getFile(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil {
 		t.Fatal(err)
 	} else if ok {
 		t.Error("File prefixed by '/' was not removed during transition to schema 1")
@@ -186,7 +192,11 @@ func TestUpdate0to3(t *testing.T) {
 	}
 
 	found := false
-	_ = db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
+	trans, err = db.newReadOnlyTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = trans.withHaveSequence(folder, 0, func(fi FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		l.Infoln(f)
 		if found {
@@ -213,7 +223,11 @@ func TestUpdate0to3(t *testing.T) {
 		haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
 		haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
 	}
-	_ = db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
+	trans, err = db.newReadOnlyTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = trans.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
 		e, ok := need[fi.FileName()]
 		if !ok {
 			t.Error("Got unexpected needed file:", fi.FileName())

lib/db/lowlevel.go

@@ -194,405 +194,6 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
 	return t.commit()
 }
 
-func (db *Lowlevel) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	if len(prefix) > 0 {
-		unslashedPrefix := prefix
-		if bytes.HasSuffix(prefix, []byte{'/'}) {
-			unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
-		} else {
-			prefix = append(prefix, '/')
-		}
-
-		key, err := db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix)
-		if err != nil {
-			return err
-		}
-		if f, ok, err := t.getFileTrunc(key, true); err != nil {
-			return err
-		} else if ok && !fn(f) {
-			return nil
-		}
-	}
-
-	key, err := db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)
-	if err != nil {
-		return err
-	}
-	dbi, err := t.NewPrefixIterator(key)
-	if err != nil {
-		return err
-	}
-	defer dbi.Release()
-
-	for dbi.Next() {
-		name := db.keyer.NameFromDeviceFileKey(dbi.Key())
-		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
-			return nil
-		}
-
-		f, err := unmarshalTrunc(dbi.Value(), truncate)
-		if err != nil {
-			l.Debugln("unmarshal error:", err)
-			continue
-		}
-		if !fn(f) {
-			return nil
-		}
-	}
-	return dbi.Error()
-}
-
-func (db *Lowlevel) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	first, err := db.keyer.GenerateSequenceKey(nil, folder, startSeq)
-	if err != nil {
-		return err
-	}
-	last, err := db.keyer.GenerateSequenceKey(nil, folder, maxInt64)
-	if err != nil {
-		return err
-	}
-	dbi, err := t.NewRangeIterator(first, last)
-	if err != nil {
-		return err
-	}
-	defer dbi.Release()
-
-	for dbi.Next() {
-		f, ok, err := t.getFileByKey(dbi.Value())
-		if err != nil {
-			return err
-		}
-		if !ok {
-			l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key()))
-			continue
-		}
-
-		if shouldDebug() {
-			if seq := db.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq {
-				l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq)
-				panic("sequence index corruption")
-			}
-		}
-		if !fn(f) {
-			return nil
-		}
-	}
-	return dbi.Error()
-}
-
-func (db *Lowlevel) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error {
-	t, err := db.newReadWriteTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	key, err := db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil)
-	if err != nil {
-		return err
-	}
-	dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice())
-	if err != nil {
-		return err
-	}
-	defer dbi.Release()
-
-	var gk, keyBuf []byte
-	for dbi.Next() {
-		device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
-		if !ok {
-			// Not having the device in the index is bad. Clear it.
-			if err := t.Delete(dbi.Key()); err != nil {
-				return err
-			}
-			continue
-		}
-		var f FileInfoTruncated
-		// The iterator function may keep a reference to the unmarshalled
-		// struct, which in turn references the buffer it was unmarshalled
-		// from. dbi.Value() just returns an internal slice that it reuses, so
-		// we need to copy it.
-		err := f.Unmarshal(append([]byte{}, dbi.Value()...))
-		if err != nil {
-			return err
-		}
-
-		switch f.Name {
-		case "", ".", "..", "/": // A few obviously invalid filenames
-			l.Infof("Dropping invalid filename %q from database", f.Name)
-			name := []byte(f.Name)
-			gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
-			if err != nil {
-				return err
-			}
-			keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil)
-			if err != nil {
-				return err
-			}
-			if err := t.Delete(dbi.Key()); err != nil {
-				return err
-			}
-			continue
-		}
-
-		if !fn(device, f) {
-			return nil
-		}
-	}
-	if err := dbi.Error(); err != nil {
-		return err
-	}
-	return t.commit()
-}
-
-func (db *Lowlevel) getFileDirty(folder, device, file []byte) (protocol.FileInfo, bool, error) {
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return protocol.FileInfo{}, false, err
-	}
-	defer t.close()
-	return t.getFile(folder, device, file)
-}
-
-func (db *Lowlevel) getGlobalDirty(folder, file []byte, truncate bool) (FileIntf, bool, error) {
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return nil, false, err
-	}
-	defer t.close()
-	_, f, ok, err := t.getGlobal(nil, folder, file, truncate)
-	return f, ok, err
-}
-
-func (db *Lowlevel) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error {
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	if len(prefix) > 0 {
-		unslashedPrefix := prefix
-		if bytes.HasSuffix(prefix, []byte{'/'}) {
-			unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
-		} else {
-			prefix = append(prefix, '/')
-		}
-
-		if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil {
-			return err
-		} else if ok && !fn(f) {
-			return nil
-		}
-	}
-
-	key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)
-	if err != nil {
-		return err
-	}
-	dbi, err := t.NewPrefixIterator(key)
-	if err != nil {
-		return err
-	}
-	defer dbi.Release()
-
-	var dk []byte
-	for dbi.Next() {
-		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
-		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
-			return nil
-		}
-
-		vl, ok := unmarshalVersionList(dbi.Value())
-		if !ok {
-			continue
-		}
-
-		dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
-		if err != nil {
-			return err
-		}
-
-		f, ok, err := t.getFileTrunc(dk, truncate)
-		if err != nil {
-			return err
-		}
-		if !ok {
-			continue
-		}
-
-		if !fn(f) {
-			return nil
-		}
-	}
-	if err != nil {
-		return err
-	}
-	return dbi.Error()
-}
-
-func (db *Lowlevel) availability(folder, file []byte) ([]protocol.DeviceID, error) {
-	k, err := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
-	if err != nil {
-		return nil, err
-	}
-	bs, err := db.Get(k)
-	if backend.IsNotFound(err) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	vl, ok := unmarshalVersionList(bs)
-	if !ok {
-		return nil, nil
-	}
-
-	var devices []protocol.DeviceID
-	for _, v := range vl.Versions {
-		if !v.Version.Equal(vl.Versions[0].Version) {
-			break
-		}
-		if v.Invalid {
-			continue
-		}
-		n := protocol.DeviceIDFromBytes(v.Device)
-		devices = append(devices, n)
-	}
-
-	return devices, nil
-}
-
-func (db *Lowlevel) withNeed(folder, device []byte, truncate bool, fn Iterator) error {
-	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
-		return db.withNeedLocal(folder, truncate, fn)
-	}
-
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, nil)
-	if err != nil {
-		return err
-	}
-	dbi, err := t.NewPrefixIterator(key.WithoutName())
-	if err != nil {
-		return err
-	}
-	defer dbi.Release()
-
-	var dk []byte
-	devID := protocol.DeviceIDFromBytes(device)
-	for dbi.Next() {
-		vl, ok := unmarshalVersionList(dbi.Value())
-		if !ok {
-			continue
-		}
-
-		haveFV, have := vl.Get(device)
-		// XXX: This marks Concurrent (i.e. conflicting) changes as
-		// needs. Maybe we should do that, but it needs special
-		// handling in the puller.
-		if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) {
-			continue
-		}
-
-		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
-		needVersion := vl.Versions[0].Version
-		needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
-
-		for i := range vl.Versions {
-			if !vl.Versions[i].Version.Equal(needVersion) {
-				// We haven't found a valid copy of the file with the needed version.
-				break
-			}
-
-			if vl.Versions[i].Invalid {
-				// The file is marked invalid, don't use it.
-				continue
-			}
-
-			dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[i].Device, name)
-			if err != nil {
-				return err
-			}
-			gf, ok, err := t.getFileTrunc(dk, truncate)
-			if err != nil {
-				return err
-			}
-			if !ok {
-				continue
-			}
-
-			if gf.IsDeleted() && !have {
-				// We don't need deleted files that we don't have
-				break
-			}
-
-			l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice)
-
-			if !fn(gf) {
-				return nil
-			}
-
-			// This file is handled, no need to look further in the version list
-			break
-		}
-	}
-	return dbi.Error()
-}
-
-func (db *Lowlevel) withNeedLocal(folder []byte, truncate bool, fn Iterator) error {
-	t, err := db.newReadOnlyTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	key, err := db.keyer.GenerateNeedFileKey(nil, folder, nil)
-	if err != nil {
-		return err
-	}
-	dbi, err := t.NewPrefixIterator(key.WithoutName())
-	if err != nil {
-		return err
-	}
-	defer dbi.Release()
-
-	var keyBuf []byte
-	var f FileIntf
-	var ok bool
-	for dbi.Next() {
-		keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
-		if err != nil {
-			return err
-		}
-		if !ok {
-			continue
-		}
-		if !fn(f) {
-			return nil
-		}
-	}
-	return dbi.Error()
-}
-
 func (db *Lowlevel) dropFolder(folder []byte) error {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {

lib/db/meta.go

@@ -15,12 +15,16 @@ import (
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
-// metadataTracker keeps metadata on a per device, per local flag basis.
-type metadataTracker struct {
-	mut     sync.RWMutex
+type countsMap struct {
 	counts  CountsSet
 	indexes map[metaKey]int // device ID + local flags -> index in counts
-	dirty   bool
+}
+
+// metadataTracker keeps metadata on a per device, per local flag basis.
+type metadataTracker struct {
+	countsMap
+	mut   sync.RWMutex
+	dirty bool
 }
 
 type metaKey struct {
@@ -30,8 +34,10 @@ type metaKey struct {
 
 func newMetadataTracker() *metadataTracker {
 	return &metadataTracker{
-		mut:     sync.NewRWMutex(),
-		indexes: make(map[metaKey]int),
+		mut: sync.NewRWMutex(),
+		countsMap: countsMap{
+			indexes: make(map[metaKey]int),
+		},
 	}
 }
 
@@ -49,7 +55,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error {
 	return nil
 }
 
-// Unmarshal returns the protobuf representation of the metadataTracker
+// Marshal returns the protobuf representation of the metadataTracker
 func (m *metadataTracker) Marshal() ([]byte, error) {
 	return m.counts.Marshal()
 }
@@ -260,16 +266,11 @@ func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
 	m.mut.Unlock()
 }
 
-// Counts returns the counts for the given device ID and flag. `flag` should
-// be zero or have exactly one bit set.
-func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
+func (m *countsMap) Counts(dev protocol.DeviceID, flag uint32) Counts {
 	if bits.OnesCount32(flag) > 1 {
 		panic("incorrect usage: set at most one bit in flag")
 	}
 
-	m.mut.RLock()
-	defer m.mut.RUnlock()
-
 	idx, ok := m.indexes[metaKey{dev, flag}]
 	if !ok {
 		return Counts{}
@@ -278,6 +279,37 @@ func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
 	return m.counts.Counts[idx]
 }
 
+// Counts returns the counts for the given device ID and flag. `flag` should
+// be zero or have exactly one bit set.
+func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
+	m.mut.RLock()
+	defer m.mut.RUnlock()
+
+	return m.countsMap.Counts(dev, flag)
+}
+
+// Snapshot returns a copy of the metadata for reading.
+func (m *metadataTracker) Snapshot() *countsMap {
+	m.mut.RLock()
+	defer m.mut.RUnlock()
+
+	c := &countsMap{
+		counts: CountsSet{
+			Counts:  make([]Counts, len(m.counts.Counts)),
+			Created: m.counts.Created,
+		},
+		indexes: make(map[metaKey]int, len(m.indexes)),
+	}
+	for k, v := range m.indexes {
+		c.indexes[k] = v
+	}
+	for i := range m.counts.Counts {
+		c.counts.Counts[i] = m.counts.Counts[i]
+	}
+
+	return c
+}
+
 // nextLocalSeq allocates a new local sequence number
 func (m *metadataTracker) nextLocalSeq() int64 {
 	m.mut.Lock()
@@ -291,26 +323,25 @@ func (m *metadataTracker) nextLocalSeq() int64 {
 // devices returns the list of devices tracked, excluding the local device
 // (which we don't know the ID of)
 func (m *metadataTracker) devices() []protocol.DeviceID {
-	devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts))
-
 	m.mut.RLock()
+	defer m.mut.RUnlock()
+	return m.countsMap.devices()
+}
+
+func (m *countsMap) devices() []protocol.DeviceID {
+	devs := make([]protocol.DeviceID, 0, len(m.counts.Counts))
+
 	for _, dev := range m.counts.Counts {
 		if dev.Sequence > 0 {
 			id := protocol.DeviceIDFromBytes(dev.DeviceID)
 			if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID {
 				continue
 			}
-			devs[id] = struct{}{}
+			devs = append(devs, id)
 		}
 	}
-	m.mut.RUnlock()
 
-	devList := make([]protocol.DeviceID, 0, len(devs))
-	for dev := range devs {
-		devList = append(devList, dev)
-	}
-
-	return devList
+	return devs
 }
 
 func (m *metadataTracker) Created() time.Time {

lib/db/schemaupdater.go

@@ -228,7 +228,7 @@ func (db *schemaUpdater) updateSchema1to2() error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
 			sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
 			if putErr != nil {
 				return false
@@ -263,7 +263,7 @@ func (db *schemaUpdater) updateSchema2to3() error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := db.withGlobal(folder, nil, true, func(f FileIntf) bool {
+		err := t.withGlobal(folder, nil, true, func(f FileIntf) bool {
 			name := []byte(f.FileName())
 			dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
 			if putErr != nil {
@@ -339,7 +339,7 @@ func (db *schemaUpdater) updateSchema5to6() error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := db.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
 			if !f.IsInvalid() {
 				return true
 			}
@@ -382,7 +382,7 @@ func (db *schemaUpdater) updateSchema6to7() error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var delErr error
-		err := db.withNeedLocal(folder, false, func(f FileIntf) bool {
+		err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
 			name := []byte(f.FileName())
 			global := f.(protocol.FileInfo)
 			gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)

lib/db/set.go

@@ -105,8 +105,12 @@ func (s *FileSet) recalcCounts() error {
 		return err
 	}
 
+	t, err := s.db.newReadWriteTransaction()
+	if err != nil {
+		return err
+	}
 	var deviceID protocol.DeviceID
-	err := s.db.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool {
+	err = t.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool {
 		copy(deviceID[:], device)
 		s.meta.addFile(deviceID, f)
 		return true
@@ -183,75 +187,97 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 	}
 }
 
-func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
+type Snapshot struct {
+	folder string
+	t      readOnlyTransaction
+	meta   *countsMap
+}
+
+func (s *FileSet) Snapshot() *Snapshot {
+	t, err := s.db.newReadOnlyTransaction()
+	if err != nil {
+		panic(err)
+	}
+	return &Snapshot{
+		folder: s.folder,
+		t:      t,
+		meta:   s.meta.Snapshot(),
+	}
+}
+
+func (s *Snapshot) Release() {
+	s.t.close()
+}
+
+func (s *Snapshot) WithNeed(device protocol.DeviceID, fn Iterator) {
 	l.Debugf("%s WithNeed(%v)", s.folder, device)
-	if err := s.db.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
+func (s *Snapshot) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
 	l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
-	if err := s.db.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) WithHave(device protocol.DeviceID, fn Iterator) {
+func (s *Snapshot) WithHave(device protocol.DeviceID, fn Iterator) {
 	l.Debugf("%s WithHave(%v)", s.folder, device)
-	if err := s.db.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
+func (s *Snapshot) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
 	l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
-	if err := s.db.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) WithHaveSequence(startSeq int64, fn Iterator) {
+func (s *Snapshot) WithHaveSequence(startSeq int64, fn Iterator) {
 	l.Debugf("%s WithHaveSequence(%v)", s.folder, startSeq)
-	if err := s.db.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
 // Except for an item with a path equal to prefix, only children of prefix are iterated.
 // E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not.
-func (s *FileSet) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) {
+func (s *Snapshot) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) {
 	l.Debugf(`%s WithPrefixedHaveTruncated(%v, "%v")`, s.folder, device, prefix)
-	if err := s.db.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) WithGlobal(fn Iterator) {
+func (s *Snapshot) WithGlobal(fn Iterator) {
 	l.Debugf("%s WithGlobal()", s.folder)
-	if err := s.db.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) WithGlobalTruncated(fn Iterator) {
+func (s *Snapshot) WithGlobalTruncated(fn Iterator) {
 	l.Debugf("%s WithGlobalTruncated()", s.folder)
-	if err := s.db.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
 // Except for an item with a path equal to prefix, only children of prefix are iterated.
 // E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not.
-func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
+func (s *Snapshot) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
 	l.Debugf(`%s WithPrefixedGlobalTruncated("%v")`, s.folder, prefix)
-	if err := s.db.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
+	if err := s.t.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
 		panic(err)
 	}
 }
 
-func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
-	f, ok, err := s.db.getFileDirty([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
+func (s *Snapshot) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
+	f, ok, err := s.t.getFile([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
 	if backend.IsClosed(err) {
 		return protocol.FileInfo{}, false
 	} else if err != nil {
@@ -261,8 +287,8 @@ func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo,
 	return f, ok
 }
 
-func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) {
-	fi, ok, err := s.db.getGlobalDirty([]byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
+func (s *Snapshot) GetGlobal(file string) (protocol.FileInfo, bool) {
+	_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
 	if backend.IsClosed(err) {
 		return protocol.FileInfo{}, false
 	} else if err != nil {
@@ -276,8 +302,8 @@ func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) {
 	return f, true
 }
 
-func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
-	fi, ok, err := s.db.getGlobalDirty([]byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
+func (s *Snapshot) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
+	_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
 	if backend.IsClosed(err) {
 		return FileInfoTruncated{}, false
 	} else if err != nil {
@@ -291,8 +317,8 @@ func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
 	return f, true
 }
 
-func (s *FileSet) Availability(file string) []protocol.DeviceID {
-	av, err := s.db.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file)))
+func (s *Snapshot) Availability(file string) []protocol.DeviceID {
+	av, err := s.t.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file)))
 	if backend.IsClosed(err) {
 		return nil
 	} else if err != nil {
@@ -301,26 +327,110 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID {
 	return av
 }
 
-func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
-	return s.meta.Sequence(device)
+func (s *Snapshot) Sequence(device protocol.DeviceID) int64 {
+	return s.meta.Counts(device, 0).Sequence
 }
 
-func (s *FileSet) LocalSize() Counts {
+// RemoteSequence returns the change version for the given folder, as
+// sent by remote peers. This is guaranteed to increment if the contents of
+// the remote or global folder has changed.
+func (s *Snapshot) RemoteSequence() int64 {
+	var ver int64
+
+	for _, device := range s.meta.devices() {
+		ver += s.Sequence(device)
+	}
+
+	return ver
+}
+
+func (s *Snapshot) LocalSize() Counts {
 	local := s.meta.Counts(protocol.LocalDeviceID, 0)
-	recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
-	return local.Add(recvOnlyChanged)
+	return local.Add(s.ReceiveOnlyChangedSize())
 }
 
-func (s *FileSet) ReceiveOnlyChangedSize() Counts {
+func (s *Snapshot) ReceiveOnlyChangedSize() Counts {
 	return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
 }
 
-func (s *FileSet) GlobalSize() Counts {
+func (s *Snapshot) GlobalSize() Counts {
 	global := s.meta.Counts(protocol.GlobalDeviceID, 0)
 	recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly)
 	return global.Add(recvOnlyChanged)
 }
 
+func (s *Snapshot) NeedSize() Counts {
+	var result Counts
+	s.WithNeedTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
+		switch {
+		case f.IsDeleted():
+			result.Deleted++
+		case f.IsDirectory():
+			result.Directories++
+		case f.IsSymlink():
+			result.Symlinks++
+		default:
+			result.Files++
+			result.Bytes += f.FileSize()
+		}
+		return true
+	})
+	return result
+}
+
+// LocalChangedFiles returns a paginated list of currently needed files in
+// progress, queued, and to be queued on next puller iteration, as well as the
+// total number of files currently needed.
+func (s *Snapshot) LocalChangedFiles(page, perpage int) []FileInfoTruncated {
+	if s.ReceiveOnlyChangedSize().TotalItems() == 0 {
+		return nil
+	}
+
+	files := make([]FileInfoTruncated, 0, perpage)
+
+	skip := (page - 1) * perpage
+	get := perpage
+
+	s.WithHaveTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
+		if !f.IsReceiveOnlyChanged() {
+			return true
+		}
+		if skip > 0 {
+			skip--
+			return true
+		}
+		ft := f.(FileInfoTruncated)
+		files = append(files, ft)
+		get--
+		return get > 0
+	})
+
+	return files
+}
+
+// RemoteNeedFolderFiles returns paginated list of currently needed files in
+// progress, queued, and to be queued on next puller iteration, as well as the
+// total number of files currently needed.
+func (s *Snapshot) RemoteNeedFolderFiles(device protocol.DeviceID, page, perpage int) []FileInfoTruncated {
+	files := make([]FileInfoTruncated, 0, perpage)
+	skip := (page - 1) * perpage
+	get := perpage
+	s.WithNeedTruncated(device, func(f FileIntf) bool {
+		if skip > 0 {
+			skip--
+			return true
+		}
+		files = append(files, f.(FileInfoTruncated))
+		get--
+		return get > 0
+	})
+	return files
+}
+
+func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
+	return s.meta.Sequence(device)
+}
+
 func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {
 	id, err := s.db.getIndexID(device[:], []byte(s.folder))
 	if backend.IsClosed(err) {

lib/db/set_test.go

@@ -46,7 +46,9 @@ func genBlocks(n int) []protocol.BlockInfo {
 
 func globalList(s *db.FileSet) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithGlobal(func(fi db.FileIntf) bool {
+	snap := s.Snapshot()
+	defer snap.Release()
+	snap.WithGlobal(func(fi db.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -55,7 +57,9 @@ func globalList(s *db.FileSet) []protocol.FileInfo {
 }
 func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated {
 	var fs []db.FileInfoTruncated
-	s.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
+	snap := s.Snapshot()
+	defer snap.Release()
+	snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
 		fs = append(fs, f)
 		return true
@@ -65,7 +69,9 @@ func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated {
 
 func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithHave(n, func(fi db.FileIntf) bool {
+	snap := s.Snapshot()
+	defer snap.Release()
+	snap.WithHave(n, func(fi db.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -75,7 +81,9 @@ func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
 
 func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.FileInfoTruncated {
 	var fs []db.FileInfoTruncated
-	s.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool {
+	snap := s.Snapshot()
+	defer snap.Release()
+	snap.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
 		fs = append(fs, f)
 		return true
@@ -85,7 +93,9 @@ func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.Fi
 
 func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
-	s.WithNeed(n, func(fi db.FileIntf) bool {
+	snap := s.Snapshot()
+	defer snap.Release()
+	snap.WithNeed(n, func(fi db.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -206,7 +216,7 @@ func TestGlobalSet(t *testing.T) {
 		}
 		globalBytes += f.FileSize()
 	}
-	gs := m.GlobalSize()
+	gs := globalSize(m)
 	if gs.Files != globalFiles {
 		t.Errorf("Incorrect GlobalSize files; %d != %d", gs.Files, globalFiles)
 	}
@@ -242,7 +252,7 @@ func TestGlobalSet(t *testing.T) {
 		}
 		haveBytes += f.FileSize()
 	}
-	ls := m.LocalSize()
+	ls := localSize(m)
 	if ls.Files != haveFiles {
 		t.Errorf("Incorrect LocalSize files; %d != %d", ls.Files, haveFiles)
 	}
@@ -277,7 +287,9 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed)
 	}
 
-	f, ok := m.Get(protocol.LocalDeviceID, "b")
+	snap := m.Snapshot()
+	defer snap.Release()
+	f, ok := snap.Get(protocol.LocalDeviceID, "b")
 	if !ok {
 		t.Error("Unexpectedly not OK")
 	}
@@ -285,7 +297,7 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
 	}
 
-	f, ok = m.Get(remoteDevice0, "b")
+	f, ok = snap.Get(remoteDevice0, "b")
 	if !ok {
 		t.Error("Unexpectedly not OK")
 	}
@@ -293,7 +305,7 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
 	}
 
-	f, ok = m.GetGlobal("b")
+	f, ok = snap.GetGlobal("b")
 	if !ok {
 		t.Error("Unexpectedly not OK")
 	}
@@ -301,7 +313,7 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
 	}
 
-	f, ok = m.Get(protocol.LocalDeviceID, "zz")
+	f, ok = snap.Get(protocol.LocalDeviceID, "zz")
 	if ok {
 		t.Error("Unexpectedly OK")
 	}
@@ -309,7 +321,7 @@ func TestGlobalSet(t *testing.T) {
 		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
 	}
 
-	f, ok = m.GetGlobal("zz")
+	f, ok = snap.GetGlobal("zz")
 	if ok {
 		t.Error("Unexpectedly OK")
 	}
@@ -318,15 +330,15 @@ func TestGlobalSet(t *testing.T) {
 	}
 
 	av := []protocol.DeviceID{protocol.LocalDeviceID, remoteDevice0}
-	a := m.Availability("a")
+	a := snap.Availability("a")
 	if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
 		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
 	}
-	a = m.Availability("b")
+	a = snap.Availability("b")
 	if len(a) != 1 || a[0] != remoteDevice0 {
 		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteDevice0)
 	}
-	a = m.Availability("d")
+	a = snap.Availability("d")
 	if len(a) != 1 || a[0] != protocol.LocalDeviceID {
 		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalDeviceID)
 	}
@@ -446,19 +458,22 @@ func TestInvalidAvailability(t *testing.T) {
 	replace(s, remoteDevice0, remote0Have)
 	replace(s, remoteDevice1, remote1Have)
 
-	if av := s.Availability("both"); len(av) != 2 {
+	snap := s.Snapshot()
+	defer snap.Release()
+
+	if av := snap.Availability("both"); len(av) != 2 {
 		t.Error("Incorrect availability for 'both':", av)
 	}
 
-	if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 {
+	if av := snap.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 {
 		t.Error("Incorrect availability for 'r0only':", av)
 	}
 
-	if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 {
+	if av := snap.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 {
 		t.Error("Incorrect availability for 'r1only':", av)
 	}
 
-	if av := s.Availability("none"); len(av) != 0 {
+	if av := snap.Availability("none"); len(av) != 0 {
 		t.Error("Incorrect availability for 'none':", av)
 	}
 }
@@ -826,20 +841,20 @@ func TestIssue4701(t *testing.T) {
 
 	s.Update(protocol.LocalDeviceID, localHave)
 
-	if c := s.LocalSize(); c.Files != 1 {
+	if c := localSize(s); c.Files != 1 {
 		t.Errorf("Expected 1 local file, got %v", c.Files)
 	}
-	if c := s.GlobalSize(); c.Files != 1 {
+	if c := globalSize(s); c.Files != 1 {
 		t.Errorf("Expected 1 global file, got %v", c.Files)
 	}
 
 	localHave[1].LocalFlags = 0
 	s.Update(protocol.LocalDeviceID, localHave)
 
-	if c := s.LocalSize(); c.Files != 2 {
+	if c := localSize(s); c.Files != 2 {
 		t.Errorf("Expected 2 local files, got %v", c.Files)
 	}
-	if c := s.GlobalSize(); c.Files != 2 {
+	if c := globalSize(s); c.Files != 2 {
 		t.Errorf("Expected 2 global files, got %v", c.Files)
 	}
 
@@ -847,10 +862,10 @@ func TestIssue4701(t *testing.T) {
 	localHave[1].LocalFlags = protocol.FlagLocalIgnored
 	s.Update(protocol.LocalDeviceID, localHave)
 
-	if c := s.LocalSize(); c.Files != 0 {
+	if c := localSize(s); c.Files != 0 {
 		t.Errorf("Expected 0 local files, got %v", c.Files)
 	}
-	if c := s.GlobalSize(); c.Files != 0 {
+	if c := globalSize(s); c.Files != 0 {
 		t.Errorf("Expected 0 global files, got %v", c.Files)
 	}
 }
@@ -873,7 +888,9 @@ func TestWithHaveSequence(t *testing.T) {
 	replace(s, protocol.LocalDeviceID, localHave)
 
 	i := 2
-	s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
+	snap := s.Snapshot()
+	defer snap.Release()
+	snap.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
 		if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
 			t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
 		}
@@ -922,13 +939,15 @@ loop:
 			break loop
 		default:
 		}
-		s.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool {
+		snap := s.Snapshot()
+		snap.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool {
 			if fi.SequenceNo() < prevSeq+1 {
 				t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo())
 			}
 			prevSeq = fi.SequenceNo()
 			return true
 		})
+		snap.Release()
 	}
 }
 
@@ -981,12 +1000,12 @@ func TestMoveGlobalBack(t *testing.T) {
 		t.Error("Expected no need for remote 0, got", need)
 	}
 
-	ls := s.LocalSize()
+	ls := localSize(s)
 	if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
 		t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
 	}
 
-	gs := s.GlobalSize()
+	gs := globalSize(s)
 	if globalBytes := remote0Have[0].Size; gs.Bytes != globalBytes {
 		t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
 	}
@@ -1007,12 +1026,12 @@ func TestMoveGlobalBack(t *testing.T) {
 		t.Error("Expected no local need, got", need)
 	}
 
-	ls = s.LocalSize()
+	ls = localSize(s)
 	if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
 		t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
 	}
 
-	gs = s.GlobalSize()
+	gs = globalSize(s)
 	if globalBytes := localHave[0].Size; gs.Bytes != globalBytes {
 		t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
 	}
@@ -1106,22 +1125,22 @@ func TestReceiveOnlyAccounting(t *testing.T) {
 	replace(s, protocol.LocalDeviceID, files)
 	replace(s, remote, files)
 
-	if n := s.LocalSize().Files; n != 3 {
+	if n := localSize(s).Files; n != 3 {
 		t.Fatal("expected 3 local files initially, not", n)
 	}
-	if n := s.LocalSize().Bytes; n != 30 {
+	if n := localSize(s).Bytes; n != 30 {
 		t.Fatal("expected 30 local bytes initially, not", n)
 	}
-	if n := s.GlobalSize().Files; n != 3 {
+	if n := globalSize(s).Files; n != 3 {
 		t.Fatal("expected 3 global files initially, not", n)
 	}
-	if n := s.GlobalSize().Bytes; n != 30 {
+	if n := globalSize(s).Bytes; n != 30 {
 		t.Fatal("expected 30 global bytes initially, not", n)
 	}
-	if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
+	if n := receiveOnlyChangedSize(s).Files; n != 0 {
 		t.Fatal("expected 0 receive only changed files initially, not", n)
 	}
-	if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
+	if n := receiveOnlyChangedSize(s).Bytes; n != 0 {
 		t.Fatal("expected 0 receive only changed bytes initially, not", n)
 	}
 
@@ -1136,22 +1155,22 @@ func TestReceiveOnlyAccounting(t *testing.T) {
 
 	// Check that we see the files
 
-	if n := s.LocalSize().Files; n != 3 {
+	if n := localSize(s).Files; n != 3 {
 		t.Fatal("expected 3 local files after local change, not", n)
 	}
-	if n := s.LocalSize().Bytes; n != 120 {
+	if n := localSize(s).Bytes; n != 120 {
 		t.Fatal("expected 120 local bytes after local change, not", n)
 	}
-	if n := s.GlobalSize().Files; n != 3 {
+	if n := globalSize(s).Files; n != 3 {
 		t.Fatal("expected 3 global files after local change, not", n)
 	}
-	if n := s.GlobalSize().Bytes; n != 30 {
+	if n := globalSize(s).Bytes; n != 30 {
 		t.Fatal("expected 30 global files after local change, not", n)
 	}
-	if n := s.ReceiveOnlyChangedSize().Files; n != 1 {
+	if n := receiveOnlyChangedSize(s).Files; n != 1 {
 		t.Fatal("expected 1 receive only changed file after local change, not", n)
 	}
-	if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 {
+	if n := receiveOnlyChangedSize(s).Bytes; n != 100 {
 		t.Fatal("expected 100 receive only changed btyes after local change, not", n)
 	}
 
@@ -1167,22 +1186,22 @@ func TestReceiveOnlyAccounting(t *testing.T) {
 
 	// Check that we see the files, same data as initially
 
-	if n := s.LocalSize().Files; n != 3 {
+	if n := localSize(s).Files; n != 3 {
 		t.Fatal("expected 3 local files after revert, not", n)
 	}
-	if n := s.LocalSize().Bytes; n != 30 {
+	if n := localSize(s).Bytes; n != 30 {
 		t.Fatal("expected 30 local bytes after revert, not", n)
 	}
-	if n := s.GlobalSize().Files; n != 3 {
+	if n := globalSize(s).Files; n != 3 {
 		t.Fatal("expected 3 global files after revert, not", n)
 	}
-	if n := s.GlobalSize().Bytes; n != 30 {
+	if n := globalSize(s).Bytes; n != 30 {
 		t.Fatal("expected 30 global bytes after revert, not", n)
 	}
-	if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
+	if n := receiveOnlyChangedSize(s).Files; n != 0 {
 		t.Fatal("expected 0 receive only changed files after revert, not", n)
 	}
-	if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
+	if n := receiveOnlyChangedSize(s).Bytes; n != 0 {
 		t.Fatal("expected 0 receive only changed bytes after revert, not", n)
 	}
 }
@@ -1229,7 +1248,7 @@ func TestRemoteInvalidNotAccounted(t *testing.T) {
 	}
 	s.Update(remoteDevice0, files)
 
-	global := s.GlobalSize()
+	global := globalSize(s)
 	if global.Files != 1 {
 		t.Error("Expected one file in global size, not", global.Files)
 	}
@@ -1386,12 +1405,14 @@ func TestSequenceIndex(t *testing.T) {
 		// a subset of those files if we manage to run before a complete
 		// update has happened since our last iteration.
 		latest = latest[:0]
-		s.WithHaveSequence(seq+1, func(f db.FileIntf) bool {
+		snap := s.Snapshot()
+		snap.WithHaveSequence(seq+1, func(f db.FileIntf) bool {
 			seen[f.FileName()] = f
 			latest = append(latest, f)
 			seq = f.SequenceNo()
 			return true
 		})
+		snap.Release()
 
 		// Calculate the spread in sequence number.
 		var max, min int64
@@ -1449,7 +1470,9 @@ func TestIgnoreAfterReceiveOnly(t *testing.T) {
 
 	s.Update(protocol.LocalDeviceID, fs)
 
-	if f, ok := s.Get(protocol.LocalDeviceID, file); !ok {
+	snap := s.Snapshot()
+	defer snap.Release()
+	if f, ok := snap.Get(protocol.LocalDeviceID, file); !ok {
 		t.Error("File missing in db")
 	} else if f.IsReceiveOnlyChanged() {
 		t.Error("File is still receive-only changed")
@@ -1462,3 +1485,21 @@ func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo
 	fs.Drop(device)
 	fs.Update(device, files)
 }
+
+func localSize(fs *db.FileSet) db.Counts {
+	snap := fs.Snapshot()
+	defer snap.Release()
+	return snap.LocalSize()
+}
+
+func globalSize(fs *db.FileSet) db.Counts {
+	snap := fs.Snapshot()
+	defer snap.Release()
+	return snap.GlobalSize()
+}
+
+func receiveOnlyChangedSize(fs *db.FileSet) db.Counts {
+	snap := fs.Snapshot()
+	defer snap.Release()
+	return snap.ReceiveOnlyChangedSize()
+}

lib/db/transactions.go

@@ -7,6 +7,8 @@
 package db
 
 import (
+	"bytes"
+
 	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
@@ -94,6 +96,291 @@ func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate boo
 	return keyBuf, fi, true, nil
 }
 
+func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
+	if len(prefix) > 0 {
+		unslashedPrefix := prefix
+		if bytes.HasSuffix(prefix, []byte{'/'}) {
+			unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
+		} else {
+			prefix = append(prefix, '/')
+		}
+
+		key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix)
+		if err != nil {
+			return err
+		}
+		if f, ok, err := t.getFileTrunc(key, true); err != nil {
+			return err
+		} else if ok && !fn(f) {
+			return nil
+		}
+	}
+
+	key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	for dbi.Next() {
+		name := t.keyer.NameFromDeviceFileKey(dbi.Key())
+		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
+			return nil
+		}
+
+		f, err := unmarshalTrunc(dbi.Value(), truncate)
+		if err != nil {
+			l.Debugln("unmarshal error:", err)
+			continue
+		}
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}
+
+func (t *readOnlyTransaction) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
+	first, err := t.keyer.GenerateSequenceKey(nil, folder, startSeq)
+	if err != nil {
+		return err
+	}
+	last, err := t.keyer.GenerateSequenceKey(nil, folder, maxInt64)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewRangeIterator(first, last)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	for dbi.Next() {
+		f, ok, err := t.getFileByKey(dbi.Value())
+		if err != nil {
+			return err
+		}
+		if !ok {
+			l.Debugln("missing file for sequence number", t.keyer.SequenceFromSequenceKey(dbi.Key()))
+			continue
+		}
+
+		if shouldDebug() {
+			if seq := t.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq {
+				l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq)
+				panic("sequence index corruption")
+			}
+		}
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}
+
+func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error {
+	if len(prefix) > 0 {
+		unslashedPrefix := prefix
+		if bytes.HasSuffix(prefix, []byte{'/'}) {
+			unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
+		} else {
+			prefix = append(prefix, '/')
+		}
+
+		if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil {
+			return err
+		} else if ok && !fn(f) {
+			return nil
+		}
+	}
+
+	key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, prefix)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var dk []byte
+	for dbi.Next() {
+		name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
+		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
+			return nil
+		}
+
+		vl, ok := unmarshalVersionList(dbi.Value())
+		if !ok {
+			continue
+		}
+
+		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
+		if err != nil {
+			return err
+		}
+
+		f, ok, err := t.getFileTrunc(dk, truncate)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+
+		if !fn(f) {
+			return nil
+		}
+	}
+	if err != nil {
+		return err
+	}
+	return dbi.Error()
+}
+
+func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.DeviceID, error) {
+	k, err := t.keyer.GenerateGlobalVersionKey(nil, folder, file)
+	if err != nil {
+		return nil, err
+	}
+	bs, err := t.Get(k)
+	if backend.IsNotFound(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	vl, ok := unmarshalVersionList(bs)
+	if !ok {
+		return nil, nil
+	}
+
+	var devices []protocol.DeviceID
+	for _, v := range vl.Versions {
+		if !v.Version.Equal(vl.Versions[0].Version) {
+			break
+		}
+		if v.Invalid {
+			continue
+		}
+		n := protocol.DeviceIDFromBytes(v.Device)
+		devices = append(devices, n)
+	}
+
+	return devices, nil
+}
+
+func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn Iterator) error {
+	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
+		return t.withNeedLocal(folder, truncate, fn)
+	}
+
+	key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key.WithoutName())
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var dk []byte
+	devID := protocol.DeviceIDFromBytes(device)
+	for dbi.Next() {
+		vl, ok := unmarshalVersionList(dbi.Value())
+		if !ok {
+			continue
+		}
+
+		haveFV, have := vl.Get(device)
+		// XXX: This marks Concurrent (i.e. conflicting) changes as
+		// needs. Maybe we should do that, but it needs special
+		// handling in the puller.
+		if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) {
+			continue
+		}
+
+		name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
+		needVersion := vl.Versions[0].Version
+		needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
+
+		for i := range vl.Versions {
+			if !vl.Versions[i].Version.Equal(needVersion) {
+				// We haven't found a valid copy of the file with the needed version.
+				break
+			}
+
+			if vl.Versions[i].Invalid {
+				// The file is marked invalid, don't use it.
+				continue
+			}
+
+			dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[i].Device, name)
+			if err != nil {
+				return err
+			}
+			gf, ok, err := t.getFileTrunc(dk, truncate)
+			if err != nil {
+				return err
+			}
+			if !ok {
+				continue
+			}
+
+			if gf.IsDeleted() && !have {
+				// We don't need deleted files that we don't have
+				break
+			}
+
+			l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice)
+
+			if !fn(gf) {
+				return nil
+			}
+
+			// This file is handled, no need to look further in the version list
+			break
+		}
+	}
+	return dbi.Error()
+}
+
+func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Iterator) error {
+	key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key.WithoutName())
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var keyBuf []byte
+	var f FileIntf
+	var ok bool
+	for dbi.Next() {
+		keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}
+
 // A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
 // The batch will be committed on close() or by checkFlush() if it exceeds the
 // batch size.
@@ -353,6 +640,65 @@ func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) error {
 	return dbi.Error()
 }
 
+func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error {
+	key, err := t.keyer.GenerateDeviceFileKey(nil, folder, nil, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice())
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var gk, keyBuf []byte
+	for dbi.Next() {
+		device, ok := t.keyer.DeviceFromDeviceFileKey(dbi.Key())
+		if !ok {
+			// Not having the device in the index is bad. Clear it.
+			if err := t.Delete(dbi.Key()); err != nil {
+				return err
+			}
+			continue
+		}
+		var f FileInfoTruncated
+		// The iterator function may keep a reference to the unmarshalled
+		// struct, which in turn references the buffer it was unmarshalled
+		// from. dbi.Value() just returns an internal slice that it reuses, so
+		// we need to copy it.
+		err := f.Unmarshal(append([]byte{}, dbi.Value()...))
+		if err != nil {
+			return err
+		}
+
+		switch f.Name {
+		case "", ".", "..", "/": // A few obviously invalid filenames
+			l.Infof("Dropping invalid filename %q from database", f.Name)
+			name := []byte(f.Name)
+			gk, err = t.keyer.GenerateGlobalVersionKey(gk, folder, name)
+			if err != nil {
+				return err
+			}
+			keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil)
+			if err != nil {
+				return err
+			}
+			if err := t.Delete(dbi.Key()); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if !fn(device, f) {
+			return nil
+		}
+	}
+	if err := dbi.Error(); err != nil {
+		return err
+	}
+	return t.commit()
+}
+
 type marshaller interface {
 	Marshal() ([]byte, error)
 }