all: Transactionalize db.FileSet (fixes #5952) (#6239)

lib/model/folder.go

@@ -328,11 +328,16 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		subDirs[i] = sub
 	}
 
+	snap := f.fset.Snapshot()
+	// We release explicitly later in this function, however we might exit early
+	// and it's ok to release twice.
+	defer snap.Release()
+
 	// Clean the list of subitems to ensure that we start at a known
 	// directory, and don't scan subdirectories of things we've already
 	// scanned.
 	subDirs = unifySubs(subDirs, func(file string) bool {
-		_, ok := f.fset.Get(protocol.LocalDeviceID, file)
+		_, ok := snap.Get(protocol.LocalDeviceID, file)
 		return ok
 	})
 
@@ -344,7 +349,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		Subs:                  subDirs,
 		Matcher:               f.ignores,
 		TempLifetime:          time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
-		CurrentFiler:          cFiler{f.fset},
+		CurrentFiler:          cFiler{snap},
 		Filesystem:            mtimefs,
 		IgnorePerms:           f.IgnorePerms,
 		AutoNormalize:         f.AutoNormalize,
@@ -369,7 +374,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		oldBatchFn := batchFn // can't reference batchFn directly (recursion)
 		batchFn = func(fs []protocol.FileInfo) error {
 			for i := range fs {
-				switch gf, ok := f.fset.GetGlobal(fs[i].Name); {
+				switch gf, ok := snap.GetGlobal(fs[i].Name); {
 				case !ok:
 					continue
 				case gf.IsEquivalentOptional(fs[i], f.ModTimeWindow(), false, false, protocol.FlagLocalReceiveOnly):
@@ -426,10 +431,15 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 	// ignored files.
 	var toIgnore []db.FileInfoTruncated
 	ignoredParent := ""
+
+	snap.Release()
+	snap = f.fset.Snapshot()
+	defer snap.Release()
+
 	for _, sub := range subDirs {
 		var iterError error
 
-		f.fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
+		snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
 			select {
 			case <-f.ctx.Done():
 				return false
@@ -891,7 +901,7 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string {
 }
 
 type cFiler struct {
-	*db.FileSet
+	*db.Snapshot
 }
 
 // Implements scanner.CurrentFiler

lib/model/folder_recvonly.go

@@ -79,7 +79,9 @@ func (f *receiveOnlyFolder) Revert() {
 
 	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
 	batchSizeBytes := 0
-	f.fset.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap := f.fset.Snapshot()
+	defer snap.Release()
+	snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
 		fi := intf.(protocol.FileInfo)
 		if !fi.IsReceiveOnlyChanged() {
 			// We're only interested in files that have changed locally in
@@ -93,7 +95,7 @@ func (f *receiveOnlyFolder) Revert() {
 			// We'll delete files directly, directories get queued and
 			// handled below.
 
-			handled, err := delQueue.handle(fi)
+			handled, err := delQueue.handle(fi, snap)
 			if err != nil {
 				l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
 				return true // continue
@@ -138,7 +140,7 @@ func (f *receiveOnlyFolder) Revert() {
 	batchSizeBytes = 0
 
 	// Handle any queued directories
-	deleted, err := delQueue.flush()
+	deleted, err := delQueue.flush(snap)
 	if err != nil {
 		l.Infoln("Revert:", err)
 	}
@@ -167,15 +169,15 @@ func (f *receiveOnlyFolder) Revert() {
 // directories for last.
 type deleteQueue struct {
 	handler interface {
-		deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) error
-		deleteDirOnDisk(dir string, scanChan chan<- string) error
+		deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error
+		deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error
 	}
 	ignores  *ignore.Matcher
 	dirs     []string
 	scanChan chan<- string
 }
 
-func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
+func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, error) {
 	// Things that are ignored but not marked deletable are not processed.
 	ign := q.ignores.Match(fi.Name)
 	if ign.IsIgnored() && !ign.IsDeletable() {
@@ -189,11 +191,11 @@ func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
 	}
 
 	// Kill it.
-	err := q.handler.deleteItemOnDisk(fi, q.scanChan)
+	err := q.handler.deleteItemOnDisk(fi, snap, q.scanChan)
 	return true, err
 }
 
-func (q *deleteQueue) flush() ([]string, error) {
+func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) {
 	// Process directories from the leaves inward.
 	sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))
 
@@ -201,7 +203,7 @@ func (q *deleteQueue) flush() ([]string, error) {
 	var deleted []string
 
 	for _, dir := range q.dirs {
-		if err := q.handler.deleteDirOnDisk(dir, q.scanChan); err == nil {
+		if err := q.handler.deleteDirOnDisk(dir, snap, q.scanChan); err == nil {
 			deleted = append(deleted, dir)
 		} else if err != nil && firstError == nil {
 			firstError = err

lib/model/folder_recvonly_test.go

@@ -48,7 +48,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 	m.Index(device1, "ro", knownFiles)
 	f.updateLocalsFromScanning(knownFiles)
 
-	size := m.GlobalSize("ro")
+	size := globalSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
 	}
@@ -60,15 +60,15 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 
 	// We should now have two files and two directories.
 
-	size = m.GlobalSize("ro")
+	size = globalSize(t, m, "ro")
 	if size.Files != 2 || size.Directories != 2 {
 		t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
 	}
-	size = m.LocalSize("ro")
+	size = localSize(t, m, "ro")
 	if size.Files != 2 || size.Directories != 2 {
 		t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
 	}
-	size = m.ReceiveOnlyChangedSize("ro")
+	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files+size.Directories == 0 {
 		t.Fatalf("ROChanged: expected something: %+v", size)
 	}
@@ -93,11 +93,11 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 
 	// We should now have one file and directory again.
 
-	size = m.GlobalSize("ro")
+	size = globalSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
 	}
-	size = m.LocalSize("ro")
+	size = localSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
 	}
@@ -131,19 +131,19 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 
 	// Everything should be in sync.
 
-	size := m.GlobalSize("ro")
+	size := globalSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
 	}
-	size = m.LocalSize("ro")
+	size = localSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
 	}
-	size = m.NeedSize("ro")
+	size = needSize(t, m, "ro")
 	if size.Files+size.Directories > 0 {
 		t.Fatalf("Need: expected nothing: %+v", size)
 	}
-	size = m.ReceiveOnlyChangedSize("ro")
+	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files+size.Directories > 0 {
 		t.Fatalf("ROChanged: expected nothing: %+v", size)
 	}
@@ -159,20 +159,20 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 
 	// We now have a newer file than the rest of the cluster. Global state should reflect this.
 
-	size = m.GlobalSize("ro")
+	size = globalSize(t, m, "ro")
 	const sizeOfDir = 128
 	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
 		t.Fatalf("Global: expected no change due to the new file: %+v", size)
 	}
-	size = m.LocalSize("ro")
+	size = localSize(t, m, "ro")
 	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
 		t.Fatalf("Local: expected the new file to be reflected: %+v", size)
 	}
-	size = m.NeedSize("ro")
+	size = needSize(t, m, "ro")
 	if size.Files+size.Directories > 0 {
 		t.Fatalf("Need: expected nothing: %+v", size)
 	}
-	size = m.ReceiveOnlyChangedSize("ro")
+	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files+size.Directories == 0 {
 		t.Fatalf("ROChanged: expected something: %+v", size)
 	}
@@ -181,15 +181,15 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 
 	m.Revert("ro")
 
-	size = m.GlobalSize("ro")
+	size = globalSize(t, m, "ro")
 	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
 		t.Fatalf("Global: expected the global size to revert: %+v", size)
 	}
-	size = m.LocalSize("ro")
+	size = localSize(t, m, "ro")
 	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
 		t.Fatalf("Local: expected the local size to remain: %+v", size)
 	}
-	size = m.NeedSize("ro")
+	size = needSize(t, m, "ro")
 	if size.Files != 1 || size.Bytes != int64(len(oldData)) {
 		t.Fatalf("Local: expected to need the old file data: %+v", size)
 	}
@@ -227,19 +227,19 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 
 	// Everything should be in sync.
 
-	size := m.GlobalSize("ro")
+	size := globalSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
 	}
-	size = m.LocalSize("ro")
+	size = localSize(t, m, "ro")
 	if size.Files != 1 || size.Directories != 1 {
 		t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
 	}
-	size = m.NeedSize("ro")
+	size = needSize(t, m, "ro")
 	if size.Files+size.Directories > 0 {
 		t.Fatalf("Need: expected nothing: %+v", size)
 	}
-	size = m.ReceiveOnlyChangedSize("ro")
+	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files+size.Directories > 0 {
 		t.Fatalf("ROChanged: expected nothing: %+v", size)
 	}
@@ -253,7 +253,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 
 	m.ScanFolder("ro")
 
-	size = m.ReceiveOnlyChangedSize("ro")
+	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files != 2 {
 		t.Fatalf("Receive only: expected 2 files: %+v", size)
 	}
@@ -266,7 +266,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 
 	m.ScanFolder("ro")
 
-	size = m.ReceiveOnlyChangedSize("ro")
+	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files+size.Directories+size.Deleted != 0 {
 		t.Fatalf("Receive only: expected all zero: %+v", size)
 	}

lib/model/folder_sendonly.go

@@ -50,7 +50,9 @@ func (f *sendOnlyFolder) pull() bool {
 	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
 	batchSizeBytes := 0
 
-	f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap := f.fset.Snapshot()
+	defer snap.Release()
+	snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
 		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
 			f.updateLocalsFromPulling(batch)
 			batch = batch[:0]
@@ -66,7 +68,7 @@ func (f *sendOnlyFolder) pull() bool {
 			return true
 		}
 
-		curFile, ok := f.fset.Get(protocol.LocalDeviceID, intf.FileName())
+		curFile, ok := snap.Get(protocol.LocalDeviceID, intf.FileName())
 		if !ok {
 			if intf.IsDeleted() {
 				l.Debugln("Should never get a deleted file as needed when we don't have it")
@@ -98,7 +100,9 @@ func (f *sendOnlyFolder) Override() {
 	f.setState(FolderScanning)
 	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
 	batchSizeBytes := 0
-	f.fset.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+	snap := f.fset.Snapshot()
+	defer snap.Release()
+	snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 		need := fi.(protocol.FileInfo)
 		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
 			f.updateLocalsFromScanning(batch)
@@ -106,7 +110,7 @@ func (f *sendOnlyFolder) Override() {
 			batchSizeBytes = 0
 		}
 
-		have, ok := f.fset.Get(protocol.LocalDeviceID, need.Name)
+		have, ok := snap.Get(protocol.LocalDeviceID, need.Name)
 		// Don't override files that are in a bad state (ignored,
 		// unsupported, must rescan, ...).
 		if ok && have.IsInvalid() {

lib/model/folder_sendrecv.go

@@ -149,10 +149,12 @@ func (f *sendReceiveFolder) pull() bool {
 
 	// If there is nothing to do, don't even enter pulling state.
 	abort := true
-	f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap := f.fset.Snapshot()
+	snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
 		abort = false
 		return false
 	})
+	snap.Release()
 	if abort {
 		return true
 	}
@@ -234,6 +236,9 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 	f.pullErrors = make(map[string]string)
 	f.pullErrorsMut.Unlock()
 
+	snap := f.fset.Snapshot()
+	defer snap.Release()
+
 	pullChan := make(chan pullBlockState)
 	copyChan := make(chan copyBlocksState)
 	finisherChan := make(chan *sharedPullerState)
@@ -272,11 +277,11 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 	doneWg.Add(1)
 	// finisherRoutine finishes when finisherChan is closed
 	go func() {
-		f.finisherRoutine(finisherChan, dbUpdateChan, scanChan)
+		f.finisherRoutine(snap, finisherChan, dbUpdateChan, scanChan)
 		doneWg.Done()
 	}()
 
-	changed, fileDeletions, dirDeletions, err := f.processNeeded(dbUpdateChan, copyChan, scanChan)
+	changed, fileDeletions, dirDeletions, err := f.processNeeded(snap, dbUpdateChan, copyChan, scanChan)
 
 	// Signal copy and puller routines that we are done with the in data for
 	// this iteration. Wait for them to finish.
@@ -291,7 +296,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 	doneWg.Wait()
 
 	if err == nil {
-		f.processDeletions(fileDeletions, dirDeletions, dbUpdateChan, scanChan)
+		f.processDeletions(fileDeletions, dirDeletions, snap, dbUpdateChan, scanChan)
 	}
 
 	// Wait for db updates and scan scheduling to complete
@@ -307,7 +312,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 	return changed
 }
 
-func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
+func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
 	changed := 0
 	var dirDeletions []protocol.FileInfo
 	fileDeletions := map[string]protocol.FileInfo{}
@@ -317,7 +322,7 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 	// Regular files to pull goes into the file queue, everything else
 	// (directories, symlinks and deletes) goes into the "process directly"
 	// pile.
-	f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
 		select {
 		case <-f.ctx.Done():
 			return false
@@ -359,9 +364,9 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 				// files to delete inside them before we get to that point.
 				dirDeletions = append(dirDeletions, file)
 			} else if file.IsSymlink() {
-				f.deleteFile(file, dbUpdateChan, scanChan)
+				f.deleteFile(file, snap, dbUpdateChan, scanChan)
 			} else {
-				df, ok := f.fset.Get(protocol.LocalDeviceID, file.Name)
+				df, ok := snap.Get(protocol.LocalDeviceID, file.Name)
 				// Local file can be already deleted, but with a lower version
 				// number, hence the deletion coming in again as part of
 				// WithNeed, furthermore, the file can simply be of the wrong
@@ -377,7 +382,7 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 			}
 
 		case file.Type == protocol.FileInfoTypeFile:
-			curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name)
+			curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
 			if _, need := blockDiff(curFile.Blocks, file.Blocks); hasCurFile && len(need) == 0 {
 				// We are supposed to copy the entire file, and then fetch nothing. We
 				// are only updating metadata, so we don't actually *need* to make the
@@ -396,13 +401,13 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
 		case file.IsDirectory() && !file.IsSymlink():
 			l.Debugln(f, "Handling directory", file.Name)
 			if f.checkParent(file.Name, scanChan) {
-				f.handleDir(file, dbUpdateChan, scanChan)
+				f.handleDir(file, snap, dbUpdateChan, scanChan)
 			}
 
 		case file.IsSymlink():
 			l.Debugln(f, "Handling symlink", file.Name)
 			if f.checkParent(file.Name, scanChan) {
-				f.handleSymlink(file, dbUpdateChan, scanChan)
+				f.handleSymlink(file, snap, dbUpdateChan, scanChan)
 			}
 
 		default:
@@ -451,7 +456,7 @@ nextFile:
 			break
 		}
 
-		fi, ok := f.fset.GetGlobal(fileName)
+		fi, ok := snap.GetGlobal(fileName)
 		if !ok {
 			// File is no longer in the index. Mark it as done and drop it.
 			f.queue.Done(fileName)
@@ -484,7 +489,7 @@ nextFile:
 				// desired state with the delete bit set is in the deletion
 				// map.
 				desired := fileDeletions[candidate.Name]
-				if err := f.renameFile(candidate, desired, fi, dbUpdateChan, scanChan); err != nil {
+				if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil {
 					// Failed to rename, try to handle files as separate
 					// deletions and updates.
 					break
@@ -498,11 +503,11 @@ nextFile:
 			}
 		}
 
-		devices := f.fset.Availability(fileName)
+		devices := snap.Availability(fileName)
 		for _, dev := range devices {
 			if _, ok := f.model.Connection(dev); ok {
 				// Handle the file normally, by coping and pulling, etc.
-				f.handleFile(fi, copyChan, dbUpdateChan)
+				f.handleFile(fi, snap, copyChan)
 				continue nextFile
 			}
 		}
@@ -513,7 +518,7 @@ nextFile:
 	return changed, fileDeletions, dirDeletions, nil
 }
 
-func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
+func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
 	for _, file := range fileDeletions {
 		select {
 		case <-f.ctx.Done():
@@ -521,7 +526,7 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F
 		default:
 		}
 
-		f.deleteFile(file, dbUpdateChan, scanChan)
+		f.deleteFile(file, snap, dbUpdateChan, scanChan)
 	}
 
 	// Process in reverse order to delete depth first
@@ -534,12 +539,12 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F
 
 		dir := dirDeletions[len(dirDeletions)-i-1]
 		l.Debugln(f, "Deleting dir", dir.Name)
-		f.deleteDir(dir, dbUpdateChan, scanChan)
+		f.deleteDir(dir, snap, dbUpdateChan, scanChan)
 	}
 }
 
 // handleDir creates or updates the given directory
-func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
+func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
 	// Used in the defer closure below, updated by the function body. Take
 	// care not declare another err.
 	var err error
@@ -567,7 +572,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 	}
 
 	if shouldDebug() {
-		curFile, _ := f.fset.Get(protocol.LocalDeviceID, file.Name)
+		curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
 		l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
 	}
 
@@ -598,7 +603,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 				return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
 			}, curFile.Name)
 		} else {
-			err = f.deleteItemOnDisk(curFile, scanChan)
+			err = f.deleteItemOnDisk(curFile, snap, scanChan)
 		}
 		if err != nil {
 			f.newPullError(file.Name, err)
@@ -693,7 +698,7 @@ func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) boo
 }
 
 // handleSymlink creates or updates the given symlink
-func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
+func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
 	// Used in the defer closure below, updated by the function body. Take
 	// care not declare another err.
 	var err error
@@ -716,7 +721,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 	}()
 
 	if shouldDebug() {
-		curFile, _ := f.fset.Get(protocol.LocalDeviceID, file.Name)
+		curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
 		l.Debugf("need symlink\n\t%v\n\t%v", file, curFile)
 	}
 
@@ -750,7 +755,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 				return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
 			}, curFile.Name)
 		} else {
-			err = f.deleteItemOnDisk(curFile, scanChan)
+			err = f.deleteItemOnDisk(curFile, snap, scanChan)
 		}
 		if err != nil {
 			f.newPullError(file.Name, errors.Wrap(err, "symlink remove"))
@@ -775,7 +780,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 }
 
 // deleteDir attempts to remove a directory that was deleted on a remote
-func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
+func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
 	// Used in the defer closure below, updated by the function body. Take
 	// care not declare another err.
 	var err error
@@ -797,7 +802,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
 		})
 	}()
 
-	if err = f.deleteDirOnDisk(file.Name, scanChan); err != nil {
+	if err = f.deleteDirOnDisk(file.Name, snap, scanChan); err != nil {
 		f.newPullError(file.Name, errors.Wrap(err, "delete dir"))
 		return
 	}
@@ -806,8 +811,8 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
 }
 
 // deleteFile attempts to delete the given file
-func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
-	cur, hasCur := f.fset.Get(protocol.LocalDeviceID, file.Name)
+func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
+	cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name)
 	f.deleteFileWithCurrent(file, cur, hasCur, dbUpdateChan, scanChan)
 }
 
@@ -895,7 +900,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 
 // renameFile attempts to rename an existing file to a destination
 // and set the right attributes on it.
-func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
+func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
 	// Used in the defer closure below, updated by the function body. Take
 	// care not declare another err.
 	var err error
@@ -937,7 +942,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 		return err
 	}
 	// Check that the target corresponds to what we have in the DB
-	curTarget, ok := f.fset.Get(protocol.LocalDeviceID, target.Name)
+	curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name)
 	switch stat, serr := f.fs.Lstat(target.Name); {
 	case serr != nil && fs.IsNotExist(serr):
 		if !ok || curTarget.IsDeleted() {
@@ -994,7 +999,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 	// of the source and the creation of the target temp file. Fix-up the metadata,
 	// update the local index of the target file and rename from temp to real name.
 
-	if err = f.performFinish(target, curTarget, true, tempName, dbUpdateChan, scanChan); err != nil {
+	if err = f.performFinish(target, curTarget, true, tempName, snap, dbUpdateChan, scanChan); err != nil {
 		return err
 	}
 
@@ -1039,8 +1044,8 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 
 // handleFile queues the copies and pulls as necessary for a single new or
 // changed file.
-func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, dbUpdateChan chan<- dbUpdateJob) {
-	curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name)
+func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot, copyChan chan<- copyBlocksState) {
+	curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
 
 	have, _ := blockDiff(curFile.Blocks, file.Blocks)
 
@@ -1493,7 +1498,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
 	out <- state.sharedPullerState
 }
 
-func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
+func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
 	// Set the correct permission bits on the new file
 	if !f.IgnorePerms && !file.NoPermissions {
 		if err := f.fs.Chmod(tempName, fs.FileMode(file.Permissions&0777)); err != nil {
@@ -1528,7 +1533,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
 				return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
 			}, curFile.Name)
 		} else {
-			err = f.deleteItemOnDisk(curFile, scanChan)
+			err = f.deleteItemOnDisk(curFile, snap, scanChan)
 		}
 		if err != nil {
 			return err
@@ -1549,7 +1554,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
 	return nil
 }
 
-func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
+func (f *sendReceiveFolder) finisherRoutine(snap *db.Snapshot, in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
 	for state := range in {
 		if closed, err := state.finalClose(); closed {
 			l.Debugln(f, "closing", state.file.Name)
@@ -1557,7 +1562,7 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpda
 			f.queue.Done(state.file.Name)
 
 			if err == nil {
-				err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, dbUpdateChan, scanChan)
+				err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, snap, dbUpdateChan, scanChan)
 			}
 
 			if err != nil {
@@ -1809,7 +1814,7 @@ func (f *sendReceiveFolder) Errors() []FileError {
 }
 
 // deleteItemOnDisk deletes the file represented by old that is about to be replaced by new.
-func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) (err error) {
+func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) (err error) {
 	defer func() {
 		err = errors.Wrap(err, contextRemovingOldItem)
 	}()
@@ -1818,7 +1823,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch
 	case item.IsDirectory():
 		// Directories aren't archived and need special treatment due
 		// to potential children.
-		return f.deleteDirOnDisk(item.Name, scanChan)
+		return f.deleteDirOnDisk(item.Name, snap, scanChan)
 
 	case !item.IsSymlink() && f.versioner != nil:
 		// If we should use versioning, let the versioner archive the
@@ -1834,7 +1839,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch
 
 // deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside
 // the directory and removes them if possible or returns an error if it fails
-func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string) error {
+func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error {
 	if err := osutil.TraversesSymlink(f.fs, filepath.Dir(dir)); err != nil {
 		return err
 	}
@@ -1853,7 +1858,7 @@ func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string)
 			toBeDeleted = append(toBeDeleted, fullDirFile)
 		} else if f.ignores != nil && f.ignores.Match(fullDirFile).IsIgnored() {
 			hasIgnored = true
-		} else if cf, ok := f.fset.Get(protocol.LocalDeviceID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
+		} else if cf, ok := snap.Get(protocol.LocalDeviceID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
 			// Something appeared in the dir that we either are not aware of
 			// at all, that we think should be deleted or that is invalid,
 			// but not currently ignored -> schedule scan. The scanChan

lib/model/folder_sendrecv_test.go

@@ -148,9 +148,8 @@ func TestHandleFile(t *testing.T) {
 	defer cleanupSRFolder(f, m)
 
 	copyChan := make(chan copyBlocksState, 1)
-	dbUpdateChan := make(chan dbUpdateJob, 1)
 
-	f.handleFile(requiredFile, copyChan, dbUpdateChan)
+	f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
 
 	// Receive the results
 	toCopy := <-copyChan
@@ -195,9 +194,8 @@ func TestHandleFileWithTemp(t *testing.T) {
 	}
 
 	copyChan := make(chan copyBlocksState, 1)
-	dbUpdateChan := make(chan dbUpdateJob, 1)
 
-	f.handleFile(requiredFile, copyChan, dbUpdateChan)
+	f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
 
 	// Receive the results
 	toCopy := <-copyChan
@@ -247,13 +245,12 @@ func TestCopierFinder(t *testing.T) {
 	copyChan := make(chan copyBlocksState)
 	pullChan := make(chan pullBlockState, 4)
 	finisherChan := make(chan *sharedPullerState, 1)
-	dbUpdateChan := make(chan dbUpdateJob, 1)
 
 	// Run a single fetcher routine
 	go f.copierRoutine(copyChan, pullChan, finisherChan)
 	defer close(copyChan)
 
-	f.handleFile(requiredFile, copyChan, dbUpdateChan)
+	f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
 
 	timeout := time.After(10 * time.Second)
 	pulls := make([]pullBlockState, 4)
@@ -378,7 +375,6 @@ func TestWeakHash(t *testing.T) {
 	copyChan := make(chan copyBlocksState)
 	pullChan := make(chan pullBlockState, expectBlocks)
 	finisherChan := make(chan *sharedPullerState, 1)
-	dbUpdateChan := make(chan dbUpdateJob, 1)
 
 	// Run a single fetcher routine
 	go fo.copierRoutine(copyChan, pullChan, finisherChan)
@@ -386,7 +382,7 @@ func TestWeakHash(t *testing.T) {
 
 	// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
 	fo.WeakHashThresholdPct = 101
-	fo.handleFile(desiredFile, copyChan, dbUpdateChan)
+	fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan)
 
 	var pulls []pullBlockState
 	timeout := time.After(10 * time.Second)
@@ -415,7 +411,7 @@ func TestWeakHash(t *testing.T) {
 
 	// Test 2 - using weak hash, expectPulls blocks pulled.
 	fo.WeakHashThresholdPct = -1
-	fo.handleFile(desiredFile, copyChan, dbUpdateChan)
+	fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan)
 
 	pulls = pulls[:0]
 	for len(pulls) < expectPulls {
@@ -495,9 +491,10 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 	finisherBufferChan := make(chan *sharedPullerState)
 	finisherChan := make(chan *sharedPullerState)
 	dbUpdateChan := make(chan dbUpdateJob, 1)
+	snap := f.fset.Snapshot()
 
 	copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
-	go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
+	go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
 
 	defer func() {
 		close(copyChan)
@@ -507,7 +504,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 		close(finisherChan)
 	}()
 
-	f.handleFile(file, copyChan, dbUpdateChan)
+	f.handleFile(file, snap, copyChan)
 
 	// Receive a block at puller, to indicate that at least a single copier
 	// loop has been performed.
@@ -589,6 +586,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 	finisherBufferChan := make(chan *sharedPullerState)
 	finisherChan := make(chan *sharedPullerState)
 	dbUpdateChan := make(chan dbUpdateJob, 1)
+	snap := f.fset.Snapshot()
 
 	copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
 	pullWg := sync.NewWaitGroup()
@@ -597,7 +595,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 		f.pullerRoutine(pullChan, finisherBufferChan)
 		pullWg.Done()
 	}()
-	go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
+	go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
 	defer func() {
 		// Unblock copier and puller
 		go func() {
@@ -612,7 +610,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 		close(finisherChan)
 	}()
 
-	f.handleFile(file, copyChan, dbUpdateChan)
+	f.handleFile(file, snap, copyChan)
 
 	// Receive at finisher, we should error out as puller has nowhere to pull
 	// from.
@@ -693,7 +691,7 @@ func TestIssue3164(t *testing.T) {
 
 	dbUpdateChan := make(chan dbUpdateJob, 1)
 
-	f.deleteDir(file, dbUpdateChan, make(chan string))
+	f.deleteDir(file, f.fset.Snapshot(), dbUpdateChan, make(chan string))
 
 	if _, err := ffs.Stat("issue3164"); !fs.IsNotExist(err) {
 		t.Fatal(err)
@@ -832,7 +830,7 @@ func TestCopyOwner(t *testing.T) {
 
 	dbUpdateChan := make(chan dbUpdateJob, 1)
 	defer close(dbUpdateChan)
-	f.handleDir(dir, dbUpdateChan, nil)
+	f.handleDir(dir, f.fset.Snapshot(), dbUpdateChan, nil)
 	<-dbUpdateChan // empty the channel for later
 
 	info, err := f.fs.Lstat("foo/bar")
@@ -858,16 +856,17 @@ func TestCopyOwner(t *testing.T) {
 	// but it's the way data is passed around. When the database update
 	// comes the finisher is done.
 
+	snap := f.fset.Snapshot()
 	finisherChan := make(chan *sharedPullerState)
 	copierChan, copyWg := startCopier(f, nil, finisherChan)
-	go f.finisherRoutine(finisherChan, dbUpdateChan, nil)
+	go f.finisherRoutine(snap, finisherChan, dbUpdateChan, nil)
 	defer func() {
 		close(copierChan)
 		copyWg.Wait()
 		close(finisherChan)
 	}()
 
-	f.handleFile(file, copierChan, nil)
+	f.handleFile(file, snap, copierChan)
 	<-dbUpdateChan
 
 	info, err = f.fs.Lstat("foo/bar/baz")
@@ -886,7 +885,7 @@ func TestCopyOwner(t *testing.T) {
 		SymlinkTarget: "over the rainbow",
 	}
 
-	f.handleSymlink(symlink, dbUpdateChan, nil)
+	f.handleSymlink(symlink, snap, dbUpdateChan, nil)
 	<-dbUpdateChan
 
 	info, err = f.fs.Lstat("foo/bar/sym")
@@ -921,7 +920,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
 	dbUpdateChan := make(chan dbUpdateJob, 1)
 	scanChan := make(chan string, 1)
 
-	f.handleDir(file, dbUpdateChan, scanChan)
+	f.handleDir(file, f.fset.Snapshot(), dbUpdateChan, scanChan)
 
 	if confls := existingConflicts(name, ffs); len(confls) != 1 {
 		t.Fatal("Expected one conflict, got", len(confls))
@@ -954,7 +953,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
 	dbUpdateChan := make(chan dbUpdateJob, 1)
 	scanChan := make(chan string, 1)
 
-	f.handleSymlink(file, dbUpdateChan, scanChan)
+	f.handleSymlink(file, f.fset.Snapshot(), dbUpdateChan, scanChan)
 
 	if confls := existingConflicts(name, ffs); len(confls) != 1 {
 		t.Fatal("Expected one conflict, got", len(confls))
@@ -996,7 +995,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
 	fi.Version = fi.Version.Update(device1.Short())
 	scanChan := make(chan string, 1)
 	dbUpdateChan := make(chan dbUpdateJob, 1)
-	f.deleteFile(fi, dbUpdateChan, scanChan)
+	f.deleteFile(fi, f.fset.Snapshot(), dbUpdateChan, scanChan)
 	select {
 	case f := <-scanChan:
 		t.Fatalf("Received %v on scanChan", f)

lib/model/folder_summary.go

@@ -77,6 +77,11 @@ func (c *folderSummaryService) String() string {
 func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, error) {
 	var res = make(map[string]interface{})
 
+	snap, err := c.model.DBSnapshot(folder)
+	if err != nil {
+		return nil, err
+	}
+
 	errors, err := c.model.FolderErrors(folder)
 	if err != nil && err != ErrFolderPaused && err != errFolderNotRunning {
 		// Stats from the db can still be obtained if the folder is just paused/being started
@@ -87,19 +92,31 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
 
 	res["invalid"] = "" // Deprecated, retains external API for now
 
-	global := c.model.GlobalSize(folder)
+	global := snap.GlobalSize()
 	res["globalFiles"], res["globalDirectories"], res["globalSymlinks"], res["globalDeleted"], res["globalBytes"], res["globalTotalItems"] = global.Files, global.Directories, global.Symlinks, global.Deleted, global.Bytes, global.TotalItems()
 
-	local := c.model.LocalSize(folder)
+	local := snap.LocalSize()
 	res["localFiles"], res["localDirectories"], res["localSymlinks"], res["localDeleted"], res["localBytes"], res["localTotalItems"] = local.Files, local.Directories, local.Symlinks, local.Deleted, local.Bytes, local.TotalItems()
 
-	need := c.model.NeedSize(folder)
+	need := snap.NeedSize()
+	need.Bytes -= c.model.FolderProgressBytesCompleted(folder)
+	// This may happen if we are in progress of pulling files that were
+	// deleted globally after the pull started.
+	if need.Bytes < 0 {
+		need.Bytes = 0
+	}
 	res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"], res["needTotalItems"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes, need.TotalItems()
 
-	if c.cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly {
+	fcfg, ok := c.cfg.Folder(folder)
+
+	if ok && fcfg.IgnoreDelete {
+		res["needDeletes"] = 0
+	}
+
+	if ok && fcfg.Type == config.FolderTypeReceiveOnly {
 		// Add statistics for things that have changed locally in a receive
 		// only folder.
-		ro := c.model.ReceiveOnlyChangedSize(folder)
+		ro := snap.ReceiveOnlyChangedSize()
 		res["receiveOnlyChangedFiles"] = ro.Files
 		res["receiveOnlyChangedDirectories"] = ro.Directories
 		res["receiveOnlyChangedSymlinks"] = ro.Symlinks
@@ -115,8 +132,8 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
 		res["error"] = err.Error()
 	}
 
-	ourSeq, _ := c.model.CurrentSequence(folder)
-	remoteSeq, _ := c.model.RemoteSequence(folder)
+	ourSeq := snap.Sequence(protocol.LocalDeviceID)
+	remoteSeq := snap.Sequence(protocol.GlobalDeviceID)
 
 	res["version"] = ourSeq + remoteSeq  // legacy
 	res["sequence"] = ourSeq + remoteSeq // new name

lib/model/model.go

@@ -90,21 +90,14 @@ type Model interface {
 	GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
 	RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)
 
-	LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated
+	DBSnapshot(folder string) (*db.Snapshot, error)
 	NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
-	RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
+	FolderProgressBytesCompleted(folder string) int64
+
 	CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
 	CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
 	Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability
 
-	GlobalSize(folder string) db.Counts
-	LocalSize(folder string) db.Counts
-	NeedSize(folder string) db.Counts
-	ReceiveOnlyChangedSize(folder string) db.Counts
-
-	CurrentSequence(folder string) (int64, bool)
-	RemoteSequence(folder string) (int64, bool)
-
 	Completion(device protocol.DeviceID, folder string) FolderCompletion
 	ConnectionStats() map[string]interface{}
 	DeviceStatistics() (map[string]stats.DeviceStatistics, error)
@@ -764,7 +757,10 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 		return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
 	}
 
-	tot := rf.GlobalSize().Bytes
+	snap := rf.Snapshot()
+	defer snap.Release()
+
+	tot := snap.GlobalSize().Bytes
 	if tot == 0 {
 		// Folder is empty, so we have all of it
 		return FolderCompletion{
@@ -777,7 +773,7 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 	m.pmut.RUnlock()
 
 	var need, items, fileNeed, downloaded, deletes int64
-	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
+	snap.WithNeedTruncated(device, func(f db.FileIntf) bool {
 		ft := f.(db.FileInfoTruncated)
 
 		// If the file is deleted, we account it only in the deleted column.
@@ -822,84 +818,19 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 	}
 }
 
-func addSizeOfFile(s *db.Counts, f db.FileIntf) {
-	switch {
-	case f.IsDeleted():
-		s.Deleted++
-	case f.IsDirectory():
-		s.Directories++
-	case f.IsSymlink():
-		s.Symlinks++
-	default:
-		s.Files++
-	}
-	s.Bytes += f.FileSize()
-}
-
-// GlobalSize returns the number of files, deleted files and total bytes for all
-// files in the global model.
-func (m *model) GlobalSize(folder string) db.Counts {
+// DBSnapshot returns a snapshot of the database content relevant to the given folder.
+func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) {
 	m.fmut.RLock()
 	rf, ok := m.folderFiles[folder]
 	m.fmut.RUnlock()
-	if ok {
-		return rf.GlobalSize()
+	if !ok {
+		return nil, errFolderMissing
 	}
-	return db.Counts{}
+	return rf.Snapshot(), nil
 }
 
-// LocalSize returns the number of files, deleted files and total bytes for all
-// files in the local folder.
-func (m *model) LocalSize(folder string) db.Counts {
-	m.fmut.RLock()
-	rf, ok := m.folderFiles[folder]
-	m.fmut.RUnlock()
-	if ok {
-		return rf.LocalSize()
-	}
-	return db.Counts{}
-}
-
-// ReceiveOnlyChangedSize returns the number of files, deleted files and
-// total bytes for all files that have changed locally in a receieve only
-// folder.
-func (m *model) ReceiveOnlyChangedSize(folder string) db.Counts {
-	m.fmut.RLock()
-	rf, ok := m.folderFiles[folder]
-	m.fmut.RUnlock()
-	if ok {
-		return rf.ReceiveOnlyChangedSize()
-	}
-	return db.Counts{}
-}
-
-// NeedSize returns the number of currently needed files and their total size
-// minus the amount that has already been downloaded.
-func (m *model) NeedSize(folder string) db.Counts {
-	m.fmut.RLock()
-	rf, ok := m.folderFiles[folder]
-	cfg := m.folderCfgs[folder]
-	m.fmut.RUnlock()
-
-	var result db.Counts
-	if ok {
-		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
-			if cfg.IgnoreDelete && f.IsDeleted() {
-				return true
-			}
-
-			addSizeOfFile(&result, f)
-			return true
-		})
-	}
-	result.Bytes -= m.progressEmitter.BytesCompleted(folder)
-	// This may happen if we are in progress of pulling files that were
-	// deleted globally after the pull started.
-	if result.Bytes < 0 {
-		result.Bytes = 0
-	}
-	l.Debugf("%v NeedSize(%q): %v", m, folder, result)
-	return result
+func (m *model) FolderProgressBytesCompleted(folder string) int64 {
+	return m.progressEmitter.BytesCompleted(folder)
 }
 
 // NeedFolderFiles returns paginated list of currently needed files in
@@ -915,6 +846,8 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 		return nil, nil, nil
 	}
 
+	snap := rf.Snapshot()
+	defer snap.Release()
 	var progress, queued, rest []db.FileInfoTruncated
 	var seen map[string]struct{}
 
@@ -929,14 +862,14 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 		seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))
 
 		for i, name := range progressNames {
-			if f, ok := rf.GetGlobalTruncated(name); ok {
+			if f, ok := snap.GetGlobalTruncated(name); ok {
 				progress[i] = f
 				seen[name] = struct{}{}
 			}
 		}
 
 		for i, name := range queuedNames {
-			if f, ok := rf.GetGlobalTruncated(name); ok {
+			if f, ok := snap.GetGlobalTruncated(name); ok {
 				queued[i] = f
 				seen[name] = struct{}{}
 			}
@@ -950,7 +883,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 	}
 
 	rest = make([]db.FileInfoTruncated, 0, perpage)
-	rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
+	snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
 		if cfg.IgnoreDelete && f.IsDeleted() {
 			return true
 		}
@@ -970,77 +903,6 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 	return progress, queued, rest
 }
 
-// LocalChangedFiles returns a paginated list of currently needed files in
-// progress, queued, and to be queued on next puller iteration, as well as the
-// total number of files currently needed.
-func (m *model) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
-	m.fmut.RLock()
-	rf, ok := m.folderFiles[folder]
-	fcfg := m.folderCfgs[folder]
-	m.fmut.RUnlock()
-
-	if !ok {
-		return nil
-	}
-	if fcfg.Type != config.FolderTypeReceiveOnly {
-		return nil
-	}
-	if rf.ReceiveOnlyChangedSize().TotalItems() == 0 {
-		return nil
-	}
-
-	files := make([]db.FileInfoTruncated, 0, perpage)
-
-	skip := (page - 1) * perpage
-	get := perpage
-
-	rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
-		if !f.IsReceiveOnlyChanged() {
-			return true
-		}
-		if skip > 0 {
-			skip--
-			return true
-		}
-		ft := f.(db.FileInfoTruncated)
-		files = append(files, ft)
-		get--
-		return get > 0
-	})
-
-	return files
-}
-
-// RemoteNeedFolderFiles returns paginated list of currently needed files in
-// progress, queued, and to be queued on next puller iteration, as well as the
-// total number of files currently needed.
-func (m *model) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
-	m.fmut.RLock()
-	m.pmut.RLock()
-	err := m.checkDeviceFolderConnectedLocked(device, folder)
-	rf := m.folderFiles[folder]
-	m.pmut.RUnlock()
-	m.fmut.RUnlock()
-	if err != nil {
-		return nil, err
-	}
-
-	files := make([]db.FileInfoTruncated, 0, perpage)
-	skip := (page - 1) * perpage
-	get := perpage
-	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
-		if skip > 0 {
-			skip--
-			return true
-		}
-		files = append(files, f.(db.FileInfoTruncated))
-		get--
-		return get > 0
-	})
-
-	return files, nil
-}
-
 // Index is called when a new device is connected and we receive their full index.
 // Implements the protocol.Model interface.
 func (m *model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) error {
@@ -1752,7 +1614,9 @@ func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo
 	if !ok {
 		return protocol.FileInfo{}, false
 	}
-	return fs.Get(protocol.LocalDeviceID, file)
+	snap := fs.Snapshot()
+	defer snap.Release()
+	return snap.Get(protocol.LocalDeviceID, file)
 }
 
 func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) {
@@ -1762,7 +1626,9 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
 	if !ok {
 		return protocol.FileInfo{}, false
 	}
-	return fs.GetGlobal(file)
+	snap := fs.Snapshot()
+	defer snap.Release()
+	return snap.GetGlobal(file)
 }
 
 // Connection returns the current connection for device, and a boolean whether a connection was found.
@@ -2075,7 +1941,9 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
 
 	var err error
 	var f protocol.FileInfo
-	s.fset.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
+	snap := s.fset.Snapshot()
+	defer snap.Release()
+	snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
 		if err = batch.flushIfFull(); err != nil {
 			return false
 		}
@@ -2347,45 +2215,6 @@ func (m *model) Revert(folder string) {
 	runner.Revert()
 }
 
-// CurrentSequence returns the change version for the given folder.
-// This is guaranteed to increment if the contents of the local folder has
-// changed.
-func (m *model) CurrentSequence(folder string) (int64, bool) {
-	m.fmut.RLock()
-	fs, ok := m.folderFiles[folder]
-	m.fmut.RUnlock()
-	if !ok {
-		// The folder might not exist, since this can be called with a user
-		// specified folder name from the REST interface.
-		return 0, false
-	}
-
-	return fs.Sequence(protocol.LocalDeviceID), true
-}
-
-// RemoteSequence returns the change version for the given folder, as
-// sent by remote peers. This is guaranteed to increment if the contents of
-// the remote or global folder has changed.
-func (m *model) RemoteSequence(folder string) (int64, bool) {
-	m.fmut.RLock()
-	fs, ok := m.folderFiles[folder]
-	cfg := m.folderCfgs[folder]
-	m.fmut.RUnlock()
-
-	if !ok {
-		// The folder might not exist, since this can be called with a user
-		// specified folder name from the REST interface.
-		return 0, false
-	}
-
-	var ver int64
-	for _, device := range cfg.Devices {
-		ver += fs.Sequence(device.DeviceID)
-	}
-
-	return ver, true
-}
-
 func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} {
 	m.fmut.RLock()
 	files, ok := m.folderFiles[folder]
@@ -2402,7 +2231,9 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 		prefix = prefix + sep
 	}
 
-	files.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
+	snap := files.Snapshot()
+	defer snap.Release()
+	snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
 
 		// Don't include the prefix itself.
@@ -2514,8 +2345,10 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc
 	}
 
 	var availabilities []Availability
+	snap := fs.Snapshot()
+	defer snap.Release()
 next:
-	for _, device := range fs.Availability(file.Name) {
+	for _, device := range snap.Availability(file.Name) {
 		for _, pausedFolder := range m.remotePausedFolders[device] {
 			if pausedFolder == folder {
 				continue next

lib/model/model_test.go

@@ -2146,8 +2146,8 @@ func TestIssue3028(t *testing.T) {
 
 	// Get a count of how many files are there now
 
-	locorigfiles := m.LocalSize("default").Files
-	globorigfiles := m.GlobalSize("default").Files
+	locorigfiles := localSize(t, m, "default").Files
+	globorigfiles := globalSize(t, m, "default").Files
 
 	// Delete and rescan specifically these two
 
@@ -2158,8 +2158,8 @@ func TestIssue3028(t *testing.T) {
 	// Verify that the number of files decreased by two and the number of
 	// deleted files increases by two
 
-	loc := m.LocalSize("default")
-	glob := m.GlobalSize("default")
+	loc := localSize(t, m, "default")
+	glob := globalSize(t, m, "default")
 	if loc.Files != locorigfiles-2 {
 		t.Errorf("Incorrect local accounting; got %d current files, expected %d", loc.Files, locorigfiles-2)
 	}
@@ -2439,10 +2439,12 @@ func TestIssue3496(t *testing.T) {
 	fs := m.folderFiles["default"]
 	m.fmut.RUnlock()
 	var localFiles []protocol.FileInfo
-	fs.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
+	snap := fs.Snapshot()
+	snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
 		localFiles = append(localFiles, i.(protocol.FileInfo))
 		return true
 	})
+	snap.Release()
 
 	// Mark all files as deleted and fake it as update from device1
 
@@ -2482,7 +2484,7 @@ func TestIssue3496(t *testing.T) {
 	t.Log(comp)
 
 	// Check that NeedSize does the correct thing
-	need := m.NeedSize("default")
+	need := needSize(t, m, "default")
 	if need.Files != 1 || need.Bytes != 1234 {
 		// The one we added synthetically above
 		t.Errorf("Incorrect need size; %d, %d != 1, 1234", need.Files, need.Bytes)

lib/model/testutils_test.go

@@ -9,6 +9,7 @@ package model
 import (
 	"io/ioutil"
 	"os"
+	"testing"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
@@ -171,3 +172,40 @@ func (c *alwaysChanged) Seen(fs fs.Filesystem, name string) bool {
 func (c *alwaysChanged) Changed() bool {
 	return true
 }
+
+func localSize(t *testing.T, m Model, folder string) db.Counts {
+	t.Helper()
+	snap := dbSnapshot(t, m, folder)
+	defer snap.Release()
+	return snap.LocalSize()
+}
+
+func globalSize(t *testing.T, m Model, folder string) db.Counts {
+	t.Helper()
+	snap := dbSnapshot(t, m, folder)
+	defer snap.Release()
+	return snap.GlobalSize()
+}
+
+func receiveOnlyChangedSize(t *testing.T, m Model, folder string) db.Counts {
+	t.Helper()
+	snap := dbSnapshot(t, m, folder)
+	defer snap.Release()
+	return snap.ReceiveOnlyChangedSize()
+}
+
+func needSize(t *testing.T, m Model, folder string) db.Counts {
+	t.Helper()
+	snap := dbSnapshot(t, m, folder)
+	defer snap.Release()
+	return snap.NeedSize()
+}
+
+func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {
+	t.Helper()
+	snap, err := m.DBSnapshot(folder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return snap
+}