lib/db, lib/model: Keep need stats in metadata (ref #5899) (#6413)

lib/model/devicedownloadstate.go

@@ -18,6 +18,7 @@ import (
 type deviceFolderFileDownloadState struct {
 	blockIndexes []int32
 	version      protocol.Vector
+	blockSize    int
 }
 
 // deviceFolderDownloadState holds current download state of all files that
@@ -62,10 +63,12 @@ func (p *deviceFolderDownloadState) Update(updates []protocol.FileDownloadProgre
 				local = deviceFolderFileDownloadState{
 					blockIndexes: update.BlockIndexes,
 					version:      update.Version,
+					blockSize:    int(update.BlockSize),
 				}
 			} else if !local.version.Equal(update.Version) {
 				local.blockIndexes = append(local.blockIndexes[:0], update.BlockIndexes...)
 				local.version = update.Version
+				local.blockSize = int(update.BlockSize)
 			} else {
 				local.blockIndexes = append(local.blockIndexes, update.BlockIndexes...)
 			}
@@ -74,6 +77,20 @@ func (p *deviceFolderDownloadState) Update(updates []protocol.FileDownloadProgre
 	}
 }
 
+func (p *deviceFolderDownloadState) BytesDownloaded() int64 {
+	var res int64
+	for _, state := range p.files {
+		// BlockSize is a new field introduced in 1.4.1, thus a fallback
+		// is required (will potentially underrepresent downloaded bytes).
+		if state.blockSize != 0 {
+			res += int64(len(state.blockIndexes) * state.blockSize)
+		} else {
+			res += int64(len(state.blockIndexes) * protocol.MinBlockSize)
+		}
+	}
+	return res
+}
+
 // GetBlockCounts returns a map filename -> number of blocks downloaded.
 func (p *deviceFolderDownloadState) GetBlockCounts() map[string]int {
 	p.mut.RLock()
@@ -150,6 +167,22 @@ func (t *deviceDownloadState) GetBlockCounts(folder string) map[string]int {
 	return nil
 }
 
+func (t *deviceDownloadState) BytesDownloaded(folder string) int64 {
+	if t == nil {
+		return 0
+	}
+
+	t.mut.RLock()
+	defer t.mut.RUnlock()
+
+	for name, state := range t.folders {
+		if name == folder {
+			return state.BytesDownloaded()
+		}
+	}
+	return 0
+}
+
 func newDeviceDownloadState() *deviceDownloadState {
 	return &deviceDownloadState{
 		mut:     sync.NewRWMutex(),

lib/model/folder_summary.go

@@ -86,7 +86,7 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
 		if snap, err = c.model.DBSnapshot(folder); err == nil {
 			global = snap.GlobalSize()
 			local = snap.LocalSize()
-			need = snap.NeedSize()
+			need = snap.NeedSize(protocol.LocalDeviceID)
 			ro = snap.ReceiveOnlyChangedSize()
 			ourSeq = snap.Sequence(protocol.LocalDeviceID)
 			remoteSeq = snap.Sequence(protocol.GlobalDeviceID)

lib/model/model.go

@@ -714,9 +714,9 @@ func (m *model) FolderStatistics() (map[string]stats.FolderStatistics, error) {
 type FolderCompletion struct {
 	CompletionPct float64
 	NeedBytes     int64
-	NeedItems     int64
 	GlobalBytes   int64
-	NeedDeletes   int64
+	NeedItems     int32
+	NeedDeletes   int32
 }
 
 // Map returns the members as a map, e.g. used in api to serialize as Json.
@@ -752,52 +752,35 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 	}
 
 	m.pmut.RLock()
-	counts := m.deviceDownloads[device].GetBlockCounts(folder)
+	downloaded := m.deviceDownloads[device].BytesDownloaded(folder)
 	m.pmut.RUnlock()
 
-	var need, items, fileNeed, downloaded, deletes int64
-	snap.WithNeedTruncated(device, func(f db.FileIntf) bool {
-		ft := f.(db.FileInfoTruncated)
+	need := snap.NeedSize(device)
+	need.Bytes -= downloaded
+	// This might might be more than it really is, because some blocks can be of a smaller size.
+	if need.Bytes < 0 {
+		need.Bytes = 0
+	}
 
-		// If the file is deleted, we account it only in the deleted column.
-		if ft.Deleted {
-			deletes++
-			return true
-		}
-
-		// This might might be more than it really is, because some blocks can be of a smaller size.
-		downloaded = int64(counts[ft.Name]) * int64(ft.BlockSize())
-
-		fileNeed = ft.FileSize() - downloaded
-		if fileNeed < 0 {
-			fileNeed = 0
-		}
-
-		need += fileNeed
-		items++
-
-		return true
-	})
-
-	needRatio := float64(need) / float64(tot)
+	needRatio := float64(need.Bytes) / float64(tot)
 	completionPct := 100 * (1 - needRatio)
 
 	// If the completion is 100% but there are deletes we need to handle,
 	// drop it down a notch. Hack for consumers that look only at the
 	// percentage (our own GUI does the same calculation as here on its own
 	// and needs the same fixup).
-	if need == 0 && deletes > 0 {
+	if need.Bytes == 0 && need.Deleted > 0 {
 		completionPct = 95 // chosen by fair dice roll
 	}
 
-	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)
+	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need.Bytes, tot, needRatio)
 
 	return FolderCompletion{
 		CompletionPct: completionPct,
-		NeedBytes:     need,
-		NeedItems:     items,
+		NeedBytes:     need.Bytes,
+		NeedItems:     need.Files + need.Directories + need.Symlinks,
 		GlobalBytes:   tot,
-		NeedDeletes:   deletes,
+		NeedDeletes:   need.Deleted,
 	}
 }
 

lib/model/sentdownloadstate.go

@@ -19,6 +19,7 @@ type sentFolderFileDownloadState struct {
 	version      protocol.Vector
 	updated      time.Time
 	created      time.Time
+	blockSize    int
 }
 
 // sentFolderDownloadState represents a state of what we've announced as available
@@ -43,6 +44,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 		pullerVersion := puller.file.Version
 		pullerBlockIndexesUpdated := puller.AvailableUpdated()
 		pullerCreated := puller.created
+		pullerBlockSize := int32(puller.file.BlockSize())
 
 		localFile, ok := s.files[name]
 
@@ -55,6 +57,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 					updated:      pullerBlockIndexesUpdated,
 					version:      pullerVersion,
 					created:      pullerCreated,
+					blockSize:    int(pullerBlockSize),
 				}
 
 				updates = append(updates, protocol.FileDownloadProgressUpdate{
@@ -62,6 +65,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 					Version:      pullerVersion,
 					UpdateType:   protocol.UpdateTypeAppend,
 					BlockIndexes: pullerBlockIndexes,
+					BlockSize:    pullerBlockSize,
 				})
 			}
 			continue
@@ -86,11 +90,13 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 				Version:      pullerVersion,
 				UpdateType:   protocol.UpdateTypeAppend,
 				BlockIndexes: pullerBlockIndexes,
+				BlockSize:    pullerBlockSize,
 			})
 			localFile.blockIndexes = pullerBlockIndexes
 			localFile.updated = pullerBlockIndexesUpdated
 			localFile.version = pullerVersion
 			localFile.created = pullerCreated
+			localFile.blockSize = int(pullerBlockSize)
 			continue
 		}
 
@@ -108,6 +114,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
 				Version:      localFile.version,
 				UpdateType:   protocol.UpdateTypeAppend,
 				BlockIndexes: newBlocks,
+				BlockSize:    pullerBlockSize,
 			})
 		}
 	}

lib/model/testutils_test.go

@@ -207,7 +207,7 @@ func needSize(t *testing.T, m Model, folder string) db.Counts {
 	t.Helper()
 	snap := dbSnapshot(t, m, folder)
 	defer snap.Release()
-	return snap.NeedSize()
+	return snap.NeedSize(protocol.LocalDeviceID)
 }
 
 func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {