lib/model, lib/protocol: Index sending/receiving debugging (#9657)

This adds guardrails to the index sending and receiving, to verify that
what we thinks is happening is what actually happens.

lib/db/util.go

@@ -20,10 +20,14 @@ type FileInfoBatch struct {
 	infos   []protocol.FileInfo
 	size    int
 	flushFn func([]protocol.FileInfo) error
+	error   error
 }
 
 // NewFileInfoBatch returns a new FileInfoBatch that calls fn when it's time
-// to flush.
+// to flush. Errors from the flush function are considered non-recoverable;
+// once an error is returned the flush function wil not be called again, and
+// any further calls to Flush will return the same error (unless Reset is
+// called).
 func NewFileInfoBatch(fn func([]protocol.FileInfo) error) *FileInfoBatch {
 	return &FileInfoBatch{flushFn: fn}
 }
@@ -33,6 +37,9 @@ func (b *FileInfoBatch) SetFlushFunc(fn func([]protocol.FileInfo) error) {
 }
 
 func (b *FileInfoBatch) Append(f protocol.FileInfo) {
+	if b.error != nil {
+		panic("bug: calling append on a failed batch")
+	}
 	if b.infos == nil {
 		b.infos = make([]protocol.FileInfo, 0, MaxBatchSizeFiles)
 	}
@@ -45,6 +52,9 @@ func (b *FileInfoBatch) Full() bool {
 }
 
 func (b *FileInfoBatch) FlushIfFull() error {
+	if b.error != nil {
+		return b.error
+	}
 	if b.Full() {
 		return b.Flush()
 	}
@@ -52,10 +62,14 @@ func (b *FileInfoBatch) FlushIfFull() error {
 }
 
 func (b *FileInfoBatch) Flush() error {
+	if b.error != nil {
+		return b.error
+	}
 	if len(b.infos) == 0 {
 		return nil
 	}
 	if err := b.flushFn(b.infos); err != nil {
+		b.error = err
 		return err
 	}
 	b.Reset()
@@ -64,6 +78,7 @@ func (b *FileInfoBatch) Flush() error {
 
 func (b *FileInfoBatch) Reset() {
 	b.infos = nil
+	b.error = nil
 	b.size = 0
 }