lib: Revert needing invalid files (fixes #7608, ref #7476) (#7609)

lib/db/schemaupdater.go

@@ -20,7 +20,7 @@ import (
 // do not put restrictions on downgrades (e.g. for repairs after a bugfix).
 const (
 	dbVersion             = 14
-	dbMigrationVersion    = 16
+	dbMigrationVersion    = 17
 	dbMinSyncthingVersion = "v1.9.0"
 )
 
@@ -101,6 +101,7 @@ func (db *schemaUpdater) updateSchema() error {
 		{13, 13, "v1.7.0", db.updateSchemaTo13},
 		{14, 14, "v1.9.0", db.updateSchemaTo14},
 		{14, 16, "v1.9.0", db.checkRepairMigration},
+		{14, 17, "v1.9.0", db.migration17},
 	}
 
 	for _, m := range migrations {
@@ -783,6 +784,53 @@ func (db *schemaUpdater) checkRepairMigration(_ int) error {
 	return nil
 }
 
+// migration17 finds all files that were pulled as invalid from an invalid
+// global and make sure they get scanned/pulled again.
+func (db *schemaUpdater) migration17(prev int) error {
+	if prev < 16 {
+		// Issue was introduced in migration to 16
+		return nil
+	}
+	t, err := db.newReadOnlyTransaction()
+	if err != nil {
+		return err
+	}
+	defer t.close()
+
+	for _, folderStr := range db.ListFolders() {
+		folder := []byte(folderStr)
+		meta, err := db.loadMetadataTracker(folderStr)
+		if err != nil {
+			return err
+		}
+		batch := NewFileInfoBatch(func(fs []protocol.FileInfo) error {
+			return db.updateLocalFiles(folder, fs, meta)
+		})
+		var innerErr error
+		err = t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(fi protocol.FileIntf) bool {
+			if fi.IsInvalid() && fi.FileLocalFlags() == 0 {
+				f := fi.(protocol.FileInfo)
+				f.SetMustRescan()
+				f.Version = protocol.Vector{}
+				batch.Append(f)
+				innerErr = batch.FlushIfFull()
+				return innerErr == nil
+			}
+			return true
+		})
+		if innerErr != nil {
+			return innerErr
+		}
+		if err != nil {
+			return err
+		}
+		if err := batch.Flush(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (db *schemaUpdater) rewriteGlobals(t readWriteTransaction) error {
 	it, err := t.NewPrefixIterator([]byte{KeyTypeGlobal})
 	if err != nil {

lib/db/set_test.go

@@ -472,7 +472,6 @@ func TestNeedWithInvalid(t *testing.T) {
 		remote0Have[0],
 		remote1Have[0],
 		remote0Have[2],
-		remote1Have[2],
 	}
 
 	replace(s, protocol.LocalDeviceID, localHave)

lib/db/transactions.go

@@ -774,8 +774,10 @@ func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add b
 }
 
 func Need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool {
-	// We never need a file without a valid version.
-	if global.Version.IsEmpty() {
+	// We never need an invalid file or a file without a valid version (just
+	// another way of expressing "invalid", really, until we fix that
+	// part...).
+	if global.IsInvalid() || global.Version.IsEmpty() {
 		return false
 	}
 	// We don't need a deleted file if we don't have it.

lib/db/util.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import "github.com/syncthing/syncthing/lib/protocol"
+
+// How many files to send in each Index/IndexUpdate message.
+const (
+	MaxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
+	MaxBatchSizeFiles = 1000       // Either way, don't include more files than this
+)
+
+// FileInfoBatch is a utility to do file operations on the database in suitably
+// sized batches.
+type FileInfoBatch struct {
+	infos   []protocol.FileInfo
+	size    int
+	flushFn func([]protocol.FileInfo) error
+}
+
+func NewFileInfoBatch(fn func([]protocol.FileInfo) error) *FileInfoBatch {
+	return &FileInfoBatch{
+		infos:   make([]protocol.FileInfo, 0, MaxBatchSizeFiles),
+		flushFn: fn,
+	}
+}
+
+func (b *FileInfoBatch) SetFlushFunc(fn func([]protocol.FileInfo) error) {
+	b.flushFn = fn
+}
+
+func (b *FileInfoBatch) Append(f protocol.FileInfo) {
+	b.infos = append(b.infos, f)
+	b.size += f.ProtoSize()
+}
+
+func (b *FileInfoBatch) Full() bool {
+	return len(b.infos) >= MaxBatchSizeFiles || b.size >= MaxBatchSizeBytes
+}
+
+func (b *FileInfoBatch) FlushIfFull() error {
+	if b.Full() {
+		return b.Flush()
+	}
+	return nil
+}
+
+func (b *FileInfoBatch) Flush() error {
+	if len(b.infos) == 0 {
+		return nil
+	}
+	if err := b.flushFn(b.infos); err != nil {
+		return err
+	}
+	b.Reset()
+	return nil
+}
+
+func (b *FileInfoBatch) Reset() {
+	b.infos = b.infos[:0]
+	b.size = 0
+}
+
+func (b *FileInfoBatch) Size() int {
+	return b.size
+}