fix(db): handle large numbers of blocks in update (#10025)

Avoid failure when inserting file with very large block list

internal/db/sqlite/db_test.go

@@ -1055,6 +1055,39 @@ func TestBlocklistGarbageCollection(t *testing.T) {
 	}
 }
 
+func TestInsertLargeFile(t *testing.T) {
+	t.Parallel()
+
+	sdb, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := sdb.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Add a large file (many blocks)
+
+	files := []protocol.FileInfo{genFile("test1", 16000, 1)}
+	if err := sdb.Update(folderID, protocol.LocalDeviceID, files); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify all the blocks are here
+
+	for i, block := range files[0].Blocks {
+		bs, err := itererr.Collect(sdb.AllLocalBlocksWithHash(block.Hash))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(bs) == 0 {
+			t.Error("missing blocks for", i)
+		}
+	}
+}
+
 func TestErrorWrap(t *testing.T) {
 	if wrap(nil, "foo") != nil {
 		t.Fatal("nil should wrap to nil")

internal/db/sqlite/db_update.go

@@ -310,11 +310,18 @@ func (*DB) insertBlocksLocked(tx *txPreparedStmts, blocklistHash []byte, blocks
 			"size":           b.Size,
 		}
 	}
-	_, err := tx.NamedExec(`
+
-		INSERT OR IGNORE INTO blocks (hash, blocklist_hash, idx, offset, size)
+	// Very large block lists (>8000 blocks) result in "too many variables"
-		VALUES (:hash, :blocklist_hash, :idx, :offset, :size)
+	// error. Chunk it to a reasonable size.
-	`, bs)
+	for chunk := range slices.Chunk(bs, 1000) {
-	return wrap(err)
+		if _, err := tx.NamedExec(`
+			INSERT OR IGNORE INTO blocks (hash, blocklist_hash, idx, offset, size)
+			VALUES (:hash, :blocklist_hash, :idx, :offset, :size)
+		`, chunk); err != nil {
+			return wrap(err)
+		}
+	}
+	return nil
 }
 
 func (s *DB) recalcGlobalForFolder(txp *txPreparedStmts, folderIdx int64) error {