Add indirection for large version vectors. (#6376)

This adds indirection of large version vectors in the same manner as we
already to block lists. The effect is the same: less duplicated data in
some situations.

To mitigate the impact for when this indirection
wouldn't be needed I've added an indirection cutoff for both blocks and
the new version vector stuff: we don't do the indirection at all for
small block lists or small version vectors, instead storing it directly
like we used to do. This is faster for small files and small setups.

lib/db/schemaupdater.go

@@ -23,9 +23,10 @@ import (
 //   7: v0.14.53
 //   8-9: v1.4.0
 //   10-11: v1.6.0
+//   12: v1.7.0
 const (
-	dbVersion             = 11
-	dbMinSyncthingVersion = "v1.6.0"
+	dbVersion             = 12
+	dbMinSyncthingVersion = "v1.7.0"
 )
 
 type databaseDowngradeError struct {
@@ -88,6 +89,7 @@ func (db *schemaUpdater) updateSchema() error {
 		{9, db.updateSchemaTo9},
 		{10, db.updateSchemaTo10},
 		{11, db.updateSchemaTo11},
+		{12, db.updateSchemaTo12},
 	}
 
 	for _, m := range migrations {
@@ -453,7 +455,6 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 
 func (db *schemaUpdater) updateSchemaTo9(prev int) error {
 	// Loads and rewrites all files with blocks, to deduplicate block lists.
-	// Checks for missing or incorrect sequence entries and rewrites those.
 
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
@@ -461,6 +462,16 @@ func (db *schemaUpdater) updateSchemaTo9(prev int) error {
 	}
 	defer t.close()
 
+	if err := db.rewriteFiles(t); err != nil {
+		return err
+	}
+
+	db.recordTime(indirectGCTimeKey)
+
+	return t.Commit()
+}
+
+func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
 	it, err := t.NewPrefixIterator([]byte{KeyTypeDevice})
 	if err != nil {
 		return err
@@ -491,13 +502,7 @@ func (db *schemaUpdater) updateSchemaTo9(prev int) error {
 		}
 	}
 	it.Release()
-	if err := it.Error(); err != nil {
-		return err
-	}
-
-	db.recordTime(indirectGCTimeKey)
-
-	return t.Commit()
+	return it.Error()
 }
 
 func (db *schemaUpdater) updateSchemaTo10(_ int) error {
@@ -610,3 +615,19 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
 	}
 	return t.Commit()
 }
+
+func (db *schemaUpdater) updateSchemaTo12(_ int) error {
+	// Loads and rewrites all files, to deduplicate version vectors.
+
+	t, err := db.newReadWriteTransaction()
+	if err != nil {
+		return err
+	}
+	defer t.close()
+
+	if err := db.rewriteFiles(t); err != nil {
+		return err
+	}
+
+	return t.Commit()
+}