lib/db: Add hacky way to adjust database parameters (#5889)

This adds a set of magical environment variables that can be used to
tweak the database parameters. It's totally undocumented and not
intended to be a long term or supported thing.

It's ugly, but there is a backstory. I have a couple of large
installations where the database options are inefficient or otherwise
suboptimal (24/7 compaction going on and stuff like that). I don't
*know* the correct database parameters, nor yet the formula or method to
derive them by, so this requires experimentation. Experimentation needs
to happen partly in production, and rolling out new builds for every
tweak isn't practical. This provides override points for all reasonable
values, while not changing anything by default.

Ideally, at the end of such experimentation, we'll know which values are
relevant to change and in what manner, and can provide a more user
friendly knob to do so - or do it automatically based on the database
size.

lib/db/lowlevel.go

@@ -8,6 +8,7 @@ package db
 
 import (
 	"os"
+	"strconv"
 	"strings"
 	"sync/atomic"
 
@@ -15,6 +16,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 const (
@@ -22,6 +24,11 @@ const (
 	dbWriteBuffer  = 4 << 20
 )
 
+var (
+	// Flush batches to disk when they contain this many records.
+	batchFlushSize = debugEnvValue("BatchFlushSize", 64)
+)
+
 // Lowlevel is the lowest level database interface. It has a very simple
 // purpose: hold the actual *leveldb.DB database, and the in-memory state
 // that belong to that database. In the same way that a single on disk
@@ -40,8 +47,31 @@ type Lowlevel struct {
 // the database is erased and created from scratch.
 func Open(location string) (*Lowlevel, error) {
 	opts := &opt.Options{
-		OpenFilesCacheCapacity: dbMaxOpenFiles,
-		WriteBuffer:            dbWriteBuffer,
+		BlockCacheCapacity:            debugEnvValue("BlockCacheCapacity", 0),
+		BlockRestartInterval:          debugEnvValue("BlockRestartInterval", 0),
+		BlockSize:                     debugEnvValue("BlockSize", 0),
+		CompactionExpandLimitFactor:   debugEnvValue("CompactionExpandLimitFactor", 0),
+		CompactionGPOverlapsFactor:    debugEnvValue("CompactionGPOverlapsFactor", 0),
+		CompactionL0Trigger:           debugEnvValue("CompactionL0Trigger", 0),
+		CompactionSourceLimitFactor:   debugEnvValue("CompactionSourceLimitFactor", 0),
+		CompactionTableSize:           debugEnvValue("CompactionTableSize", 0),
+		CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", 0)) / 10.0,
+		CompactionTotalSize:           debugEnvValue("CompactionTotalSize", 0),
+		CompactionTotalSizeMultiplier: float64(debugEnvValue("CompactionTotalSizeMultiplier", 0)) / 10.0,
+		DisableBufferPool:             debugEnvValue("DisableBufferPool", 0) != 0,
+		DisableBlockCache:             debugEnvValue("DisableBlockCache", 0) != 0,
+		DisableCompactionBackoff:      debugEnvValue("DisableCompactionBackoff", 0) != 0,
+		DisableLargeBatchTransaction:  debugEnvValue("DisableLargeBatchTransaction", 0) != 0,
+		NoSync:                        debugEnvValue("NoSync", 0) != 0,
+		NoWriteMerge:                  debugEnvValue("NoWriteMerge", 0) != 0,
+		OpenFilesCacheCapacity:        debugEnvValue("OpenFilesCacheCapacity", dbMaxOpenFiles),
+		WriteBuffer:                   debugEnvValue("WriteBuffer", dbWriteBuffer),
+		// The write slowdown and pause can be overridden, but even if they
+		// are not and the compaction trigger is overridden we need to
+		// adjust so that we don't pause writes for L0 compaction before we
+		// even *start* L0 compaction...
+		WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", opt.DefaultCompactionL0Trigger)),
+		WriteL0PauseTrigger:    debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", opt.DefaultCompactionL0Trigger)),
 	}
 	return open(location, opts)
 }
@@ -73,6 +103,12 @@ func open(location string, opts *opt.Options) (*Lowlevel, error) {
 	if err != nil {
 		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
 	}
+
+	if debugEnvValue("CompactEverything", 0) != 0 {
+		if err := db.CompactRange(util.Range{}); err != nil {
+			l.Warnln("Compacting database:", err)
+		}
+	}
 	return NewLowlevel(db, location), nil
 }
 
@@ -132,3 +168,11 @@ func leveldbIsCorrupted(err error) bool {
 
 	return false
 }
+
+func debugEnvValue(key string, def int) int {
+	v, err := strconv.ParseInt(os.Getenv("STDEBUG_"+key), 10, 63)
+	if err != nil {
+		return def
+	}
+	return int(v)
+}

lib/db/transactions.go

@@ -12,9 +12,6 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-// Flush batches to disk when they contain this many records.
-const batchFlushSize = 64
-
 // A readOnlyTransaction represents a database snapshot.
 type readOnlyTransaction struct {
 	*leveldb.Snapshot