lib/db: Add Badger backend (fixes #5910) (#6250)

lib/db/backend/backend.go

@@ -7,7 +7,12 @@
 package backend
 
 import (
+	"os"
+	"strings"
 	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/locations"
 )
 
 // The Reader interface specifies the read-only operations available on the
@@ -118,10 +123,24 @@ const (
 )
 
 func Open(path string, tuning Tuning) (Backend, error) {
+	if os.Getenv("USE_BADGER") != "" {
+		l.Warnln("Using experimental badger db")
+		if err := maybeCopyDatabase(path, strings.Replace(path, locations.BadgerDir, locations.LevelDBDir, 1), OpenBadger, OpenLevelDBRO); err != nil {
+			return nil, err
+		}
+		return OpenBadger(path)
+	}
+
+	if err := maybeCopyDatabase(path, strings.Replace(path, locations.LevelDBDir, locations.BadgerDir, 1), OpenLevelDBAuto, OpenBadger); err != nil {
+		return nil, err
+	}
 	return OpenLevelDB(path, tuning)
 }
 
 func OpenMemory() Backend {
+	if os.Getenv("USE_BADGER") != "" {
+		return OpenBadgerMemory()
+	}
 	return OpenLevelDBMemory()
 }
 
@@ -202,3 +221,70 @@ func (cg *closeWaitGroup) CloseWait() {
 	cg.closeMut.Unlock()
 	cg.WaitGroup.Wait()
 }
+
+type opener func(path string) (Backend, error)
+
+// maybeCopyDatabase copies the database if the destination doesn't exist
+// but the source does.
+func maybeCopyDatabase(toPath, fromPath string, toOpen, fromOpen opener) error {
+	if _, err := os.Lstat(toPath); !os.IsNotExist(err) {
+		// Destination database exists (or is otherwise unavailable), do not
+		// attempt to overwrite it.
+		return nil
+	}
+
+	if _, err := os.Lstat(fromPath); err != nil {
+		// Source database is not available, so nothing to copy
+		return nil
+	}
+
+	fromDB, err := fromOpen(fromPath)
+	if err != nil {
+		return err
+	}
+	defer fromDB.Close()
+
+	toDB, err := toOpen(toPath)
+	if err != nil {
+		// That's odd, but it will be handled & reported in the usual path
+		// so we can ignore it here.
+		return err
+	}
+	defer toDB.Close()
+
+	l.Infoln("Copying database for format conversion...")
+	if err := copyBackend(toDB, fromDB); err != nil {
+		return err
+	}
+
+	// Move the old database out of the way to mark it as migrated.
+	fromDB.Close()
+	_ = os.Rename(fromPath, fromPath+".migrated."+time.Now().Format("20060102150405"))
+	return nil
+}
+
+func copyBackend(to, from Backend) error {
+	srcIt, err := from.NewPrefixIterator(nil)
+	if err != nil {
+		return err
+	}
+	defer srcIt.Release()
+
+	dstTx, err := to.NewWriteTransaction()
+	if err != nil {
+		return err
+	}
+	defer dstTx.Release()
+
+	for srcIt.Next() {
+		if err := dstTx.Put(srcIt.Key(), srcIt.Value()); err != nil {
+			return err
+		}
+	}
+	if srcIt.Error() != nil {
+		return err
+	}
+	srcIt.Release()
+
+	return dstTx.Commit()
+}

lib/db/backend/badger_backend.go

@@ -0,0 +1,443 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"bytes"
+	"errors"
+	"time"
+
+	badger "github.com/dgraph-io/badger/v2"
+)
+
+const (
+	checkpointFlushMinSize = 128 << KiB
+	maxCacheSize           = 64 << MiB
+)
+
+func OpenBadger(path string) (Backend, error) {
+	opts := badger.DefaultOptions(path)
+	opts = opts.WithMaxCacheSize(maxCacheSize).WithCompactL0OnClose(false)
+	opts.Logger = nil
+	return openBadger(opts)
+}
+
+func OpenBadgerMemory() Backend {
+	opts := badger.DefaultOptions("").WithInMemory(true)
+	opts.Logger = nil
+	backend, err := openBadger(opts)
+	if err != nil {
+		// Opening in-memory should never be able to fail, and is anyway
+		// used just by tests.
+		panic(err)
+	}
+	return backend
+}
+
+func openBadger(opts badger.Options) (Backend, error) {
+	// XXX: We should find good values for memory utilization in the "small"
+	// and "large" cases we support for LevelDB. Some notes here:
+	// https://github.com/dgraph-io/badger/tree/v2.0.3#memory-usage
+	bdb, err := badger.Open(opts)
+	if err != nil {
+		return nil, wrapBadgerErr(err)
+	}
+	return &badgerBackend{
+		bdb:     bdb,
+		closeWG: &closeWaitGroup{},
+	}, nil
+}
+
+// badgerBackend implements Backend on top of a badger
+type badgerBackend struct {
+	bdb     *badger.DB
+	closeWG *closeWaitGroup
+}
+
+func (b *badgerBackend) NewReadTransaction() (ReadTransaction, error) {
+	rel, err := newReleaser(b.closeWG)
+	if err != nil {
+		return nil, err
+	}
+	return badgerSnapshot{
+		txn: b.bdb.NewTransaction(false),
+		rel: rel,
+	}, nil
+}
+
+func (b *badgerBackend) NewWriteTransaction() (WriteTransaction, error) {
+	rel1, err := newReleaser(b.closeWG)
+	if err != nil {
+		return nil, err
+	}
+	rel2, err := newReleaser(b.closeWG)
+	if err != nil {
+		rel1.Release()
+		return nil, err
+	}
+
+	// We use two transactions here to preserve the property that our
+	// leveldb wrapper has, that writes in a transaction are completely
+	// invisible until it's committed, even inside that same transaction.
+	rtxn := b.bdb.NewTransaction(false)
+	wtxn := b.bdb.NewTransaction(true)
+	return &badgerTransaction{
+		badgerSnapshot: badgerSnapshot{
+			txn: rtxn,
+			rel: rel1,
+		},
+		txn: wtxn,
+		bdb: b.bdb,
+		rel: rel2,
+	}, nil
+}
+
+func (b *badgerBackend) Close() error {
+	b.closeWG.CloseWait()
+	return wrapBadgerErr(b.bdb.Close())
+}
+
+func (b *badgerBackend) Get(key []byte) ([]byte, error) {
+	if err := b.closeWG.Add(1); err != nil {
+		return nil, err
+	}
+	defer b.closeWG.Done()
+
+	txn := b.bdb.NewTransaction(false)
+	defer txn.Discard()
+	item, err := txn.Get(key)
+	if err != nil {
+		return nil, wrapBadgerErr(err)
+	}
+	val, err := item.ValueCopy(nil)
+	if err != nil {
+		return nil, wrapBadgerErr(err)
+	}
+	return val, nil
+}
+
+func (b *badgerBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	if err := b.closeWG.Add(1); err != nil {
+		return nil, err
+	}
+
+	txn := b.bdb.NewTransaction(false)
+	it := badgerPrefixIterator(txn, prefix)
+	it.releaseFn = func() {
+		defer b.closeWG.Done()
+		txn.Discard()
+	}
+	return it, nil
+}
+
+func (b *badgerBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
+	if err := b.closeWG.Add(1); err != nil {
+		return nil, err
+	}
+
+	txn := b.bdb.NewTransaction(false)
+	it := badgerRangeIterator(txn, first, last)
+	it.releaseFn = func() {
+		defer b.closeWG.Done()
+		txn.Discard()
+	}
+	return it, nil
+}
+
+func (b *badgerBackend) Put(key, val []byte) error {
+	if err := b.closeWG.Add(1); err != nil {
+		return err
+	}
+	defer b.closeWG.Done()
+
+	txn := b.bdb.NewTransaction(true)
+	if err := txn.Set(key, val); err != nil {
+		txn.Discard()
+		return wrapBadgerErr(err)
+	}
+	return wrapBadgerErr(txn.Commit())
+}
+
+func (b *badgerBackend) Delete(key []byte) error {
+	if err := b.closeWG.Add(1); err != nil {
+		return err
+	}
+	defer b.closeWG.Done()
+
+	txn := b.bdb.NewTransaction(true)
+	if err := txn.Delete(key); err != nil {
+		txn.Discard()
+		return wrapBadgerErr(err)
+	}
+	return wrapBadgerErr(txn.Commit())
+}
+
+func (b *badgerBackend) Compact() error {
+	if err := b.closeWG.Add(1); err != nil {
+		return err
+	}
+	defer b.closeWG.Done()
+
+	// This weird looking loop is as recommended in the README
+	// (https://github.com/dgraph-io/badger/tree/v2.0.3#garbage-collection).
+	// Basically, the RunValueLogGC will pick some promising thing to
+	// garbage collect at random and return nil if it improved the
+	// situation, then return ErrNoRewrite when there is nothing more to GC.
+	// The 0.5 is the discard ratio, for which the method docs say they
+	// "recommend setting discardRatio to 0.5, thus indicating that a file
+	// be rewritten if half the space can be discarded".
+	var err error
+	t0 := time.Now()
+	for err == nil {
+		if time.Since(t0) > time.Hour {
+			l.Warnln("Database compaction is taking a long time, performance may be impacted. Consider investigating and/or opening an issue if this warning repeats.")
+			t0 = time.Now()
+		}
+		err = b.bdb.RunValueLogGC(0.5)
+	}
+
+	if errors.Is(err, badger.ErrNoRewrite) {
+		// GC did nothing, because nothing needed to be done
+		return nil
+	}
+	if errors.Is(err, badger.ErrRejected) {
+		// GC was already running (could possibly happen), or the database
+		// is closed (can't happen).
+		return nil
+	}
+	if errors.Is(err, badger.ErrGCInMemoryMode) {
+		// GC in in-memory mode, which is fine.
+		return nil
+	}
+	return err
+}
+
+// badgerSnapshot implements backend.ReadTransaction
+type badgerSnapshot struct {
+	txn *badger.Txn
+	rel *releaser
+}
+
+func (l badgerSnapshot) Get(key []byte) ([]byte, error) {
+	item, err := l.txn.Get(key)
+	if err != nil {
+		return nil, wrapBadgerErr(err)
+	}
+	val, err := item.ValueCopy(nil)
+	if err != nil {
+		return nil, wrapBadgerErr(err)
+	}
+	return val, nil
+}
+
+func (l badgerSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return badgerPrefixIterator(l.txn, prefix), nil
+}
+
+func (l badgerSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return badgerRangeIterator(l.txn, first, last), nil
+}
+
+func (l badgerSnapshot) Release() {
+	defer l.rel.Release()
+	l.txn.Discard()
+}
+
+type badgerTransaction struct {
+	badgerSnapshot
+	txn  *badger.Txn
+	bdb  *badger.DB
+	rel  *releaser
+	size int
+}
+
+func (t *badgerTransaction) Delete(key []byte) error {
+	t.size += len(key)
+	kc := make([]byte, len(key))
+	copy(kc, key)
+	return t.transactionRetried(func(txn *badger.Txn) error {
+		return txn.Delete(kc)
+	})
+}
+
+func (t *badgerTransaction) Put(key, val []byte) error {
+	t.size += len(key) + len(val)
+	kc := make([]byte, len(key))
+	copy(kc, key)
+	vc := make([]byte, len(val))
+	copy(vc, val)
+	return t.transactionRetried(func(txn *badger.Txn) error {
+		return txn.Set(kc, vc)
+	})
+}
+
+// transactionRetried performs the given operation in the current
+// transaction, with commit and retry if Badger says the transaction has
+// grown too large.
+func (t *badgerTransaction) transactionRetried(fn func(*badger.Txn) error) error {
+	if err := fn(t.txn); err == badger.ErrTxnTooBig {
+		if err := t.txn.Commit(); err != nil {
+			return wrapBadgerErr(err)
+		}
+		t.size = 0
+		t.txn = t.bdb.NewTransaction(true)
+		return wrapBadgerErr(fn(t.txn))
+	} else if err != nil {
+		return wrapBadgerErr(err)
+	}
+	return nil
+}
+
+func (t *badgerTransaction) Commit() error {
+	defer t.rel.Release()
+	defer t.badgerSnapshot.Release()
+	return wrapBadgerErr(t.txn.Commit())
+}
+
+func (t *badgerTransaction) Checkpoint(preFlush ...func() error) error {
+	if t.size < checkpointFlushMinSize {
+		return nil
+	}
+	for _, hook := range preFlush {
+		if err := hook(); err != nil {
+			return err
+		}
+	}
+	err := t.txn.Commit()
+	if err == nil {
+		t.size = 0
+		t.txn = t.bdb.NewTransaction(true)
+	}
+	return wrapBadgerErr(err)
+}
+
+func (t *badgerTransaction) Release() {
+	defer t.rel.Release()
+	defer t.badgerSnapshot.Release()
+	t.txn.Discard()
+}
+
+type badgerIterator struct {
+	it        *badger.Iterator
+	prefix    []byte
+	first     []byte
+	last      []byte
+	releaseFn func()
+	didSeek   bool
+	err       error
+}
+
+func (i *badgerIterator) Next() bool {
+	if i.err != nil {
+		return false
+	}
+	for {
+		if !i.didSeek {
+			if i.first != nil {
+				// Range iterator
+				i.it.Seek(i.first)
+			} else {
+				// Prefix iterator
+				i.it.Seek(i.prefix)
+			}
+			i.didSeek = true
+		} else {
+			i.it.Next()
+		}
+
+		if !i.it.ValidForPrefix(i.prefix) {
+			// Done
+			return false
+		}
+		if i.first == nil && i.last == nil {
+			// No range checks required
+			return true
+		}
+
+		key := i.it.Item().Key()
+		if bytes.Compare(key, i.last) > 0 {
+			// Key is after range last
+			return false
+		}
+		return true
+	}
+}
+
+func (i *badgerIterator) Key() []byte {
+	if i.err != nil {
+		return nil
+	}
+	return i.it.Item().Key()
+}
+
+func (i *badgerIterator) Value() []byte {
+	if i.err != nil {
+		return nil
+	}
+	val, err := i.it.Item().ValueCopy(nil)
+	if err != nil {
+		i.err = err
+	}
+	return val
+}
+
+func (i *badgerIterator) Error() error {
+	return wrapBadgerErr(i.err)
+}
+
+func (i *badgerIterator) Release() {
+	i.it.Close()
+	if i.releaseFn != nil {
+		i.releaseFn()
+	}
+}
+
+// wrapBadgerErr wraps errors so that the backend package can recognize them
+func wrapBadgerErr(err error) error {
+	if err == nil {
+		return nil
+	}
+	if err == badger.ErrDiscardedTxn {
+		return errClosed{}
+	}
+	if err == badger.ErrKeyNotFound {
+		return errNotFound{}
+	}
+	return err
+}
+
+func badgerPrefixIterator(txn *badger.Txn, prefix []byte) *badgerIterator {
+	it := iteratorForPrefix(txn, prefix)
+	return &badgerIterator{it: it, prefix: prefix}
+}
+
+func badgerRangeIterator(txn *badger.Txn, first, last []byte) *badgerIterator {
+	prefix := commonPrefix(first, last)
+	it := iteratorForPrefix(txn, prefix)
+	return &badgerIterator{it: it, prefix: prefix, first: first, last: last}
+}
+
+func iteratorForPrefix(txn *badger.Txn, prefix []byte) *badger.Iterator {
+	opts := badger.DefaultIteratorOptions
+	opts.Prefix = prefix
+	return txn.NewIterator(opts)
+}
+
+func commonPrefix(a, b []byte) []byte {
+	minLen := len(a)
+	if len(b) < minLen {
+		minLen = len(b)
+	}
+	prefix := make([]byte, 0, minLen)
+	for i := 0; i < minLen; i++ {
+		if a[i] != b[i] {
+			break
+		}
+		prefix = append(prefix, a[i])
+	}
+	return prefix
+}

lib/db/backend/badger_test.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import "testing"
+
+func TestCommonPrefix(t *testing.T) {
+	cases := []struct {
+		a      string
+		b      string
+		common string
+	}{
+		{"", "", ""},
+		{"a", "b", ""},
+		{"aa", "ab", "a"},
+		{"aa", "a", "a"},
+		{"a", "aa", "a"},
+		{"aabab", "ab", "a"},
+		{"ab", "aabab", "a"},
+		{"abac", "ababab", "aba"},
+		{"ababab", "abac", "aba"},
+	}
+
+	for _, tc := range cases {
+		pref := string(commonPrefix([]byte(tc.a), []byte(tc.b)))
+		if pref != tc.common {
+			t.Errorf("commonPrefix(%q, %q) => %q, expected %q", tc.a, tc.b, pref, tc.common)
+		}
+	}
+}
+
+func TestBadgerBackendBehavior(t *testing.T) {
+	testBackendBehavior(t, OpenBadgerMemory)
+}

lib/db/backend/leveldb_open.go

@@ -33,7 +33,7 @@ const (
 	MiB = 20
 )
 
-// Open attempts to open the database at the given location, and runs
+// OpenLevelDB attempts to open the database at the given location, and runs
 // recovery on it if opening fails. Worst case, if recovery is not possible,
 // the database is erased and created from scratch.
 func OpenLevelDB(location string, tuning Tuning) (Backend, error) {
@@ -45,7 +45,13 @@ func OpenLevelDB(location string, tuning Tuning) (Backend, error) {
 	return newLeveldbBackend(ldb), nil
 }
 
-// OpenRO attempts to open the database at the given location, read only.
+// OpenLevelDBAuto is OpenLevelDB with TuningAuto tuning.
+func OpenLevelDBAuto(location string) (Backend, error) {
+	return OpenLevelDB(location, TuningAuto)
+}
+
+// OpenLevelDBRO attempts to open the database at the given location, read
+// only.
 func OpenLevelDBRO(location string) (Backend, error) {
 	opts := &opt.Options{
 		OpenFilesCacheCapacity: dbMaxOpenFiles,

lib/db/lowlevel.go

@@ -382,6 +382,8 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	if err != nil {
 		return err
 	}
+	defer dbi.Release()
+
 	var gk, keyBuf []byte
 	for dbi.Next() {
 		name := db.keyer.NameFromDeviceFileKey(dbi.Key())
@@ -400,10 +402,10 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 			return err
 		}
 	}
+	dbi.Release()
 	if err := dbi.Error(); err != nil {
 		return err
 	}
-	dbi.Release()
 
 	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
 		key, err := db.keyer.GenerateBlockMapKey(nil, folder, nil, nil)
@@ -491,6 +493,7 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
 			}
 		}
 	}
+	dbi.Release()
 	if err := dbi.Error(); err != nil {
 		return err
 	}

lib/db/schemaupdater.go

@@ -212,6 +212,10 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			return err
 		}
 	}
+	dbi.Release()
+	if err != dbi.Error() {
+		return err
+	}
 
 	for folder := range changedFolders {
 		if err := db.dropFolderMeta([]byte(folder)); err != nil {

lib/db/transactions.go

@@ -509,7 +509,9 @@ func (db *Lowlevel) newReadWriteTransaction() (readWriteTransaction, error) {
 }
 
 func (t readWriteTransaction) Commit() error {
-	t.readOnlyTransaction.close()
+	// The readOnlyTransaction must close after commit, because they may be
+	// backed by the same actual lower level transaction.
+	defer t.readOnlyTransaction.close()
 	return t.WriteTransaction.Commit()
 }
 

lib/locations/locations.go

@@ -44,12 +44,20 @@ const (
 	DataBaseDir   BaseDirEnum = "data"
 	// User's home directory, *not* -home flag
 	UserHomeBaseDir BaseDirEnum = "userHome"
+
+	LevelDBDir = "index-v0.14.0.db"
+	BadgerDir  = "indexdb.badger"
 )
 
 // Platform dependent directories
 var baseDirs = make(map[BaseDirEnum]string, 3)
 
 func init() {
+	if os.Getenv("USE_BADGER") != "" {
+		// XXX: Replace the leveldb name with the badger name.
+		locationTemplates[Database] = strings.Replace(locationTemplates[Database], LevelDBDir, BadgerDir, 1)
+	}
+
 	userHome := userHomeDir()
 	config := defaultConfigDir(userHome)
 	baseDirs[UserHomeBaseDir] = userHome
@@ -80,8 +88,6 @@ func GetBaseDir(baseDir BaseDirEnum) string {
 	return baseDirs[baseDir]
 }
 
-var databaseDirname = "index-v0.14.0.db"
-
 // Use the variables from baseDirs here
 var locationTemplates = map[LocationEnum]string{
 	ConfigFile:    "${config}/config.xml",
@@ -89,7 +95,7 @@ var locationTemplates = map[LocationEnum]string{
 	KeyFile:       "${config}/key.pem",
 	HTTPSCertFile: "${config}/https-cert.pem",
 	HTTPSKeyFile:  "${config}/https-key.pem",
-	Database:      "${data}/" + databaseDirname,
+	Database:      "${data}/" + LevelDBDir,
 	LogFile:       "${data}/syncthing.log", // -logfile on Windows
 	CsrfTokens:    "${data}/csrftokens.txt",
 	PanicLog:      "${data}/panic-${timestamp}.log",
@@ -150,7 +156,14 @@ func defaultDataDir(userHome, config string) string {
 
 	default:
 		// If a database exists at the "normal" location, use that anyway.
-		if _, err := os.Lstat(filepath.Join(config, databaseDirname)); err == nil {
+		// We look for both LevelDB and Badger variants here regardless of
+		// what we're currently configured to use, because we might be
+		// starting up in Badger mode with only a LevelDB database present
+		// (will be converted), or vice versa.
+		if _, err := os.Lstat(filepath.Join(config, LevelDBDir)); err == nil {
+			return config
+		}
+		if _, err := os.Lstat(filepath.Join(config, BadgerDir)); err == nil {
 			return config
 		}
 		// Always use this env var, as it's explicitly set by the user