chore: switch database engine to sqlite (fixes #9954) (#9965)

Switch the database from LevelDB to SQLite, for greater stability and
simpler code.

Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl>
Co-authored-by: bt90 <btom1990@googlemail.com>

internal/db/olddb/backend/backend.go

@@ -0,0 +1,166 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"errors"
+	"sync"
+)
+
+// CommitHook is a function that is executed before a WriteTransaction is
+// committed or before it is flushed to disk, e.g. on calling CheckPoint. The
+// transaction can be accessed via a closure.
+type CommitHook func(WriteTransaction) error
+
+// The Reader interface specifies the read-only operations available on the
+// main database and on read-only transactions (snapshots). Note that when
+// called directly on the database handle these operations may take implicit
+// transactions and performance may suffer.
+type Reader interface {
+	Get(key []byte) ([]byte, error)
+	NewPrefixIterator(prefix []byte) (Iterator, error)
+	NewRangeIterator(first, last []byte) (Iterator, error)
+}
+
+// The Writer interface specifies the mutating operations available on the
+// main database and on writable transactions. Note that when called
+// directly on the database handle these operations may take implicit
+// transactions and performance may suffer.
+type Writer interface {
+	Put(key, val []byte) error
+	Delete(key []byte) error
+}
+
+// The ReadTransaction interface specifies the operations on read-only
+// transactions. Every ReadTransaction must be released when no longer
+// required.
+type ReadTransaction interface {
+	Reader
+	Release()
+}
+
+// The WriteTransaction interface specifies the operations on writable
+// transactions. Every WriteTransaction must be either committed or released
+// (i.e., discarded) when no longer required. No further operations must be
+// performed after release or commit (regardless of whether commit succeeded),
+// with one exception -- it's fine to release an already committed or released
+// transaction.
+//
+// A Checkpoint is a potential partial commit of the transaction so far, for
+// purposes of saving memory when transactions are in-RAM. Note that
+// transactions may be checkpointed *anyway* even if this is not called, due to
+// resource constraints, but this gives you a chance to decide when. If, and
+// only if, calling Checkpoint will result in a partial commit/flush, the
+// CommitHooks passed to Backend.NewWriteTransaction are called before
+// committing. If any of those returns an error, committing is aborted and the
+// error bubbled.
+type WriteTransaction interface {
+	ReadTransaction
+	Writer
+	Checkpoint() error
+	Commit() error
+}
+
+// The Iterator interface specifies the operations available on iterators
+// returned by NewPrefixIterator and NewRangeIterator. The iterator pattern
+// is to loop while Next returns true, then check Error after the loop. Next
+// will return false when iteration is complete (Error() == nil) or when
+// there is an error preventing iteration, which is then returned by
+// Error(). For example:
+//
+//	it, err := db.NewPrefixIterator(nil)
+//	if err != nil {
+//	    // problem preventing iteration
+//	}
+//	defer it.Release()
+//	for it.Next() {
+//	    // ...
+//	}
+//	if err := it.Error(); err != nil {
+//	    // there was a database problem while iterating
+//	}
+//
+// An iterator must be Released when no longer required. The Error method
+// can be called either before or after Release with the same results. If an
+// iterator was created in a transaction (whether read-only or write) it
+// must be released before the transaction is released (or committed).
+type Iterator interface {
+	Next() bool
+	Key() []byte
+	Value() []byte
+	Error() error
+	Release()
+}
+
+// The Backend interface represents the main database handle. It supports
+// both read/write operations and opening read-only or writable
+// transactions. Depending on the actual implementation, individual
+// read/write operations may be implicitly wrapped in transactions, making
+// them perform quite badly when used repeatedly. For bulk operations,
+// consider always using a transaction of the appropriate type. The
+// transaction isolation level is "read committed" - there are no dirty
+// reads.
+// Location returns the path to the database, as given to Open. The returned string
+// is empty for a db in memory.
+type Backend interface {
+	Reader
+	NewReadTransaction() (ReadTransaction, error)
+	Close() error
+}
+
+var (
+	errClosed   = errors.New("database is closed")
+	errNotFound = errors.New("key not found")
+)
+
+func IsClosed(err error) bool   { return errors.Is(err, errClosed) }
+func IsNotFound(err error) bool { return errors.Is(err, errNotFound) }
+
+// releaser manages counting on top of a waitgroup
+type releaser struct {
+	wg   *closeWaitGroup
+	once sync.Once
+}
+
+func newReleaser(wg *closeWaitGroup) (*releaser, error) {
+	if err := wg.Add(1); err != nil {
+		return nil, err
+	}
+	return &releaser{wg: wg}, nil
+}
+
+func (r *releaser) Release() {
+	// We use the Once because we may get called multiple times from
+	// Commit() and deferred Release().
+	r.once.Do(r.wg.Done)
+}
+
+// closeWaitGroup behaves just like a sync.WaitGroup, but does not require
+// a single routine to do the Add and Wait calls. If Add is called after
+// CloseWait, it will return an error, and both are safe to be used concurrently.
+type closeWaitGroup struct {
+	sync.WaitGroup
+	closed   bool
+	closeMut sync.RWMutex
+}
+
+func (cg *closeWaitGroup) Add(i int) error {
+	cg.closeMut.RLock()
+	defer cg.closeMut.RUnlock()
+	if cg.closed {
+		return errClosed
+	}
+	cg.WaitGroup.Add(i)
+	return nil
+}
+
+func (cg *closeWaitGroup) CloseWait() {
+	cg.closeMut.Lock()
+	cg.closed = true
+	cg.closeMut.Unlock()
+	cg.WaitGroup.Wait()
+}

internal/db/olddb/backend/leveldb_backend.go

@@ -0,0 +1,113 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// leveldbBackend implements Backend on top of a leveldb
+type leveldbBackend struct {
+	ldb      *leveldb.DB
+	closeWG  *closeWaitGroup
+	location string
+}
+
+func newLeveldbBackend(ldb *leveldb.DB, location string) *leveldbBackend {
+	return &leveldbBackend{
+		ldb:      ldb,
+		closeWG:  &closeWaitGroup{},
+		location: location,
+	}
+}
+
+func (b *leveldbBackend) NewReadTransaction() (ReadTransaction, error) {
+	return b.newSnapshot()
+}
+
+func (b *leveldbBackend) newSnapshot() (leveldbSnapshot, error) {
+	rel, err := newReleaser(b.closeWG)
+	if err != nil {
+		return leveldbSnapshot{}, err
+	}
+	snap, err := b.ldb.GetSnapshot()
+	if err != nil {
+		rel.Release()
+		return leveldbSnapshot{}, wrapLeveldbErr(err)
+	}
+	return leveldbSnapshot{
+		snap: snap,
+		rel:  rel,
+	}, nil
+}
+
+func (b *leveldbBackend) Close() error {
+	b.closeWG.CloseWait()
+	return wrapLeveldbErr(b.ldb.Close())
+}
+
+func (b *leveldbBackend) Get(key []byte) ([]byte, error) {
+	val, err := b.ldb.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (b *leveldbBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return &leveldbIterator{b.ldb.NewIterator(util.BytesPrefix(prefix), nil)}, nil
+}
+
+func (b *leveldbBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return &leveldbIterator{b.ldb.NewIterator(&util.Range{Start: first, Limit: last}, nil)}, nil
+}
+
+func (b *leveldbBackend) Location() string {
+	return b.location
+}
+
+// leveldbSnapshot implements backend.ReadTransaction
+type leveldbSnapshot struct {
+	snap *leveldb.Snapshot
+	rel  *releaser
+}
+
+func (l leveldbSnapshot) Get(key []byte) ([]byte, error) {
+	val, err := l.snap.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (l leveldbSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return l.snap.NewIterator(util.BytesPrefix(prefix), nil), nil
+}
+
+func (l leveldbSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return l.snap.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
+}
+
+func (l leveldbSnapshot) Release() {
+	l.snap.Release()
+	l.rel.Release()
+}
+
+type leveldbIterator struct {
+	iterator.Iterator
+}
+
+func (it *leveldbIterator) Error() error {
+	return wrapLeveldbErr(it.Iterator.Error())
+}
+
+// wrapLeveldbErr wraps errors so that the backend package can recognize them
+func wrapLeveldbErr(err error) error {
+	switch err {
+	case leveldb.ErrClosed:
+		return errClosed
+	case leveldb.ErrNotFound:
+		return errNotFound
+	}
+	return err
+}

internal/db/olddb/backend/leveldb_open.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+const dbMaxOpenFiles = 100
+
+// OpenLevelDBRO attempts to open the database at the given location, read
+// only.
+func OpenLevelDBRO(location string) (Backend, error) {
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: dbMaxOpenFiles,
+		ReadOnly:               true,
+	}
+	ldb, err := open(location, opts)
+	if err != nil {
+		return nil, err
+	}
+	return newLeveldbBackend(ldb, location), nil
+}
+
+func open(location string, opts *opt.Options) (*leveldb.DB, error) {
+	return leveldb.OpenFile(location, opts)
+}

internal/db/olddb/keyer.go

@@ -0,0 +1,409 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package olddb
+
+import (
+	"encoding/binary"
+)
+
+const (
+	keyPrefixLen   = 1
+	keyFolderLen   = 4 // indexed
+	keyDeviceLen   = 4 // indexed
+	keySequenceLen = 8
+	keyHashLen     = 32
+
+	maxInt64 int64 = 1<<63 - 1
+)
+
+const (
+	// KeyTypeDevice <int32 folder ID> <int32 device ID> <file name> = FileInfo
+	KeyTypeDevice byte = 0
+
+	// KeyTypeGlobal <int32 folder ID> <file name> = VersionList
+	KeyTypeGlobal byte = 1
+
+	// KeyTypeBlock <int32 folder ID> <32 bytes hash> <§file name> = int32 (block index)
+	KeyTypeBlock byte = 2
+
+	// KeyTypeDeviceStatistic <device ID as string> <some string> = some value
+	KeyTypeDeviceStatistic byte = 3
+
+	// KeyTypeFolderStatistic <folder ID as string> <some string> = some value
+	KeyTypeFolderStatistic byte = 4
+
+	// KeyTypeVirtualMtime <int32 folder ID> <file name> = mtimeMapping
+	KeyTypeVirtualMtime byte = 5
+
+	// KeyTypeFolderIdx <int32 id> = string value
+	KeyTypeFolderIdx byte = 6
+
+	// KeyTypeDeviceIdx <int32 id> = string value
+	KeyTypeDeviceIdx byte = 7
+
+	// KeyTypeIndexID <int32 device ID> <int32 folder ID> = protocol.IndexID
+	KeyTypeIndexID byte = 8
+
+	// KeyTypeFolderMeta <int32 folder ID> = CountsSet
+	KeyTypeFolderMeta byte = 9
+
+	// KeyTypeMiscData <some string> = some value
+	KeyTypeMiscData byte = 10
+
+	// KeyTypeSequence <int32 folder ID> <int64 sequence number> = KeyTypeDevice key
+	KeyTypeSequence byte = 11
+
+	// KeyTypeNeed <int32 folder ID> <file name> = <nothing>
+	KeyTypeNeed byte = 12
+
+	// KeyTypeBlockList <block list hash> = BlockList
+	KeyTypeBlockList byte = 13
+
+	// KeyTypeBlockListMap <int32 folder ID> <block list hash> <file name> = <nothing>
+	KeyTypeBlockListMap byte = 14
+
+	// KeyTypeVersion <version hash> = Vector
+	KeyTypeVersion byte = 15
+
+	// KeyTypePendingFolder <int32 device ID> <folder ID as string> = ObservedFolder
+	KeyTypePendingFolder byte = 16
+
+	// KeyTypePendingDevice <device ID in wire format> = ObservedDevice
+	KeyTypePendingDevice byte = 17
+)
+
+type keyer interface {
+	// device file key stuff
+	GenerateDeviceFileKey(key, folder, device, name []byte) (deviceFileKey, error)
+	NameFromDeviceFileKey(key []byte) []byte
+	DeviceFromDeviceFileKey(key []byte) ([]byte, bool)
+	FolderFromDeviceFileKey(key []byte) ([]byte, bool)
+
+	// global version key stuff
+	GenerateGlobalVersionKey(key, folder, name []byte) (globalVersionKey, error)
+	NameFromGlobalVersionKey(key []byte) []byte
+
+	// block map key stuff (former BlockMap)
+	GenerateBlockMapKey(key, folder, hash, name []byte) (blockMapKey, error)
+	NameFromBlockMapKey(key []byte) []byte
+	GenerateBlockListMapKey(key, folder, hash, name []byte) (blockListMapKey, error)
+	NameFromBlockListMapKey(key []byte) []byte
+
+	// file need index
+	GenerateNeedFileKey(key, folder, name []byte) (needFileKey, error)
+
+	// file sequence index
+	GenerateSequenceKey(key, folder []byte, seq int64) (sequenceKey, error)
+	SequenceFromSequenceKey(key []byte) int64
+
+	// index IDs
+	GenerateIndexIDKey(key, device, folder []byte) (indexIDKey, error)
+	FolderFromIndexIDKey(key []byte) ([]byte, bool)
+	DeviceFromIndexIDKey(key []byte) ([]byte, bool)
+
+	// Mtimes
+	GenerateMtimesKey(key, folder []byte) (mtimesKey, error)
+
+	// Folder metadata
+	GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error)
+
+	// Block lists
+	GenerateBlockListKey(key []byte, hash []byte) blockListKey
+
+	// Version vectors
+	GenerateVersionKey(key []byte, hash []byte) versionKey
+
+	// Pending (unshared) folders and devices
+	GeneratePendingFolderKey(key, device, folder []byte) (pendingFolderKey, error)
+	FolderFromPendingFolderKey(key []byte) []byte
+	DeviceFromPendingFolderKey(key []byte) ([]byte, bool)
+
+	GeneratePendingDeviceKey(key, device []byte) pendingDeviceKey
+	DeviceFromPendingDeviceKey(key []byte) []byte
+}
+
+// defaultKeyer implements our key scheme. It needs folder and device
+// indexes.
+type defaultKeyer struct {
+	folderIdx *smallIndex
+	deviceIdx *smallIndex
+}
+
+func newDefaultKeyer(folderIdx, deviceIdx *smallIndex) defaultKeyer {
+	return defaultKeyer{
+		folderIdx: folderIdx,
+		deviceIdx: deviceIdx,
+	}
+}
+
+type deviceFileKey []byte
+
+func (k deviceFileKey) WithoutNameAndDevice() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k deviceFileKey) WithoutName() []byte {
+	return k[:keyPrefixLen+keyFolderLen+keyDeviceLen]
+}
+
+func (k defaultKeyer) GenerateDeviceFileKey(key, folder, device, name []byte) (deviceFileKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	deviceID, err := k.deviceIdx.ID(device)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen+keyDeviceLen+len(name))
+	key[0] = KeyTypeDevice
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	binary.BigEndian.PutUint32(key[keyPrefixLen+keyFolderLen:], deviceID)
+	copy(key[keyPrefixLen+keyFolderLen+keyDeviceLen:], name)
+	return key, nil
+}
+
+func (defaultKeyer) NameFromDeviceFileKey(key []byte) []byte {
+	return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
+}
+
+func (k defaultKeyer) DeviceFromDeviceFileKey(key []byte) ([]byte, bool) {
+	return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
+}
+
+func (k defaultKeyer) FolderFromDeviceFileKey(key []byte) ([]byte, bool) {
+	return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
+}
+
+type globalVersionKey []byte
+
+func (k globalVersionKey) WithoutName() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k defaultKeyer) GenerateGlobalVersionKey(key, folder, name []byte) (globalVersionKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen+len(name))
+	key[0] = KeyTypeGlobal
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	copy(key[keyPrefixLen+keyFolderLen:], name)
+	return key, nil
+}
+
+func (defaultKeyer) NameFromGlobalVersionKey(key []byte) []byte {
+	return key[keyPrefixLen+keyFolderLen:]
+}
+
+type blockMapKey []byte
+
+func (k defaultKeyer) GenerateBlockMapKey(key, folder, hash, name []byte) (blockMapKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen+keyHashLen+len(name))
+	key[0] = KeyTypeBlock
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	copy(key[keyPrefixLen+keyFolderLen:], hash)
+	copy(key[keyPrefixLen+keyFolderLen+keyHashLen:], name)
+	return key, nil
+}
+
+func (defaultKeyer) NameFromBlockMapKey(key []byte) []byte {
+	return key[keyPrefixLen+keyFolderLen+keyHashLen:]
+}
+
+func (k blockMapKey) WithoutHashAndName() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+type blockListMapKey []byte
+
+func (k defaultKeyer) GenerateBlockListMapKey(key, folder, hash, name []byte) (blockListMapKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen+keyHashLen+len(name))
+	key[0] = KeyTypeBlockListMap
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	copy(key[keyPrefixLen+keyFolderLen:], hash)
+	copy(key[keyPrefixLen+keyFolderLen+keyHashLen:], name)
+	return key, nil
+}
+
+func (defaultKeyer) NameFromBlockListMapKey(key []byte) []byte {
+	return key[keyPrefixLen+keyFolderLen+keyHashLen:]
+}
+
+func (k blockListMapKey) WithoutHashAndName() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+type needFileKey []byte
+
+func (k needFileKey) WithoutName() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k defaultKeyer) GenerateNeedFileKey(key, folder, name []byte) (needFileKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen+len(name))
+	key[0] = KeyTypeNeed
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	copy(key[keyPrefixLen+keyFolderLen:], name)
+	return key, nil
+}
+
+type sequenceKey []byte
+
+func (k sequenceKey) WithoutSequence() []byte {
+	return k[:keyPrefixLen+keyFolderLen]
+}
+
+func (k defaultKeyer) GenerateSequenceKey(key, folder []byte, seq int64) (sequenceKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen+keySequenceLen)
+	key[0] = KeyTypeSequence
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	binary.BigEndian.PutUint64(key[keyPrefixLen+keyFolderLen:], uint64(seq))
+	return key, nil
+}
+
+func (defaultKeyer) SequenceFromSequenceKey(key []byte) int64 {
+	return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
+}
+
+type indexIDKey []byte
+
+func (k defaultKeyer) GenerateIndexIDKey(key, device, folder []byte) (indexIDKey, error) {
+	deviceID, err := k.deviceIdx.ID(device)
+	if err != nil {
+		return nil, err
+	}
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyDeviceLen+keyFolderLen)
+	key[0] = KeyTypeIndexID
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], deviceID)
+	binary.BigEndian.PutUint32(key[keyPrefixLen+keyDeviceLen:], folderID)
+	return key, nil
+}
+
+func (k defaultKeyer) FolderFromIndexIDKey(key []byte) ([]byte, bool) {
+	return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyDeviceLen:]))
+}
+
+func (k defaultKeyer) DeviceFromIndexIDKey(key []byte) ([]byte, bool) {
+	return k.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen : keyPrefixLen+keyDeviceLen]))
+}
+
+type mtimesKey []byte
+
+func (k defaultKeyer) GenerateMtimesKey(key, folder []byte) (mtimesKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen)
+	key[0] = KeyTypeVirtualMtime
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	return key, nil
+}
+
+type folderMetaKey []byte
+
+func (k defaultKeyer) GenerateFolderMetaKey(key, folder []byte) (folderMetaKey, error) {
+	folderID, err := k.folderIdx.ID(folder)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyFolderLen)
+	key[0] = KeyTypeFolderMeta
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], folderID)
+	return key, nil
+}
+
+type blockListKey []byte
+
+func (defaultKeyer) GenerateBlockListKey(key []byte, hash []byte) blockListKey {
+	key = resize(key, keyPrefixLen+len(hash))
+	key[0] = KeyTypeBlockList
+	copy(key[keyPrefixLen:], hash)
+	return key
+}
+
+func (k blockListKey) Hash() []byte {
+	return k[keyPrefixLen:]
+}
+
+type versionKey []byte
+
+func (defaultKeyer) GenerateVersionKey(key []byte, hash []byte) versionKey {
+	key = resize(key, keyPrefixLen+len(hash))
+	key[0] = KeyTypeVersion
+	copy(key[keyPrefixLen:], hash)
+	return key
+}
+
+func (k versionKey) Hash() []byte {
+	return k[keyPrefixLen:]
+}
+
+type pendingFolderKey []byte
+
+func (k defaultKeyer) GeneratePendingFolderKey(key, device, folder []byte) (pendingFolderKey, error) {
+	deviceID, err := k.deviceIdx.ID(device)
+	if err != nil {
+		return nil, err
+	}
+	key = resize(key, keyPrefixLen+keyDeviceLen+len(folder))
+	key[0] = KeyTypePendingFolder
+	binary.BigEndian.PutUint32(key[keyPrefixLen:], deviceID)
+	copy(key[keyPrefixLen+keyDeviceLen:], folder)
+	return key, nil
+}
+
+func (defaultKeyer) FolderFromPendingFolderKey(key []byte) []byte {
+	return key[keyPrefixLen+keyDeviceLen:]
+}
+
+func (k defaultKeyer) DeviceFromPendingFolderKey(key []byte) ([]byte, bool) {
+	return k.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
+}
+
+type pendingDeviceKey []byte
+
+func (defaultKeyer) GeneratePendingDeviceKey(key, device []byte) pendingDeviceKey {
+	key = resize(key, keyPrefixLen+len(device))
+	key[0] = KeyTypePendingDevice
+	copy(key[keyPrefixLen:], device)
+	return key
+}
+
+func (defaultKeyer) DeviceFromPendingDeviceKey(key []byte) []byte {
+	return key[keyPrefixLen:]
+}
+
+// resize returns a byte slice of the specified size, reusing bs if possible
+func resize(bs []byte, size int) []byte {
+	if cap(bs) < size {
+		return make([]byte, size)
+	}
+	return bs[:size]
+}

internal/db/olddb/lowlevel.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package olddb
+
+import (
+	"encoding/binary"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/db/olddb/backend"
+)
+
+// deprecatedLowlevel is the lowest level database interface. It has a very simple
+// purpose: hold the actual backend database, and the in-memory state
+// that belong to that database. In the same way that a single on disk
+// database can only be opened once, there should be only one deprecatedLowlevel for
+// any given backend.
+type deprecatedLowlevel struct {
+	backend.Backend
+	folderIdx *smallIndex
+	deviceIdx *smallIndex
+	keyer     keyer
+}
+
+func NewLowlevel(backend backend.Backend) (*deprecatedLowlevel, error) {
+	// Only log restarts in debug mode.
+	db := &deprecatedLowlevel{
+		Backend:   backend,
+		folderIdx: newSmallIndex(backend, []byte{KeyTypeFolderIdx}),
+		deviceIdx: newSmallIndex(backend, []byte{KeyTypeDeviceIdx}),
+	}
+	db.keyer = newDefaultKeyer(db.folderIdx, db.deviceIdx)
+	return db, nil
+}
+
+// ListFolders returns the list of folders currently in the database
+func (db *deprecatedLowlevel) ListFolders() []string {
+	return db.folderIdx.Values()
+}
+
+func (db *deprecatedLowlevel) IterateMtimes(fn func(folder, name string, ondisk, virtual time.Time) error) error {
+	it, err := db.NewPrefixIterator([]byte{KeyTypeVirtualMtime})
+	if err != nil {
+		return err
+	}
+	defer it.Release()
+	for it.Next() {
+		key := it.Key()[1:]
+		folderID, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key))
+		if !ok {
+			continue
+		}
+		name := key[4:]
+		val := it.Value()
+		var ondisk, virtual time.Time
+		if err := ondisk.UnmarshalBinary(val[:len(val)/2]); err != nil {
+			continue
+		}
+		if err := virtual.UnmarshalBinary(val[len(val)/2:]); err != nil {
+			continue
+		}
+		if err := fn(string(folderID), string(name), ondisk, virtual); err != nil {
+			return err
+		}
+	}
+	return it.Error()
+}

internal/db/olddb/set.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Package db provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
+package olddb
+
+import (
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type deprecatedFileSet struct {
+	folder string
+	db     *deprecatedLowlevel
+}
+
+// The Iterator is called with either a protocol.FileInfo or a
+// FileInfoTruncated (depending on the method) and returns true to
+// continue iteration, false to stop.
+type Iterator func(f protocol.FileInfo) bool
+
+func NewFileSet(folder string, db *deprecatedLowlevel) (*deprecatedFileSet, error) {
+	s := &deprecatedFileSet{
+		folder: folder,
+		db:     db,
+	}
+	return s, nil
+}
+
+type Snapshot struct {
+	folder string
+	t      readOnlyTransaction
+}
+
+func (s *deprecatedFileSet) Snapshot() (*Snapshot, error) {
+	t, err := s.db.newReadOnlyTransaction()
+	if err != nil {
+		return nil, err
+	}
+	return &Snapshot{
+		folder: s.folder,
+		t:      t,
+	}, nil
+}
+
+func (s *Snapshot) Release() {
+	s.t.close()
+}
+
+func (s *Snapshot) WithHaveSequence(startSeq int64, fn Iterator) error {
+	return s.t.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn))
+}
+
+func nativeFileIterator(fn Iterator) Iterator {
+	return func(fi protocol.FileInfo) bool {
+		fi.Name = osutil.NativeFilename(fi.Name)
+		return fn(fi)
+	}
+}

internal/db/olddb/smallindex.go

@@ -0,0 +1,109 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package olddb
+
+import (
+	"encoding/binary"
+	"sort"
+
+	"github.com/syncthing/syncthing/internal/db/olddb/backend"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
+// fast lookups in both directions and persists to the database. Don't use for
+// storing more items than fit comfortably in RAM.
+type smallIndex struct {
+	db     backend.Backend
+	prefix []byte
+	id2val map[uint32]string
+	val2id map[string]uint32
+	nextID uint32
+	mut    sync.Mutex
+}
+
+func newSmallIndex(db backend.Backend, prefix []byte) *smallIndex {
+	idx := &smallIndex{
+		db:     db,
+		prefix: prefix,
+		id2val: make(map[uint32]string),
+		val2id: make(map[string]uint32),
+		mut:    sync.NewMutex(),
+	}
+	idx.load()
+	return idx
+}
+
+// load iterates over the prefix space in the database and populates the in
+// memory maps.
+func (i *smallIndex) load() {
+	it, err := i.db.NewPrefixIterator(i.prefix)
+	if err != nil {
+		panic("loading small index: " + err.Error())
+	}
+	defer it.Release()
+	for it.Next() {
+		val := string(it.Value())
+		id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
+		if val != "" {
+			// Empty value means the entry has been deleted.
+			i.id2val[id] = val
+			i.val2id[val] = id
+		}
+		if id >= i.nextID {
+			i.nextID = id + 1
+		}
+	}
+}
+
+// ID returns the index number for the given byte slice, allocating a new one
+// and persisting this to the database if necessary.
+func (i *smallIndex) ID(val []byte) (uint32, error) {
+	i.mut.Lock()
+	// intentionally avoiding defer here as we want this call to be as fast as
+	// possible in the general case (folder ID already exists). The map lookup
+	// with the conversion of []byte to string is compiler optimized to not
+	// copy the []byte, which is why we don't assign it to a temp variable
+	// here.
+	if id, ok := i.val2id[string(val)]; ok {
+		i.mut.Unlock()
+		return id, nil
+	}
+
+	panic("missing ID")
+}
+
+// Val returns the value for the given index number, or (nil, false) if there
+// is no such index number.
+func (i *smallIndex) Val(id uint32) ([]byte, bool) {
+	i.mut.Lock()
+	val, ok := i.id2val[id]
+	i.mut.Unlock()
+	if !ok {
+		return nil, false
+	}
+
+	return []byte(val), true
+}
+
+// Values returns the set of values in the index
+func (i *smallIndex) Values() []string {
+	// In principle this method should return [][]byte because all the other
+	// methods deal in []byte keys. However, in practice, where it's used
+	// wants a []string and it's easier to just create that here rather than
+	// having to convert both here and there...
+
+	i.mut.Lock()
+	vals := make([]string, 0, len(i.val2id))
+	for val := range i.val2id {
+		vals = append(vals, val)
+	}
+	i.mut.Unlock()
+
+	sort.Strings(vals)
+	return vals
+}

internal/db/olddb/transactions.go

@@ -0,0 +1,193 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package olddb
+
+import (
+	"fmt"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/syncthing/syncthing/internal/db/olddb/backend"
+	"github.com/syncthing/syncthing/internal/gen/bep"
+	"github.com/syncthing/syncthing/internal/gen/dbproto"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+// A readOnlyTransaction represents a database snapshot.
+type readOnlyTransaction struct {
+	backend.ReadTransaction
+	keyer keyer
+}
+
+func (db *deprecatedLowlevel) newReadOnlyTransaction() (readOnlyTransaction, error) {
+	tran, err := db.NewReadTransaction()
+	if err != nil {
+		return readOnlyTransaction{}, err
+	}
+	return db.readOnlyTransactionFromBackendTransaction(tran), nil
+}
+
+func (db *deprecatedLowlevel) readOnlyTransactionFromBackendTransaction(tran backend.ReadTransaction) readOnlyTransaction {
+	return readOnlyTransaction{
+		ReadTransaction: tran,
+		keyer:           db.keyer,
+	}
+}
+
+func (t readOnlyTransaction) close() {
+	t.Release()
+}
+
+func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool, error) {
+	f, ok, err := t.getFileTrunc(key, false)
+	if err != nil || !ok {
+		return protocol.FileInfo{}, false, err
+	}
+	return f, true, nil
+}
+
+func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileInfo, bool, error) {
+	bs, err := t.Get(key)
+	if backend.IsNotFound(err) {
+		return protocol.FileInfo{}, false, nil
+	}
+	if err != nil {
+		return protocol.FileInfo{}, false, err
+	}
+	f, err := t.unmarshalTrunc(bs, trunc)
+	if backend.IsNotFound(err) {
+		return protocol.FileInfo{}, false, nil
+	}
+	if err != nil {
+		return protocol.FileInfo{}, false, err
+	}
+	return f, true, nil
+}
+
+func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileInfo, error) {
+	if trunc {
+		var bfi dbproto.FileInfoTruncated
+		err := proto.Unmarshal(bs, &bfi)
+		if err != nil {
+			return protocol.FileInfo{}, err
+		}
+		if err := t.fillTruncated(&bfi); err != nil {
+			return protocol.FileInfo{}, err
+		}
+		return protocol.FileInfoFromDBTruncated(&bfi), nil
+	}
+
+	var bfi bep.FileInfo
+	err := proto.Unmarshal(bs, &bfi)
+	if err != nil {
+		return protocol.FileInfo{}, err
+	}
+	if err := t.fillFileInfo(&bfi); err != nil {
+		return protocol.FileInfo{}, err
+	}
+	return protocol.FileInfoFromDB(&bfi), nil
+}
+
+type blocksIndirectionError struct {
+	err error
+}
+
+func (e *blocksIndirectionError) Error() string {
+	return fmt.Sprintf("filling Blocks: %v", e.err)
+}
+
+func (e *blocksIndirectionError) Unwrap() error {
+	return e.err
+}
+
+// fillFileInfo follows the (possible) indirection of blocks and version
+// vector and fills it out.
+func (t readOnlyTransaction) fillFileInfo(fi *bep.FileInfo) error {
+	var key []byte
+
+	if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
+		// The blocks list is indirected and we need to load it.
+		key = t.keyer.GenerateBlockListKey(key, fi.BlocksHash)
+		bs, err := t.Get(key)
+		if err != nil {
+			return &blocksIndirectionError{err}
+		}
+		var bl dbproto.BlockList
+		if err := proto.Unmarshal(bs, &bl); err != nil {
+			return err
+		}
+		fi.Blocks = bl.Blocks
+	}
+
+	if len(fi.VersionHash) != 0 {
+		key = t.keyer.GenerateVersionKey(key, fi.VersionHash)
+		bs, err := t.Get(key)
+		if err != nil {
+			return fmt.Errorf("filling Version: %w", err)
+		}
+		var v bep.Vector
+		if err := proto.Unmarshal(bs, &v); err != nil {
+			return err
+		}
+		fi.Version = &v
+	}
+
+	return nil
+}
+
+// fillTruncated follows the (possible) indirection of version vector and
+// fills it.
+func (t readOnlyTransaction) fillTruncated(fi *dbproto.FileInfoTruncated) error {
+	var key []byte
+
+	if len(fi.VersionHash) == 0 {
+		return nil
+	}
+
+	key = t.keyer.GenerateVersionKey(key, fi.VersionHash)
+	bs, err := t.Get(key)
+	if err != nil {
+		return err
+	}
+	var v bep.Vector
+	if err := proto.Unmarshal(bs, &v); err != nil {
+		return err
+	}
+	fi.Version = &v
+	return nil
+}
+
+func (t *readOnlyTransaction) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
+	first, err := t.keyer.GenerateSequenceKey(nil, folder, startSeq)
+	if err != nil {
+		return err
+	}
+	last, err := t.keyer.GenerateSequenceKey(nil, folder, maxInt64)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewRangeIterator(first, last)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	for dbi.Next() {
+		f, ok, err := t.getFileByKey(dbi.Value())
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}