felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Handle errors opening db/creating file-set (ref #5907) (#7150)

Browse Source
This commit is contained in:
Simon Frei
2020-12-21 12:59:22 +01:00
committed by GitHub
parent b5de49917c
commit 78bd0341a8
25 changed files with 542 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/db/benchmark_test.go
+20 -21
View File
@@ -11,7 +11,6 @@ import (
"testing"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -44,11 +43,11 @@ func lazyInitBenchFiles() {
}
}
func getBenchFileSet() (*db.Lowlevel, *db.FileSet) {
func getBenchFileSet(b testing.TB) (*db.Lowlevel, *db.FileSet) {
lazyInitBenchFiles()
ldb := db.NewLowlevel(backend.OpenMemory())
benchS := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
ldb := newLowlevelMemory(b)
benchS := newFileSet(b, "test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
replace(benchS, remoteDevice0, files)
replace(benchS, protocol.LocalDeviceID, firstHalf)
@@ -56,12 +55,12 @@ func getBenchFileSet() (*db.Lowlevel, *db.FileSet) {
}
func BenchmarkReplaceAll(b *testing.B) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(b)
defer ldb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(b, "test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
replace(m, protocol.LocalDeviceID, files)
}
@@ -69,7 +68,7 @@ func BenchmarkReplaceAll(b *testing.B) {
}
func BenchmarkUpdateOneChanged(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
changed := make([]protocol.FileInfo, 1)
@@ -89,7 +88,7 @@ func BenchmarkUpdateOneChanged(b *testing.B) {
}
func BenchmarkUpdate100Changed(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -118,7 +117,7 @@ func setup10Remotes(benchS *db.FileSet) {
}
func BenchmarkUpdate100Changed10Remotes(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
setup10Remotes(benchS)
@@ -136,7 +135,7 @@ func BenchmarkUpdate100Changed10Remotes(b *testing.B) {
}
func BenchmarkUpdate100ChangedRemote(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -152,7 +151,7 @@ func BenchmarkUpdate100ChangedRemote(b *testing.B) {
}
func BenchmarkUpdate100ChangedRemote10Remotes(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -168,7 +167,7 @@ func BenchmarkUpdate100ChangedRemote10Remotes(b *testing.B) {
}
func BenchmarkUpdateOneUnchanged(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -180,7 +179,7 @@ func BenchmarkUpdateOneUnchanged(b *testing.B) {
}
func BenchmarkNeedHalf(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -201,9 +200,9 @@ func BenchmarkNeedHalf(b *testing.B) {
}
func BenchmarkNeedHalfRemote(b *testing.B) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(b)
defer ldb.Close()
fset := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
fset := newFileSet(b, "test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
replace(fset, remoteDevice0, firstHalf)
replace(fset, protocol.LocalDeviceID, files)
@@ -225,7 +224,7 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
}
func BenchmarkHave(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -246,7 +245,7 @@ func BenchmarkHave(b *testing.B) {
}
func BenchmarkGlobal(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -267,7 +266,7 @@ func BenchmarkGlobal(b *testing.B) {
}
func BenchmarkNeedHalfTruncated(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -288,7 +287,7 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
}
func BenchmarkHaveTruncated(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -309,7 +308,7 @@ func BenchmarkHaveTruncated(b *testing.B) {
}
func BenchmarkGlobalTruncated(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
b.ResetTimer()
@@ -330,7 +329,7 @@ func BenchmarkGlobalTruncated(b *testing.B) {
}
func BenchmarkNeedCount(b *testing.B) {
ldb, benchS := getBenchFileSet()
ldb, benchS := getBenchFileSet(b)
defer ldb.Close()
benchS.Update(protocol.LocalDeviceID, changed100)
lib/db/blockmap_test.go
+5 -7
View File
@@ -10,7 +10,6 @@ import (
"encoding/binary"
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -36,10 +35,9 @@ func init() {
}
}
func setup() (*Lowlevel, *BlockFinder) {
// Setup
db := NewLowlevel(backend.OpenMemory())
func setup(t testing.TB) (*Lowlevel, *BlockFinder) {
t.Helper()
db := newLowlevelMemory(t)
return db, NewBlockFinder(db)
}
@@ -105,7 +103,7 @@ func discardFromBlockMap(db *Lowlevel, folder []byte, fs []protocol.FileInfo) er
}
func TestBlockMapAddUpdateWipe(t *testing.T) {
db, f := setup()
db, f := setup(t)
defer db.Close()
if !dbEmpty(db) {
@@ -193,7 +191,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
}
func TestBlockFinderLookup(t *testing.T) {
db, f := setup()
db, f := setup(t)
defer db.Close()
folder1 := []byte("folder1")
lib/db/db_test.go
+28 -21
View File
@@ -13,6 +13,7 @@ import (
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -35,17 +36,17 @@ func TestIgnoredFiles(t *testing.T) {
if err != nil {
t.Fatal(err)
}
db := NewLowlevel(ldb)
db := newLowlevel(t, ldb)
defer db.Close()
if err := UpdateSchema(db); err != nil {
t.Fatal(err)
}
fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// The contents of the database are like this:
//
// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
// { // invalid (ignored) file
// Name: "foo",
@@ -164,7 +165,7 @@ func TestUpdate0to3(t *testing.T) {
t.Fatal(err)
}
db := NewLowlevel(ldb)
db := newLowlevel(t, ldb)
defer db.Close()
updater := schemaUpdater{db}
@@ -293,7 +294,7 @@ func TestUpdate0to3(t *testing.T) {
// TestRepairSequence checks that a few hand-crafted messed-up sequence entries get fixed.
func TestRepairSequence(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
folderStr := "test"
@@ -397,7 +398,7 @@ func TestRepairSequence(t *testing.T) {
// Loading the metadata for the first time means a "re"calculation happens,
// along which the sequences get repaired too.
db.gcMut.RLock()
_ = db.loadMetadataTracker(folderStr)
_, err = db.loadMetadataTracker(folderStr)
db.gcMut.RUnlock()
if err != nil {
t.Fatal(err)
@@ -466,7 +467,7 @@ func TestRepairSequence(t *testing.T) {
}
func TestDowngrade(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
// sets the min version etc
if err := UpdateSchema(db); err != nil {
@@ -491,10 +492,10 @@ func TestDowngrade(t *testing.T) {
}
func TestCheckGlobals(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
// Add any file
name := "foo"
@@ -532,14 +533,17 @@ func TestUpdateTo10(t *testing.T) {
if err != nil {
t.Fatal(err)
}
db := NewLowlevel(ldb)
db := newLowlevel(t, ldb)
defer db.Close()
UpdateSchema(db)
folder := "test"
meta := db.getMetaAndCheck(folder)
meta, err := db.getMetaAndCheck(folder)
if err != nil {
t.Fatal(err)
}
empty := Counts{}
@@ -643,9 +647,9 @@ func TestDropDuplicates(t *testing.T) {
func TestGCIndirect(t *testing.T) {
// Verify that the gcIndirect run actually removes block lists.
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
meta := newMetadataTracker(db.keyer)
meta := newMetadataTracker(db.keyer, events.NoopLogger)
// Add three files with different block lists
@@ -731,7 +735,7 @@ func TestGCIndirect(t *testing.T) {
}
func TestUpdateTo14(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
folderStr := "default"
@@ -741,7 +745,10 @@ func TestUpdateTo14(t *testing.T) {
file.BlocksHash = protocol.BlocksHash(file.Blocks)
fileWOBlocks := file
fileWOBlocks.Blocks = nil
meta := db.loadMetadataTracker(folderStr)
meta, err := db.loadMetadataTracker(folderStr)
if err != nil {
t.Fatal(err)
}
// Initally add the correct file the usual way, all good here.
if err := db.updateLocalFiles(folder, []protocol.FileInfo{file}, meta); err != nil {
@@ -800,7 +807,7 @@ func TestFlushRecursion(t *testing.T) {
// Verify that a commit hook can write to the transaction without
// causing another flush and thus recursion.
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
// A commit hook that writes a small piece of data to the transaction.
@@ -838,11 +845,11 @@ func TestFlushRecursion(t *testing.T) {
}
func TestCheckLocalNeed(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
folderStr := "test"
fs := NewFileSet(folderStr, fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
fs := newFileSet(t, folderStr, fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
// Add files such that we are in sync for a and b, and need c and d.
files := []protocol.FileInfo{
@@ -913,13 +920,13 @@ func TestCheckLocalNeed(t *testing.T) {
}
func TestDuplicateNeedCount(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
folder := "test"
testFs := fs.NewFilesystem(fs.FilesystemTypeFake, "")
fs := NewFileSet(folder, testFs, db)
fs := newFileSet(t, folder, testFs, db)
files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}}
fs.Update(protocol.LocalDeviceID, files)
files[0].Version = files[0].Version.Update(remoteDevice0.Short())
@@ -927,7 +934,7 @@ func TestDuplicateNeedCount(t *testing.T) {
db.checkRepair()
fs = NewFileSet(folder, testFs, db)
fs = newFileSet(t, folder, testFs, db)
found := false
for _, c := range fs.meta.counts.Counts {
if bytes.Equal(protocol.LocalDeviceID[:], c.DeviceID) && c.LocalFlags == needFlag {
lib/db/keyer_test.go
+3 -5
View File
@@ -9,8 +9,6 @@ package db
import (
"bytes"
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
)
func TestDeviceKey(t *testing.T) {
@@ -18,7 +16,7 @@ func TestDeviceKey(t *testing.T) {
dev := []byte("device67890123456789012345678901")
name := []byte("name")
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
key, err := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
@@ -50,7 +48,7 @@ func TestGlobalKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
name := []byte("name")
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
key, err := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
@@ -67,7 +65,7 @@ func TestGlobalKey(t *testing.T) {
func TestSequenceKey(t *testing.T) {
fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
defer db.Close()
const seq = 1234567890
lib/db/lowlevel.go
+53 -37
View File
@@ -10,6 +10,7 @@ import (
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"os"
@@ -19,6 +20,7 @@ import (
"github.com/dchest/siphash"
"github.com/greatroar/blobloom"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
@@ -66,9 +68,10 @@ type Lowlevel struct {
indirectGCInterval time.Duration
recheckInterval time.Duration
oneFileSetCreated chan struct{}
evLogger events.Logger
}
func NewLowlevel(backend backend.Backend, opts ...Option) *Lowlevel {
func NewLowlevel(backend backend.Backend, evLogger events.Logger, opts ...Option) (*Lowlevel, error) {
// Only log restarts in debug mode.
spec := util.SpecWithDebugLogger(l)
db := &Lowlevel{
@@ -80,6 +83,7 @@ func NewLowlevel(backend backend.Backend, opts ...Option) *Lowlevel {
indirectGCInterval: indirectGCDefaultInterval,
recheckInterval: recheckDefaultInterval,
oneFileSetCreated: make(chan struct{}),
evLogger: evLogger,
}
for _, opt := range opts {
opt(db)
@@ -89,11 +93,14 @@ func NewLowlevel(backend backend.Backend, opts ...Option) *Lowlevel {
if path := db.needsRepairPath(); path != "" {
if _, err := os.Lstat(path); err == nil {
l.Infoln("Database was marked for repair - this may take a while")
db.checkRepair()
if err := db.checkRepair(); err != nil {
db.handleFailure(err)
return nil, err
}
os.Remove(path)
}
}
return db
return db, nil
}
type Option func(*Lowlevel)
@@ -822,29 +829,22 @@ func (b *bloomFilter) hash(id []byte) uint64 {
}
// checkRepair checks folder metadata and sequences for miscellaneous errors.
func (db *Lowlevel) checkRepair() {
func (db *Lowlevel) checkRepair() error {
for _, folder := range db.ListFolders() {
_ = db.getMetaAndCheck(folder)
if _, err := db.getMetaAndCheck(folder); err != nil {
return err
}
}
return nil
}
func (db *Lowlevel) getMetaAndCheck(folder string) *metadataTracker {
func (db *Lowlevel) getMetaAndCheck(folder string) (*metadataTracker, error) {
db.gcMut.RLock()
defer db.gcMut.RUnlock()
var err error
defer func() {
if err != nil && !backend.IsClosed(err) {
l.Warnf("Fatal error: %v", err)
obfuscateAndPanic(err)
}
}()
var fixed int
fixed, err = db.checkLocalNeed([]byte(folder))
fixed, err := db.checkLocalNeed([]byte(folder))
if err != nil {
err = fmt.Errorf("checking local need: %w", err)
return nil
return nil, fmt.Errorf("checking local need: %w", err)
}
if fixed != 0 {
l.Infof("Repaired %d local need entries for folder %v in database", fixed, folder)
@@ -852,24 +852,22 @@ func (db *Lowlevel) getMetaAndCheck(folder string) *metadataTracker {
meta, err := db.recalcMeta(folder)
if err != nil {
err = fmt.Errorf("recalculating metadata: %w", err)
return nil
return nil, fmt.Errorf("recalculating metadata: %w", err)
}
fixed, err = db.repairSequenceGCLocked(folder, meta)
if err != nil {
err = fmt.Errorf("repairing sequences: %w", err)
return nil
return nil, fmt.Errorf("repairing sequences: %w", err)
}
if fixed != 0 {
l.Infof("Repaired %d sequence entries for folder %v in database", fixed, folder)
}
return meta
return meta, nil
}
func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
meta := newMetadataTracker(db.keyer)
func (db *Lowlevel) loadMetadataTracker(folder string) (*metadataTracker, error) {
meta := newMetadataTracker(db.keyer, db.evLogger)
if err := meta.fromDB(db, []byte(folder)); err != nil {
if err == errMetaInconsistent {
l.Infof("Stored folder metadata for %q is inconsistent; recalculating", folder)
@@ -881,7 +879,9 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
}
curSeq := meta.Sequence(protocol.LocalDeviceID)
if metaOK := db.verifyLocalSequence(curSeq, folder); !metaOK {
if metaOK, err := db.verifyLocalSequence(curSeq, folder); err != nil {
return nil, fmt.Errorf("verifying sequences: %w", err)
} else if !metaOK {
l.Infof("Stored folder metadata for %q is out of date after crash; recalculating", folder)
return db.getMetaAndCheck(folder)
}
@@ -891,13 +891,13 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
return db.getMetaAndCheck(folder)
}
return meta
return meta, nil
}
func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
folder := []byte(folderStr)
meta := newMetadataTracker(db.keyer)
meta := newMetadataTracker(db.keyer, db.evLogger)
if err := db.checkGlobals(folder); err != nil {
return nil, fmt.Errorf("checking globals: %w", err)
}
@@ -951,7 +951,7 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
// Verify the local sequence number from actual sequence entries. Returns
// true if it was all good, or false if a fixup was necessary.
func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool {
func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) (bool, error) {
// Walk the sequence index from the current (supposedly) highest
// sequence number and raise the alarm if we get anything. This recovers
// from the occasion where we have written sequence entries to disk but
@@ -964,20 +964,18 @@ func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool {
t, err := db.newReadOnlyTransaction()
if err != nil {
l.Warnf("Fatal error: %v", err)
obfuscateAndPanic(err)
return false, err
}
ok := true
if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi protocol.FileIntf) bool {
ok = false // we got something, which we should not have
return false
}); err != nil && !backend.IsClosed(err) {
l.Warnf("Fatal error: %v", err)
obfuscateAndPanic(err)
}); err != nil {
return false, err
}
t.close()
return ok
return ok, nil
}
// repairSequenceGCLocked makes sure the sequence numbers in the sequence keys
@@ -1177,6 +1175,17 @@ func (db *Lowlevel) needsRepairPath() string {
return path + needsRepairSuffix
}
func (db *Lowlevel) checkErrorForRepair(err error) {
if errors.Is(err, errEntryFromGlobalMissing) || errors.Is(err, errEmptyGlobal) {
// Inconsistency error, mark db for repair on next start.
if path := db.needsRepairPath(); path != "" {
if fd, err := os.Create(path); err == nil {
fd.Close()
}
}
}
}
// unchanged checks if two files are the same and thus don't need to be updated.
// Local flags or the invalid bit might change without the version
// being bumped.
@@ -1184,8 +1193,15 @@ func unchanged(nf, ef protocol.FileIntf) bool {
return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags()
}
func (db *Lowlevel) handleFailure(err error) {
db.checkErrorForRepair(err)
if shouldReportFailure(err) {
db.evLogger.Log(events.Failure, err)
}
}
var ldbPathRe = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
func obfuscateAndPanic(err error) {
panic(ldbPathRe.ReplaceAllString(err.Error(), "$1 x: "))
func shouldReportFailure(err error) bool {
return !ldbPathRe.MatchString(err.Error())
}
lib/db/meta.go
+11 -3
View File
@@ -9,10 +9,12 @@ package db
import (
"bytes"
"errors"
"fmt"
"math/bits"
"time"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -28,8 +30,9 @@ type countsMap struct {
type metadataTracker struct {
keyer keyer
countsMap
mut sync.RWMutex
dirty bool
mut sync.RWMutex
dirty bool
evLogger events.Logger
}
type metaKey struct {
@@ -39,13 +42,14 @@ type metaKey struct {
const needFlag uint32 = 1 << 31 // Last bit, as early ones are local flags
func newMetadataTracker(keyer keyer) *metadataTracker {
func newMetadataTracker(keyer keyer, evLogger events.Logger) *metadataTracker {
return &metadataTracker{
keyer: keyer,
mut: sync.NewRWMutex(),
countsMap: countsMap{
indexes: make(map[metaKey]int),
},
evLogger: evLogger,
}
}
@@ -296,18 +300,22 @@ func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f
// the created timestamp to zero. Next time we start up the metadata
// will be seen as infinitely old and recalculated from scratch.
if cp.Deleted < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta deleted count for flag 0x%x dropped below zero", flag))
cp.Deleted = 0
m.counts.Created = 0
}
if cp.Files < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta files count for flag 0x%x dropped below zero", flag))
cp.Files = 0
m.counts.Created = 0
}
if cp.Directories < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta directories count for flag 0x%x dropped below zero", flag))
cp.Directories = 0
m.counts.Created = 0
}
if cp.Symlinks < 0 {
m.evLogger.Log(events.Failure, fmt.Sprintf("meta deleted count for flag 0x%x dropped below zero", flag))
cp.Symlinks = 0
m.counts.Created = 0
}
lib/db/meta_test.go
+6 -6
View File
@@ -11,7 +11,7 @@ import (
"sort"
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -52,7 +52,7 @@ func TestEachFlagBit(t *testing.T) {
func TestMetaDevices(t *testing.T) {
d1 := protocol.DeviceID{1}
d2 := protocol.DeviceID{2}
meta := newMetadataTracker(nil)
meta := newMetadataTracker(nil, events.NoopLogger)
meta.addFile(d1, protocol.FileInfo{Sequence: 1})
meta.addFile(d1, protocol.FileInfo{Sequence: 2, LocalFlags: 1})
@@ -85,7 +85,7 @@ func TestMetaDevices(t *testing.T) {
func TestMetaSequences(t *testing.T) {
d1 := protocol.DeviceID{1}
meta := newMetadataTracker(nil)
meta := newMetadataTracker(nil, events.NoopLogger)
meta.addFile(d1, protocol.FileInfo{Sequence: 1})
meta.addFile(d1, protocol.FileInfo{Sequence: 2, RawInvalid: true})
@@ -105,11 +105,11 @@ func TestMetaSequences(t *testing.T) {
}
func TestRecalcMeta(t *testing.T) {
ldb := NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
// Add some files
s1 := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, "fake"), ldb)
s1 := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, "fake"), ldb)
files := []protocol.FileInfo{
{Name: "a", Size: 1000},
{Name: "b", Size: 2000},
@@ -161,7 +161,7 @@ func TestRecalcMeta(t *testing.T) {
}
// Create a new fileset, which will realize the inconsistency and recalculate
s2 := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, "fake"), ldb)
s2 := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, "fake"), ldb)
// Verify local/global size
snap = s2.Snapshot()
lib/db/namespaced_test.go
+4 -6
View File
@@ -9,12 +9,10 @@ package db
import (
"testing"
"time"
"github.com/syncthing/syncthing/lib/db/backend"
)
func TestNamespacedInt(t *testing.T) {
ldb := NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
@@ -62,7 +60,7 @@ func TestNamespacedInt(t *testing.T) {
}
func TestNamespacedTime(t *testing.T) {
ldb := NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
@@ -86,7 +84,7 @@ func TestNamespacedTime(t *testing.T) {
}
func TestNamespacedString(t *testing.T) {
ldb := NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
@@ -109,7 +107,7 @@ func TestNamespacedString(t *testing.T) {
}
func TestNamespacedReset(t *testing.T) {
ldb := NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
n1 := NewNamespacedKV(ldb, "foo")
lib/db/schemaupdater.go
+1 -1
View File
@@ -719,7 +719,7 @@ func (db *schemaUpdater) updateSchemaTo14(_ int) error {
var key, gk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
meta := newMetadataTracker(db.keyer)
meta := newMetadataTracker(db.keyer, db.evLogger)
meta.counts.Created = 0 // Recalculate metadata afterwards
t, err := db.newReadWriteTransaction(meta.CommitHook(folder))
lib/db/set.go
+10 -14
View File
@@ -13,9 +13,7 @@
package db
import (
"errors"
"fmt"
"os"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/fs"
@@ -38,17 +36,22 @@ type FileSet struct {
// continue iteration, false to stop.
type Iterator func(f protocol.FileIntf) bool
func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) (*FileSet, error) {
select {
case <-db.oneFileSetCreated:
default:
close(db.oneFileSetCreated)
}
meta, err := db.loadMetadataTracker(folder)
if err != nil {
db.handleFailure(err)
return nil, err
}
s := &FileSet{
folder: folder,
fs: fs,
db: db,
meta: db.loadMetadataTracker(folder),
meta: meta,
updateMutex: sync.NewMutex(),
}
if id := s.IndexID(protocol.LocalDeviceID); id == 0 {
@@ -59,7 +62,7 @@ func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
fatalError(err, fmt.Sprintf("%s Creating new IndexID", s.folder), s.db)
}
}
return s
return s, nil
}
func (s *FileSet) Drop(device protocol.DeviceID) {
@@ -500,14 +503,7 @@ func nativeFileIterator(fn Iterator) Iterator {
}
func fatalError(err error, opStr string, db *Lowlevel) {
if errors.Is(err, errEntryFromGlobalMissing) || errors.Is(err, errEmptyGlobal) {
// Inconsistency error, mark db for repair on next start.
if path := db.needsRepairPath(); path != "" {
if fd, err := os.Create(path); err == nil {
fd.Close()
}
}
}
db.checkErrorForRepair(err)
l.Warnf("Fatal error: %v: %v", opStr, err)
obfuscateAndPanic(err)
panic(ldbPathRe.ReplaceAllString(err.Error(), "$1 x: "))
}
lib/db/set_test.go
+92 -69
View File
@@ -18,6 +18,7 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -142,10 +143,10 @@ func setBlocksHash(files fileList) {
}
func TestGlobalSet(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
m := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@@ -448,10 +449,10 @@ func TestGlobalSet(t *testing.T) {
}
func TestNeedWithInvalid(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@@ -488,11 +489,11 @@ func TestNeedWithInvalid(t *testing.T) {
}
func TestUpdateToInvalid(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
f := db.NewBlockFinder(ldb)
localHave := fileList{
@@ -545,10 +546,10 @@ func TestUpdateToInvalid(t *testing.T) {
}
func TestInvalidAvailability(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@@ -587,10 +588,10 @@ func TestInvalidAvailability(t *testing.T) {
}
func TestGlobalReset(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
m := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -626,10 +627,10 @@ func TestGlobalReset(t *testing.T) {
}
func TestNeed(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
m := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -667,10 +668,10 @@ func TestNeed(t *testing.T) {
}
func TestSequence(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
m := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -698,10 +699,10 @@ func TestSequence(t *testing.T) {
}
func TestListDropFolder(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s0 := db.NewFileSet("test0", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s0 := newFileSet(t, "test0", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -709,7 +710,7 @@ func TestListDropFolder(t *testing.T) {
}
replace(s0, protocol.LocalDeviceID, local1)
s1 := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s1 := newFileSet(t, "test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local2 := []protocol.FileInfo{
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
@@ -749,10 +750,10 @@ func TestListDropFolder(t *testing.T) {
}
func TestGlobalNeedWithInvalid(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
rem0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
@@ -792,10 +793,10 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
}
func TestLongPath(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
var b bytes.Buffer
for i := 0; i < 100; i++ {
@@ -833,13 +834,13 @@ func BenchmarkUpdateOneFile(b *testing.B) {
if err != nil {
b.Fatal(err)
}
ldb := db.NewLowlevel(be)
ldb := newLowlevel(b, be)
defer func() {
ldb.Close()
os.RemoveAll("testdata/benchmarkupdate.db")
}()
m := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(b, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
replace(m, protocol.LocalDeviceID, local0)
l := local0[4:5]
@@ -852,10 +853,10 @@ func BenchmarkUpdateOneFile(b *testing.B) {
}
func TestIndexID(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
// The Index ID for some random device is zero by default.
id := s.IndexID(remoteDevice0)
@@ -885,9 +886,9 @@ func TestIndexID(t *testing.T) {
}
func TestDropFiles(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
m := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local0 := fileList{
protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@@ -948,10 +949,10 @@ func TestDropFiles(t *testing.T) {
}
func TestIssue4701(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -990,11 +991,11 @@ func TestIssue4701(t *testing.T) {
}
func TestWithHaveSequence(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
// The files must not be in alphabetical order
localHave := fileList{
@@ -1028,11 +1029,11 @@ func TestStressWithHaveSequence(t *testing.T) {
t.Skip("Takes a long time")
}
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
var localHave []protocol.FileInfo
for i := 0; i < 100; i++ {
@@ -1073,11 +1074,11 @@ loop:
}
func TestIssue4925(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{
protocol.FileInfo{Name: "dir"},
@@ -1100,12 +1101,12 @@ func TestIssue4925(t *testing.T) {
}
func TestMoveGlobalBack(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Blocks: genBlocks(1), ModifiedS: 10, Size: 1}}
remote0Have := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}, {ID: remoteDevice0.Short(), Value: 1}}}, Blocks: genBlocks(2), ModifiedS: 0, Size: 2}}
@@ -1169,12 +1170,12 @@ func TestMoveGlobalBack(t *testing.T) {
// needed files.
// https://github.com/syncthing/syncthing/issues/5007
func TestIssue5007(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}}
@@ -1199,12 +1200,12 @@ func TestIssue5007(t *testing.T) {
// TestNeedDeleted checks that a file that doesn't exist locally isn't needed
// when the global file is deleted.
func TestNeedDeleted(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Deleted: true}}
@@ -1237,11 +1238,11 @@ func TestNeedDeleted(t *testing.T) {
}
func TestReceiveOnlyAccounting(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := protocol.DeviceID{1}
remote := protocol.DeviceID{2}
@@ -1342,12 +1343,12 @@ func TestReceiveOnlyAccounting(t *testing.T) {
}
func TestNeedAfterUnignore(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
remID := remoteDevice0.Short()
@@ -1376,9 +1377,9 @@ func TestNeedAfterUnignore(t *testing.T) {
func TestRemoteInvalidNotAccounted(t *testing.T) {
// Remote files with the invalid bit should not count.
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
files := []protocol.FileInfo{
{Name: "a", Size: 1234, Sequence: 42, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}}, // valid, should count
@@ -1396,10 +1397,10 @@ func TestRemoteInvalidNotAccounted(t *testing.T) {
}
func TestNeedWithNewerInvalid(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("default", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "default", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
rem0ID := remoteDevice0.Short()
rem1ID := remoteDevice1.Short()
@@ -1437,11 +1438,11 @@ func TestNeedWithNewerInvalid(t *testing.T) {
}
func TestNeedAfterDeviceRemove(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
file := "foo"
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}}
@@ -1466,9 +1467,9 @@ func TestNeedAfterDeviceRemove(t *testing.T) {
func TestCaseSensitive(t *testing.T) {
// Normal case sensitive lookup should work
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: filepath.FromSlash("D1/f1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -1504,9 +1505,9 @@ func TestSequenceIndex(t *testing.T) {
// Set up a db and a few files that we will manipulate.
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: filepath.FromSlash("banana"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -1598,11 +1599,11 @@ func TestSequenceIndex(t *testing.T) {
}
func TestIgnoreAfterReceiveOnly(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
file := "foo"
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
fs := fileList{{
Name: file,
@@ -1629,11 +1630,11 @@ func TestIgnoreAfterReceiveOnly(t *testing.T) {
// https://github.com/syncthing/syncthing/issues/6650
func TestUpdateWithOneFileTwice(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
file := "foo"
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
fs := fileList{{
Name: file,
@@ -1667,10 +1668,10 @@ func TestUpdateWithOneFileTwice(t *testing.T) {
// https://github.com/syncthing/syncthing/issues/6668
func TestNeedRemoteOnly(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@@ -1685,10 +1686,10 @@ func TestNeedRemoteOnly(t *testing.T) {
// https://github.com/syncthing/syncthing/issues/6784
func TestNeedRemoteAfterReset(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
files := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@@ -1711,10 +1712,10 @@ func TestNeedRemoteAfterReset(t *testing.T) {
// https://github.com/syncthing/syncthing/issues/6850
func TestIgnoreLocalChanged(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
// Add locally changed file
files := fileList{
@@ -1745,10 +1746,10 @@ func TestIgnoreLocalChanged(t *testing.T) {
// an Index (as opposed to an IndexUpdate), and we don't want to loose the index
// ID when that happens.
func TestNoIndexIDResetOnDrop(t *testing.T) {
ldb := db.NewLowlevel(backend.OpenMemory())
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
s.SetIndexID(remoteDevice0, 1)
s.Drop(remoteDevice0)
@@ -1770,8 +1771,8 @@ func TestConcurrentIndexID(t *testing.T) {
max = 10
}
for i := 0; i < max; i++ {
ldb := db.NewLowlevel(backend.OpenMemory())
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
ldb := newLowlevelMemory(t)
s := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), ldb)
go setID(s, 0)
go setID(s, 1)
<-done
@@ -1837,3 +1838,25 @@ func checkNeed(t testing.TB, s *db.FileSet, dev protocol.DeviceID, expected []pr
t.Errorf("Count incorrect (%v): expected %v, got %v", dev, exp, counts)
}
}
func newLowlevel(t testing.TB, backend backend.Backend) *db.Lowlevel {
t.Helper()
ll, err := db.NewLowlevel(backend, events.NoopLogger)
if err != nil {
t.Fatal(err)
}
return ll
}
func newLowlevelMemory(t testing.TB) *db.Lowlevel {
return newLowlevel(t, backend.OpenMemory())
}
func newFileSet(t testing.TB, folder string, fs fs.Filesystem, ll *db.Lowlevel) *db.FileSet {
t.Helper()
fset, err := db.NewFileSet(folder, fs, ll)
if err != nil {
t.Fatal(err)
}
return fset
}
lib/db/smallindex_test.go
+1 -3
View File
@@ -8,12 +8,10 @@ package db
import (
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
)
func TestSmallIndex(t *testing.T) {
db := NewLowlevel(backend.OpenMemory())
db := newLowlevelMemory(t)
idx := newSmallIndex(db, []byte{12, 34})
// ID zero should be unallocated
lib/db/transactions.go
+5 -1
View File
@@ -12,6 +12,7 @@ import (
"fmt"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -25,7 +26,8 @@ var (
// A readOnlyTransaction represents a database snapshot.
type readOnlyTransaction struct {
backend.ReadTransaction
keyer keyer
keyer keyer
evLogger events.Logger
}
func (db *Lowlevel) newReadOnlyTransaction() (readOnlyTransaction, error) {
@@ -36,6 +38,7 @@ func (db *Lowlevel) newReadOnlyTransaction() (readOnlyTransaction, error) {
return readOnlyTransaction{
ReadTransaction: tran,
keyer: db.keyer,
evLogger: db.evLogger,
}, nil
}
@@ -800,6 +803,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
if !haveOldGlobal {
// Shouldn't ever happen, but doesn't hurt to handle.
t.evLogger.Log(events.Failure, "encountered empty global while removing item")
return keyBuf, t.Delete(gk)
}
lib/db/util_test.go
+29 -6
View File
@@ -10,10 +10,11 @@ import (
"encoding/json"
"io"
"os"
// "testing"
"testing"
"github.com/syncthing/syncthing/lib/db/backend"
// "github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
// "github.com/syncthing/syncthing/lib/protocol"
)
@@ -69,6 +70,28 @@ func openJSONS(file string) (backend.Backend, error) {
return db, nil
}
func newLowlevel(t testing.TB, backend backend.Backend) *Lowlevel {
t.Helper()
ll, err := NewLowlevel(backend, events.NoopLogger)
if err != nil {
t.Fatal(err)
}
return ll
}
func newLowlevelMemory(t testing.TB) *Lowlevel {
return newLowlevel(t, backend.OpenMemory())
}
func newFileSet(t testing.TB, folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
t.Helper()
fset, err := NewFileSet(folder, fs, db)
if err != nil {
t.Fatal(err)
}
return fset
}
// The following commented tests were used to generate jsons files to stdout for
// future tests and are kept here for reference (reuse).
@@ -76,7 +99,7 @@ func openJSONS(file string) (backend.Backend, error) {
// local and remote, in the format used in 0.14.48.
// func TestGenerateIgnoredFilesDB(t *testing.T) {
// db := OpenMemory()
// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
// { // invalid (ignored) file
// Name: "foo",
@@ -111,7 +134,7 @@ func openJSONS(file string) (backend.Backend, error) {
// format used in 0.14.45.
// func TestGenerateUpdate0to3DB(t *testing.T) {
// db := OpenMemory()
// fs := NewFileSet(update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs := newFileSet(t, update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// for devID, files := range haveUpdate0to3 {
// fs.Update(devID, files)
// }
@@ -119,14 +142,14 @@ func openJSONS(file string) (backend.Backend, error) {
// }
// func TestGenerateUpdateTo10(t *testing.T) {
// db := NewLowlevel(backend.OpenMemory())
// db := newLowlevelMemory(t)
// defer db.Close()
// if err := UpdateSchema(db); err != nil {
// t.Fatal(err)
// }
// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
// fs := newFileSet(t, "test", fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
// files := []protocol.FileInfo{
// {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Deleted: true, Sequence: 1},