felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

refactor: use modern Protobuf encoder (#9817)

Browse Source 
At a high level, this is what I've done and why:

- I'm moving the protobuf generation for the `protocol`, `discovery` and
`db` packages to the modern alternatives, and using `buf` to generate
because it's nice and simple.
- After trying various approaches on how to integrate the new types with
the existing code, I opted for splitting off our own data model types
from the on-the-wire generated types. This means we can have a
`FileInfo` type with nicer ergonomics and lots of methods, while the
protobuf generated type stays clean and close to the wire protocol. It
does mean copying between the two when required, which certainly adds a
small amount of inefficiency. If we want to walk this back in the future
and use the raw generated type throughout, that's possible, this however
makes the refactor smaller (!) as it doesn't change everything about the
type for everyone at the same time.
- I have simply removed in cold blood a significant number of old
database migrations. These depended on previous generations of generated
messages of various kinds and were annoying to support in the new
fashion. The oldest supported database version now is the one from
Syncthing 1.9.0 from Sep 7, 2020.
- I changed config structs to be regular manually defined structs.

For the sake of discussion, some things I tried that turned out not to
work...

### Embedding / wrapping

Embedding the protobuf generated structs in our existing types as a data
container and keeping our methods and stuff:

```
package protocol

type FileInfo struct {
  *generated.FileInfo
}
```

This generates a lot of problems because the internal shape of the
generated struct is quite different (different names, different types,
more pointers), because initializing it doesn't work like you'd expect
(i.e., you end up with an embedded nil pointer and a panic), and because
the types of child types don't get wrapped. That is, even if we also
have a similar wrapper around a `Vector`, that's not the type you get
when accessing `someFileInfo.Version`, you get the `*generated.Vector`
that doesn't have methods, etc.

### Aliasing

```
package protocol

type FileInfo = generated.FileInfo
```

Doesn't help because you can't attach methods to it, plus all the above.

### Generating the types into the target package like we do now and
attaching methods

This fails because of the different shape of the generated type (as in
the embedding case above) plus the generated struct already has a bunch
of methods that we can't necessarily override properly (like `String()`
and a bunch of getters).

### Methods to functions

I considered just moving all the methods we attach to functions in a
specific package, so that for example

```
package protocol

func (f FileInfo) Equal(other FileInfo) bool
```

would become

```
package fileinfos

func Equal(a, b *generated.FileInfo) bool
```

and this would mostly work, but becomes quite verbose and cumbersome,
and somewhat limits discoverability (you can't see what methods are
available on the type in auto completions, etc). In the end I did this
in some cases, like in the database layer where a lot of things like
`func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv
*generated.FileVersion)` because they were anyway just internal methods.

Fixes #8247
This commit is contained in:
Jakob Borg
2024-12-01 16:50:17 +01:00
committed by GitHub
parent 2b8ee4c7a5
commit 77970d5113
203 changed files with 7437 additions and 28636 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/db/benchmark_test.go
+7 -7
View File
@@ -185,7 +185,7 @@ func BenchmarkNeedHalf(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
@@ -209,7 +209,7 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, fset)
snap.WithNeed(remoteDevice0, func(fi protocol.FileIntf) bool {
snap.WithNeed(remoteDevice0, func(fi protocol.FileInfo) bool {
count++
return true
})
@@ -230,7 +230,7 @@ func BenchmarkHave(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
@@ -251,7 +251,7 @@ func BenchmarkGlobal(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithGlobal(func(fi protocol.FileIntf) bool {
snap.WithGlobal(func(fi protocol.FileInfo) bool {
count++
return true
})
@@ -272,7 +272,7 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
@@ -293,7 +293,7 @@ func BenchmarkHaveTruncated(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
count++
return true
})
@@ -314,7 +314,7 @@ func BenchmarkGlobalTruncated(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := snapshot(b, benchS)
snap.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
snap.WithGlobalTruncated(func(fi protocol.FileInfo) bool {
count++
return true
})
lib/db/db_test.go
+7 -316
View File
@@ -30,107 +30,10 @@ func genBlocks(n int) []protocol.BlockInfo {
return b
}
func TestIgnoredFiles(t *testing.T) {
ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons")
if err != nil {
t.Fatal(err)
}
db := newLowlevel(t, ldb)
defer db.Close()
if err := UpdateSchema(db); err != nil {
t.Fatal(err)
}
fs := newFileSet(t, "test", db)
// The contents of the database are like this:
//
// fs := newFileSet(t, "test", db)
// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
// { // invalid (ignored) file
// Name: "foo",
// Type: protocol.FileInfoTypeFile,
// Invalid: true,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
// },
// { // regular file
// Name: "bar",
// Type: protocol.FileInfoTypeFile,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
// },
// })
// fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
// { // invalid file
// Name: "baz",
// Type: protocol.FileInfoTypeFile,
// Invalid: true,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
// },
// { // regular file
// Name: "quux",
// Type: protocol.FileInfoTypeFile,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
// },
// })
// Local files should have the "ignored" bit in addition to just being
// generally invalid if we want to look at the simulation of that bit.
snap := snapshot(t, fs)
defer snap.Release()
fi, ok := snap.Get(protocol.LocalDeviceID, "foo")
if !ok {
t.Fatal("foo should exist")
}
if !fi.IsInvalid() {
t.Error("foo should be invalid")
}
if !fi.IsIgnored() {
t.Error("foo should be ignored")
}
fi, ok = snap.Get(protocol.LocalDeviceID, "bar")
if !ok {
t.Fatal("bar should exist")
}
if fi.IsInvalid() {
t.Error("bar should not be invalid")
}
if fi.IsIgnored() {
t.Error("bar should not be ignored")
}
// Remote files have the invalid bit as usual, and the IsInvalid() method
// should pick this up too.
fi, ok = snap.Get(protocol.DeviceID{42}, "baz")
if !ok {
t.Fatal("baz should exist")
}
if !fi.IsInvalid() {
t.Error("baz should be invalid")
}
if !fi.IsInvalid() {
t.Error("baz should be invalid")
}
fi, ok = snap.Get(protocol.DeviceID{42}, "quux")
if !ok {
t.Fatal("quux should exist")
}
if fi.IsInvalid() {
t.Error("quux should not be invalid")
}
if fi.IsInvalid() {
t.Error("quux should not be invalid")
}
}
const myID = 1
var (
remoteDevice0, remoteDevice1 protocol.DeviceID
update0to3Folder = "UpdateSchema0to3"
invalid = "invalid"
slashPrefixed = "/notgood"
haveUpdate0to3 map[protocol.DeviceID][]protocol.FileInfo
@@ -157,142 +60,6 @@ func init() {
}
}
func TestUpdate0to3(t *testing.T) {
ldb, err := openJSONS("testdata/v0.14.45-update0to3.db.jsons")
if err != nil {
t.Fatal(err)
}
db := newLowlevel(t, ldb)
defer db.Close()
updater := schemaUpdater{db}
folder := []byte(update0to3Folder)
if err := updater.updateSchema0to1(0); err != nil {
t.Fatal(err)
}
trans, err := db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.Release()
if _, ok, err := trans.getFile(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil {
t.Fatal(err)
} else if ok {
t.Error("File prefixed by '/' was not removed during transition to schema 1")
}
var key []byte
key, err = db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid))
if err != nil {
t.Fatal(err)
}
if _, err := db.Get(key); err != nil {
t.Error("Invalid file wasn't added to global list")
}
if err := updater.updateSchema1to2(1); err != nil {
t.Fatal(err)
}
found := false
trans, err = db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.Release()
_ = trans.withHaveSequence(folder, 0, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
l.Infoln(f)
if found {
t.Error("Unexpected additional file via sequence", f.FileName())
return true
}
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, protocol.FileInfoComparison{IgnorePerms: true, IgnoreBlocks: true}) {
found = true
} else {
t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
}
return true
})
if !found {
t.Error("Local file wasn't added to sequence bucket", err)
}
if err := updater.updateSchema2to3(2); err != nil {
t.Fatal(err)
}
need := map[string]protocol.FileInfo{
haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0],
haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
}
trans, err = db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.Release()
key, err = trans.keyer.GenerateNeedFileKey(nil, folder, nil)
if err != nil {
t.Fatal(err)
}
dbi, err := trans.NewPrefixIterator(key)
if err != nil {
t.Fatal(err)
}
defer dbi.Release()
for dbi.Next() {
name := trans.keyer.NameFromGlobalVersionKey(dbi.Key())
key, err = trans.keyer.GenerateGlobalVersionKey(key, folder, name)
bs, err := trans.Get(key)
if err != nil {
t.Fatal(err)
}
var vl VersionListDeprecated
if err := vl.Unmarshal(bs); err != nil {
t.Fatal(err)
}
key, err = trans.keyer.GenerateDeviceFileKey(key, folder, vl.Versions[0].Device, name)
if err != nil {
t.Fatal(err)
}
fi, ok, err := trans.getFileTrunc(key, false)
if err != nil {
t.Fatal(err)
}
if !ok {
device := "<invalid>"
if dev, err := protocol.DeviceIDFromBytes(vl.Versions[0].Device); err != nil {
device = dev.String()
}
t.Fatal("surprise missing global file", string(name), device)
}
e, ok := need[fi.FileName()]
if !ok {
t.Error("Got unexpected needed file:", fi.FileName())
}
f := fi.(protocol.FileInfo)
delete(need, f.Name)
if !f.IsEquivalentOptional(e, protocol.FileInfoComparison{IgnorePerms: true, IgnoreBlocks: true}) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e)
}
}
if dbi.Error() != nil {
t.Fatal(err)
}
for n := range need {
t.Errorf(`Missing needed file "%v"`, n)
}
}
// TestRepairSequence checks that a few hand-crafted messed-up sequence entries get fixed.
func TestRepairSequence(t *testing.T) {
db := newLowlevelMemory(t)
@@ -446,11 +213,10 @@ func TestRepairSequence(t *testing.T) {
}
defer it.Release()
for it.Next() {
intf, ok, err := ro.getFileTrunc(it.Value(), false)
fi, ok, err := ro.getFileTrunc(it.Value(), false)
if err != nil {
t.Fatal(err)
}
fi := intf.(protocol.FileInfo)
seq := ro.keyer.SequenceFromSequenceKey(it.Key())
if !ok {
t.Errorf("Sequence entry %v points at nothing", seq)
@@ -531,81 +297,6 @@ func TestCheckGlobals(t *testing.T) {
}
}
func TestUpdateTo10(t *testing.T) {
ldb, err := openJSONS("./testdata/v1.4.0-updateTo10.json")
if err != nil {
t.Fatal(err)
}
db := newLowlevel(t, ldb)
defer db.Close()
UpdateSchema(db)
folder := "test"
meta, err := db.getMetaAndCheck(folder)
if err != nil {
t.Fatal(err)
}
empty := Counts{}
c := meta.Counts(protocol.LocalDeviceID, needFlag)
if c.Files != 1 {
t.Error("Expected 1 needed file locally, got", c.Files)
}
c.Files = 0
if c.Deleted != 1 {
t.Error("Expected 1 needed deletion locally, got", c.Deleted)
}
c.Deleted = 0
if !c.Equal(empty) {
t.Error("Expected all counts to be zero, got", c)
}
c = meta.Counts(remoteDevice0, needFlag)
if !c.Equal(empty) {
t.Error("Expected all counts to be zero, got", c)
}
trans, err := db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.Release()
// a
vl, err := trans.getGlobalVersions(nil, []byte(folder), []byte("a"))
if err != nil {
t.Fatal(err)
}
for _, v := range vl.RawVersions {
if !v.Deleted {
t.Error("Unexpected undeleted global version for a")
}
}
// b
vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("b"))
if err != nil {
t.Fatal(err)
}
if !vl.RawVersions[0].Deleted {
t.Error("vl.Versions[0] not deleted for b")
}
if vl.RawVersions[1].Deleted {
t.Error("vl.Versions[1] deleted for b")
}
// c
vl, err = trans.getGlobalVersions(nil, []byte(folder), []byte("c"))
if err != nil {
t.Fatal(err)
}
if vl.RawVersions[0].Deleted {
t.Error("vl.Versions[0] deleted for c")
}
if !vl.RawVersions[1].Deleted {
t.Error("vl.Versions[1] not deleted for c")
}
}
func TestDropDuplicates(t *testing.T) {
names := []string{
"foo",
@@ -769,7 +460,7 @@ func TestUpdateTo14(t *testing.T) {
if err != nil {
t.Fatal(err)
}
fiBs := mustMarshal(&fileWOBlocks)
fiBs := mustMarshal(fileWOBlocks.ToWire(true))
if err := trans.Put(key, fiBs); err != nil {
t.Fatal(err)
}
@@ -799,9 +490,9 @@ func TestUpdateTo14(t *testing.T) {
if vl, err := ro.getGlobalVersions(nil, folder, name); err != nil {
t.Fatal(err)
} else if fv, ok := vl.GetGlobal(); !ok {
} else if fv, ok := vlGetGlobal(vl); !ok {
t.Error("missing global")
} else if !fv.IsInvalid() {
} else if !fvIsInvalid(fv) {
t.Error("global not marked as invalid")
}
}
@@ -874,8 +565,8 @@ func TestCheckLocalNeed(t *testing.T) {
t.Errorf("Expected 2 needed files locally, got %v in meta", c.Files)
}
needed := make([]protocol.FileInfo, 0, 2)
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
needed = append(needed, fi.(protocol.FileInfo))
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
needed = append(needed, fi)
return true
})
if l := len(needed); l != 2 {
@@ -939,7 +630,7 @@ func TestDuplicateNeedCount(t *testing.T) {
fs = newFileSet(t, folder, db)
found := false
for _, c := range fs.meta.counts.Counts {
if bytes.Equal(protocol.LocalDeviceID[:], c.DeviceID) && c.LocalFlags == needFlag {
if protocol.LocalDeviceID == c.DeviceID && c.LocalFlags == needFlag {
if found {
t.Fatal("second need count for local device encountered")
}
lib/db/lowlevel.go
+36 -32
View File
@@ -19,6 +19,10 @@ import (
"time"
"github.com/greatroar/blobloom"
"github.com/thejerf/suture/v4"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
@@ -26,7 +30,6 @@ import (
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture/v4"
)
const (
@@ -521,8 +524,8 @@ func (db *Lowlevel) checkGlobals(folderStr string) (int, error) {
var dk []byte
ro := t.readOnlyTransaction
for dbi.Next() {
var vl VersionList
if err := vl.Unmarshal(dbi.Value()); err != nil || vl.Empty() {
var vl dbproto.VersionList
if err := proto.Unmarshal(dbi.Value(), &vl); err != nil || len(vl.Versions) == 0 {
if err := t.Delete(dbi.Key()); err != nil && !backend.IsNotFound(err) {
return 0, err
}
@@ -535,9 +538,9 @@ func (db *Lowlevel) checkGlobals(folderStr string) (int, error) {
// we find those and clear them out.
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
newVL := &VersionList{}
newVL := &dbproto.VersionList{}
var changed, changedHere bool
for _, fv := range vl.RawVersions {
for _, fv := range vl.Versions {
changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.Devices, newVL, ro)
if err != nil {
return 0, err
@@ -551,7 +554,7 @@ func (db *Lowlevel) checkGlobals(folderStr string) (int, error) {
changed = changed || changedHere
}
if newVL.Empty() {
if len(newVL.Versions) == 0 {
if err := t.Delete(dbi.Key()); err != nil && !backend.IsNotFound(err) {
return 0, err
}
@@ -572,7 +575,7 @@ func (db *Lowlevel) checkGlobals(folderStr string) (int, error) {
return fixed, t.Commit()
}
func checkGlobalsFilterDevices(dk, folder, name []byte, devices [][]byte, vl *VersionList, t readOnlyTransaction) (bool, error) {
func checkGlobalsFilterDevices(dk, folder, name []byte, devices [][]byte, vl *dbproto.VersionList, t readOnlyTransaction) (bool, error) {
var changed bool
var err error
for _, device := range devices {
@@ -588,7 +591,7 @@ func checkGlobalsFilterDevices(dk, folder, name []byte, devices [][]byte, vl *Ve
changed = true
continue
}
_, _, _, _, _, _, err = vl.update(folder, device, f, t)
_, _, _, _, _, _, err = vlUpdate(vl, folder, device, f, t)
if err != nil {
return false, err
}
@@ -610,7 +613,7 @@ func (db *Lowlevel) getIndexID(device, folder []byte) (protocol.IndexID, error)
var id protocol.IndexID
if err := id.Unmarshal(cur); err != nil {
return 0, nil
return 0, nil //nolint: nilerr
}
return id, nil
@@ -816,11 +819,11 @@ func (db *Lowlevel) gcIndirect(ctx context.Context) (err error) {
default:
}
var hashes IndirectionHashesOnly
if err := hashes.Unmarshal(it.Value()); err != nil {
var hashes dbproto.IndirectionHashesOnly
if err := proto.Unmarshal(it.Value(), &hashes); err != nil {
return err
}
db.recordIndirectionHashes(hashes)
db.recordIndirectionHashes(&hashes)
}
it.Release()
if err := it.Error(); err != nil {
@@ -924,10 +927,10 @@ func (db *Lowlevel) gcIndirect(ctx context.Context) (err error) {
}
func (db *Lowlevel) recordIndirectionHashesForFile(f *protocol.FileInfo) {
db.recordIndirectionHashes(IndirectionHashesOnly{BlocksHash: f.BlocksHash, VersionHash: f.VersionHash})
db.recordIndirectionHashes(&dbproto.IndirectionHashesOnly{BlocksHash: f.BlocksHash, VersionHash: f.VersionHash})
}
func (db *Lowlevel) recordIndirectionHashes(hs IndirectionHashesOnly) {
func (db *Lowlevel) recordIndirectionHashes(hs *dbproto.IndirectionHashesOnly) {
// must be called with gcMut held (at least read-held)
if db.blockFilter != nil && len(hs.BlocksHash) > 0 {
db.blockFilter.add(hs.BlocksHash)
@@ -966,7 +969,7 @@ func (b *bloomFilter) hash(id []byte) uint64 {
}
var h maphash.Hash
h.SetSeed(b.seed)
h.Write(id)
_, _ = h.Write(id)
return h.Sum64()
}
@@ -1035,7 +1038,7 @@ func (db *Lowlevel) getMetaAndCheckGCLocked(folder string) (*metadataTracker, er
func (db *Lowlevel) loadMetadataTracker(folder string) (*metadataTracker, error) {
meta := newMetadataTracker(db.keyer, db.evLogger)
if err := meta.fromDB(db, []byte(folder)); err != nil {
if err == errMetaInconsistent {
if errors.Is(err, errMetaInconsistent) {
l.Infof("Stored folder metadata for %q is inconsistent; recalculating", folder)
} else {
l.Infof("No stored folder metadata for %q; recalculating", folder)
@@ -1071,7 +1074,7 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
defer t.close()
var deviceID protocol.DeviceID
err = t.withAllFolderTruncated(folder, func(device []byte, f FileInfoTruncated) bool {
err = t.withAllFolderTruncated(folder, func(device []byte, f protocol.FileInfo) bool {
copy(deviceID[:], device)
meta.addFile(deviceID, f)
return true
@@ -1080,7 +1083,7 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
return nil, err
}
err = t.withGlobal(folder, nil, true, func(f protocol.FileIntf) bool {
err = t.withGlobal(folder, nil, true, func(f protocol.FileInfo) bool {
meta.addFile(protocol.GlobalDeviceID, f)
return true
})
@@ -1089,7 +1092,7 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
}
meta.emptyNeeded(protocol.LocalDeviceID)
err = t.withNeed(folder, protocol.LocalDeviceID[:], true, func(f protocol.FileIntf) bool {
err = t.withNeed(folder, protocol.LocalDeviceID[:], true, func(f protocol.FileInfo) bool {
meta.addNeeded(protocol.LocalDeviceID, f)
return true
})
@@ -1098,7 +1101,7 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
}
for _, device := range meta.devices() {
meta.emptyNeeded(device)
err = t.withNeed(folder, device[:], true, func(f protocol.FileIntf) bool {
err = t.withNeed(folder, device[:], true, func(f protocol.FileInfo) bool {
meta.addNeeded(device, f)
return true
})
@@ -1132,7 +1135,7 @@ func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) (bool, erro
return false, err
}
ok := true
if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi protocol.FileIntf) bool {
if err := t.withHaveSequence([]byte(folder), curSeq+1, func(_ protocol.FileInfo) bool {
ok = false // we got something, which we should not have
return false
}); err != nil {
@@ -1204,8 +1207,7 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack
}
return 0, err
}
fi := intf.(protocol.FileInfo)
if sk, err = t.keyer.GenerateSequenceKey(sk, folder, fi.Sequence); err != nil {
if sk, err = t.keyer.GenerateSequenceKey(sk, folder, intf.Sequence); err != nil {
return 0, err
}
switch dk, err = t.Get(sk); {
@@ -1216,14 +1218,14 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack
fallthrough
case !bytes.Equal(it.Key(), dk):
fixed++
fi.Sequence = meta.nextLocalSeq()
if sk, err = t.keyer.GenerateSequenceKey(sk, folder, fi.Sequence); err != nil {
intf.Sequence = meta.nextLocalSeq()
if sk, err = t.keyer.GenerateSequenceKey(sk, folder, intf.Sequence); err != nil {
return 0, err
}
if err := t.Put(sk, it.Key()); err != nil {
return 0, err
}
if err := t.putFile(it.Key(), fi); err != nil {
if err := t.putFile(it.Key(), intf); err != nil {
return 0, err
}
}
@@ -1307,9 +1309,8 @@ func (db *Lowlevel) checkLocalNeed(folder []byte) (int, error) {
}
}
next()
t.withNeedIteratingGlobal(folder, protocol.LocalDeviceID[:], true, func(fi protocol.FileIntf) bool {
f := fi.(FileInfoTruncated)
for !needDone && needName < f.Name {
itErr := t.withNeedIteratingGlobal(folder, protocol.LocalDeviceID[:], true, func(fi protocol.FileInfo) bool {
for !needDone && needName < fi.Name {
repaired++
if err = t.Delete(dbi.Key()); err != nil && !backend.IsNotFound(err) {
return false
@@ -1317,24 +1318,27 @@ func (db *Lowlevel) checkLocalNeed(folder []byte) (int, error) {
l.Debugln("check local need: removing", needName)
next()
}
if needName == f.Name {
if needName == fi.Name {
next()
} else {
repaired++
key, err = t.keyer.GenerateNeedFileKey(key, folder, []byte(f.Name))
key, err = t.keyer.GenerateNeedFileKey(key, folder, []byte(fi.Name))
if err != nil {
return false
}
if err = t.Put(key, nil); err != nil {
return false
}
l.Debugln("check local need: adding", f.Name)
l.Debugln("check local need: adding", fi.Name)
}
return true
})
if err != nil {
return 0, err
}
if itErr != nil {
return 0, itErr
}
for !needDone {
repaired++
lib/db/meta.go
+36 -28
View File
@@ -7,12 +7,14 @@
package db
import (
"bytes"
"errors"
"fmt"
"math/bits"
"time"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
@@ -56,24 +58,34 @@ func newMetadataTracker(keyer keyer, evLogger events.Logger) *metadataTracker {
// Unmarshal loads a metadataTracker from the corresponding protobuf
// representation
func (m *metadataTracker) Unmarshal(bs []byte) error {
if err := m.counts.Unmarshal(bs); err != nil {
var dbc dbproto.CountsSet
if err := proto.Unmarshal(bs, &dbc); err != nil {
return err
}
m.counts.Created = dbc.Created
m.counts.Counts = make([]Counts, len(dbc.Counts))
for i, c := range dbc.Counts {
m.counts.Counts[i] = countsFromWire(c)
}
// Initialize the index map
m.indexes = make(map[metaKey]int)
for i, c := range m.counts.Counts {
dev, err := protocol.DeviceIDFromBytes(c.DeviceID)
if err != nil {
return err
}
m.indexes[metaKey{dev, c.LocalFlags}] = i
m.indexes[metaKey{c.DeviceID, c.LocalFlags}] = i
}
return nil
}
// Marshal returns the protobuf representation of the metadataTracker
func (m *metadataTracker) Marshal() ([]byte, error) {
return m.counts.Marshal()
dbc := &dbproto.CountsSet{
Counts: make([]*dbproto.Counts, len(m.counts.Counts)),
Created: m.Created().UnixNano(),
}
for i, c := range m.counts.Counts {
dbc.Counts[i] = c.toWire()
}
return proto.Marshal(dbc)
}
func (m *metadataTracker) CommitHook(folder []byte) backend.CommitHook {
@@ -142,7 +154,7 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flag uint32) *Counts
idx, ok := m.indexes[key]
if !ok {
idx = len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flag})
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev, LocalFlags: flag})
m.indexes[key] = idx
// Need bucket must be initialized when a device first occurs in
// the metadatatracker, even if there's no change to the need
@@ -160,19 +172,19 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flag uint32) *Counts
// allNeeded makes sure there is a counts in case the device needs everything.
func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts {
counts := Counts{}
var counts Counts
if idx, ok := m.indexes[metaKey{protocol.GlobalDeviceID, 0}]; ok {
counts = m.counts.Counts[idx]
counts.Deleted = 0 // Don't need deletes if having nothing
}
counts.DeviceID = dev[:]
counts.DeviceID = dev
counts.LocalFlags = needFlag
return counts
}
// addFile adds a file to the counts, adjusting the sequence number as
// appropriate
func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileIntf) {
func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
@@ -181,7 +193,7 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileIntf) {
m.updateFileLocked(dev, f, m.addFileLocked)
}
func (m *metadataTracker) updateFileLocked(dev protocol.DeviceID, f protocol.FileIntf, fn func(protocol.DeviceID, uint32, protocol.FileIntf)) {
func (m *metadataTracker) updateFileLocked(dev protocol.DeviceID, f protocol.FileInfo, fn func(protocol.DeviceID, uint32, protocol.FileInfo)) {
m.dirty = true
if f.IsInvalid() && (f.FileLocalFlags() == 0 || dev == protocol.GlobalDeviceID) {
@@ -209,7 +221,7 @@ func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) {
m.dirty = true
empty := Counts{
DeviceID: dev[:],
DeviceID: dev,
LocalFlags: needFlag,
}
key := metaKey{dev, needFlag}
@@ -222,7 +234,7 @@ func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) {
}
// addNeeded adds a file to the needed counts
func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f protocol.FileIntf) {
func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
@@ -237,7 +249,7 @@ func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 {
return m.countsPtr(dev, 0).Sequence
}
func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.FileIntf) {
func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.FileInfo) {
if dev == protocol.GlobalDeviceID {
return
}
@@ -246,7 +258,7 @@ func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.File
}
}
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) {
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileInfo) {
cp := m.countsPtr(dev, flag)
switch {
@@ -263,7 +275,7 @@ func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f pr
}
// removeFile removes a file from the counts
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileIntf) {
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
@@ -271,7 +283,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileIntf)
}
// removeNeeded removes a file from the needed counts
func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileIntf) {
func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileInfo) {
m.mut.Lock()
defer m.mut.Unlock()
@@ -280,7 +292,7 @@ func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileInt
m.removeFileLocked(dev, needFlag, f)
}
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) {
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileInfo) {
cp := m.countsPtr(dev, flag)
switch {
@@ -325,7 +337,7 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
m.mut.Lock()
m.dirty = true
for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
if c.DeviceID == dev {
if c.LocalFlags != needFlag {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
@@ -346,7 +358,7 @@ func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
m.dirty = true
for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
if c.DeviceID == dev {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
Sequence: c.Sequence,
@@ -419,14 +431,10 @@ func (m *countsMap) devices() []protocol.DeviceID {
for _, dev := range m.counts.Counts {
if dev.Sequence > 0 {
id, err := protocol.DeviceIDFromBytes(dev.DeviceID)
if err != nil {
panic(err)
}
if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID {
if dev.DeviceID == protocol.GlobalDeviceID || dev.DeviceID == protocol.LocalDeviceID {
continue
}
devs = append(devs, id)
devs = append(devs, dev.DeviceID)
}
}
lib/db/observed.go
+49 -14
View File
@@ -10,21 +10,56 @@ import (
"fmt"
"time"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/protocol"
)
type ObservedFolder struct {
Time time.Time
Label string
ReceiveEncrypted bool
RemoteEncrypted bool
}
func (o *ObservedFolder) toWire() *dbproto.ObservedFolder {
return &dbproto.ObservedFolder{
Time: timestamppb.New(o.Time),
Label: o.Label,
ReceiveEncrypted: o.ReceiveEncrypted,
RemoteEncrypted: o.RemoteEncrypted,
}
}
func (o *ObservedFolder) fromWire(w *dbproto.ObservedFolder) {
o.Time = w.GetTime().AsTime()
o.Label = w.GetLabel()
o.ReceiveEncrypted = w.GetReceiveEncrypted()
o.RemoteEncrypted = w.GetRemoteEncrypted()
}
type ObservedDevice struct {
Time time.Time
Name string
Address string
}
func (o *ObservedDevice) fromWire(w *dbproto.ObservedDevice) {
o.Time = w.GetTime().AsTime()
o.Name = w.GetName()
o.Address = w.GetAddress()
}
func (db *Lowlevel) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) error {
key := db.keyer.GeneratePendingDeviceKey(nil, device[:])
od := ObservedDevice{
Time: time.Now().Truncate(time.Second),
od := &dbproto.ObservedDevice{
Time: timestamppb.New(time.Now().Truncate(time.Second)),
Name: name,
Address: address,
}
bs, err := od.Marshal()
if err != nil {
return err
}
return db.Put(key, bs)
return db.Put(key, mustMarshal(od))
}
func (db *Lowlevel) RemovePendingDevice(device protocol.DeviceID) error {
@@ -44,13 +79,15 @@ func (db *Lowlevel) PendingDevices() (map[protocol.DeviceID]ObservedDevice, erro
for iter.Next() {
keyDev := db.keyer.DeviceFromPendingDeviceKey(iter.Key())
deviceID, err := protocol.DeviceIDFromBytes(keyDev)
var protoD dbproto.ObservedDevice
var od ObservedDevice
if err != nil {
goto deleteKey
}
if err = od.Unmarshal(iter.Value()); err != nil {
if err = proto.Unmarshal(iter.Value(), &protoD); err != nil {
goto deleteKey
}
od.fromWire(&protoD)
res[deviceID] = od
continue
deleteKey:
@@ -70,11 +107,7 @@ func (db *Lowlevel) AddOrUpdatePendingFolder(id string, of ObservedFolder, devic
if err != nil {
return err
}
bs, err := of.Marshal()
if err != nil {
return err
}
return db.Put(key, bs)
return db.Put(key, mustMarshal(of.toWire()))
}
// RemovePendingFolderForDevice removes entries for specific folder / device combinations.
@@ -139,6 +172,7 @@ func (db *Lowlevel) PendingFoldersForDevice(device protocol.DeviceID) (map[strin
for iter.Next() {
keyDev, ok := db.keyer.DeviceFromPendingFolderKey(iter.Key())
deviceID, err := protocol.DeviceIDFromBytes(keyDev)
var protoF dbproto.ObservedFolder
var of ObservedFolder
var folderID string
if !ok || err != nil {
@@ -147,7 +181,7 @@ func (db *Lowlevel) PendingFoldersForDevice(device protocol.DeviceID) (map[strin
if folderID = string(db.keyer.FolderFromPendingFolderKey(iter.Key())); len(folderID) < 1 {
goto deleteKey
}
if err = of.Unmarshal(iter.Value()); err != nil {
if err = proto.Unmarshal(iter.Value(), &protoF); err != nil {
goto deleteKey
}
if _, ok := res[folderID]; !ok {
@@ -155,6 +189,7 @@ func (db *Lowlevel) PendingFoldersForDevice(device protocol.DeviceID) (map[strin
OfferedBy: map[protocol.DeviceID]ObservedFolder{},
}
}
of.fromWire(&protoF)
res[folderID].OfferedBy[deviceID] = of
continue
deleteKey:
lib/db/schemaupdater.go
+16 -846
View File
@@ -7,12 +7,11 @@
package db
import (
"bytes"
"fmt"
"sort"
"strings"
"github.com/syncthing/syncthing/lib/db/backend"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/bep"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -65,6 +64,12 @@ func (db *schemaUpdater) updateSchema() error {
return err
}
if prevVersion > 0 && prevVersion < 14 {
// This is a database version that is too old to be upgraded directly.
// The user will have to upgrade to an older version first.
return fmt.Errorf("database version %d is too old to be upgraded directly; step via Syncthing v1.27.0 to upgrade", prevVersion)
}
if prevVersion > dbVersion {
err := &databaseDowngradeError{}
if minSyncthingVersion, ok, dbErr := miscDB.String("dbMinSyncthingVersion"); dbErr != nil {
@@ -89,16 +94,6 @@ func (db *schemaUpdater) updateSchema() error {
}
migrations := []migration{
{1, 1, "v0.14.0", db.updateSchema0to1},
{2, 2, "v0.14.46", db.updateSchema1to2},
{3, 3, "v0.14.48", db.updateSchema2to3},
{5, 5, "v0.14.49", db.updateSchemaTo5},
{6, 6, "v0.14.50", db.updateSchema5to6},
{7, 7, "v0.14.53", db.updateSchema6to7},
{9, 9, "v1.4.0", db.updateSchemaTo9},
{10, 10, "v1.6.0", db.updateSchemaTo10},
{11, 11, "v1.6.0", db.updateSchemaTo11},
{13, 13, "v1.7.0", db.updateSchemaTo13},
{14, 14, "v1.9.0", db.updateSchemaTo14},
{14, 16, "v1.9.0", db.checkRepairMigration},
{14, 17, "v1.9.0", db.migration17},
@@ -143,571 +138,6 @@ func (*schemaUpdater) writeVersions(m migration, miscDB *NamespacedKV) error {
return nil
}
func (db *schemaUpdater) updateSchema0to1(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
dbi, err := t.NewPrefixIterator([]byte{KeyTypeDevice})
if err != nil {
return err
}
defer dbi.Release()
symlinkConv := 0
changedFolders := make(map[string]struct{})
ignAdded := 0
var gk []byte
ro := t.readOnlyTransaction
for dbi.Next() {
folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
if !ok {
// not having the folder in the index is bad; delete and continue
if err := t.Delete(dbi.Key()); err != nil {
return err
}
continue
}
device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
if !ok {
// not having the device in the index is bad; delete and continue
if err := t.Delete(dbi.Key()); err != nil {
return err
}
continue
}
name := db.keyer.NameFromDeviceFileKey(dbi.Key())
// Remove files with absolute path (see #4799)
if strings.HasPrefix(string(name), "/") {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
if err := t.Delete(dbi.Key()); err != nil {
return err
}
gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if err != nil {
return err
}
fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
if backend.IsNotFound(err) {
// Shouldn't happen, but not critical.
continue
} else if err != nil {
return err
}
_, _ = fl.pop(device)
if len(fl.Versions) == 0 {
err = t.Delete(gk)
} else {
err = t.Put(gk, mustMarshal(&fl))
}
if err != nil {
return err
}
continue
}
// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
// local and remote, and does not reset delta indexes. It shouldn't really
// matter what the symlink type is, but this cleans it up for a possible
// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
var f protocol.FileInfo
if err := f.Unmarshal(dbi.Value()); err != nil {
// probably can't happen
continue
}
if f.Type == protocol.FileInfoTypeSymlinkDirectory || f.Type == protocol.FileInfoTypeSymlinkFile {
f.Type = protocol.FileInfoTypeSymlink
bs, err := f.Marshal()
if err != nil {
panic("can't happen: " + err.Error())
}
if err := t.Put(dbi.Key(), bs); err != nil {
return err
}
symlinkConv++
}
// Add invalid files to global list
if f.IsInvalid() {
gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if err != nil {
return err
}
fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
if err != nil && !backend.IsNotFound(err) {
return err
}
i := sort.Search(len(fl.Versions), func(j int) bool {
return fl.Versions[j].Invalid
})
for ; i < len(fl.Versions); i++ {
ordering := fl.Versions[i].Version.Compare(f.Version)
shouldInsert := ordering == protocol.Equal
if !shouldInsert {
shouldInsert, err = shouldInsertBefore(ordering, folder, fl.Versions[i].Device, true, f, ro)
if err != nil {
return err
}
}
if shouldInsert {
nv := FileVersionDeprecated{
Device: device,
Version: f.Version,
Invalid: true,
}
fl.insertAt(i, nv)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
return err
}
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
ignAdded++
break
}
}
}
if err := t.Checkpoint(); err != nil {
return err
}
}
dbi.Release()
if err != dbi.Error() {
return err
}
return t.Commit()
}
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
// to allow iteration in sequence order (simplifies sending indexes).
func (db *schemaUpdater) updateSchema1to2(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var sk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f protocol.FileIntf) bool {
sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
if putErr != nil {
return false
}
dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
if putErr != nil {
return false
}
putErr = t.Put(sk, dk)
return putErr == nil
})
if putErr != nil {
return putErr
}
if err != nil {
return err
}
}
return t.Commit()
}
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
func (db *schemaUpdater) updateSchema2to3(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var nk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := withGlobalBefore11(folder, true, func(f protocol.FileIntf) bool {
name := []byte(f.FileName())
dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
if putErr != nil {
return false
}
var v protocol.Vector
haveFile, ok, err := t.getFileTrunc(dk, true)
if err != nil {
putErr = err
return false
}
if ok {
v = haveFile.FileVersion()
}
fv := FileVersionDeprecated{
Version: f.FileVersion(),
Invalid: f.IsInvalid(),
Deleted: f.IsDeleted(),
}
if !needDeprecated(fv, ok, v) {
return true
}
nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
if putErr != nil {
return false
}
putErr = t.Put(nk, nil)
return putErr == nil
}, t.readOnlyTransaction)
if putErr != nil {
return putErr
}
if err != nil {
return err
}
}
return t.Commit()
}
// updateSchemaTo5 resets the need bucket due to bugs existing in the v0.14.49
// release candidates (dbVersion 3 and 4)
// https://github.com/syncthing/syncthing/issues/5007
// https://github.com/syncthing/syncthing/issues/5053
func (db *schemaUpdater) updateSchemaTo5(prevVersion int) error {
if prevVersion != 3 && prevVersion != 4 {
return nil
}
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
var nk []byte
for _, folderStr := range db.ListFolders() {
nk, err = db.keyer.GenerateNeedFileKey(nk, []byte(folderStr), nil)
if err != nil {
return err
}
if err := t.deleteKeyPrefix(nk[:keyPrefixLen+keyFolderLen]); err != nil {
return err
}
}
if err := t.Commit(); err != nil {
return err
}
return db.updateSchema2to3(2)
}
func (db *schemaUpdater) updateSchema5to6(_ int) error {
// For every local file with the Invalid bit set, clear the Invalid bit and
// set LocalFlags = FlagLocalIgnored.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var iterErr error
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f protocol.FileIntf) bool {
if !f.IsInvalid() {
return true
}
fi := f.(protocol.FileInfo)
fi.RawInvalid = false
fi.LocalFlags = protocol.FlagLocalIgnored
bs, _ := fi.Marshal()
dk, iterErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(fi.Name))
if iterErr != nil {
return false
}
if iterErr = t.Put(dk, bs); iterErr != nil {
return false
}
iterErr = t.Checkpoint()
return iterErr == nil
})
if iterErr != nil {
return iterErr
}
if err != nil {
return err
}
}
return t.Commit()
}
// updateSchema6to7 checks whether all currently locally needed files are really
// needed and removes them if not.
func (db *schemaUpdater) updateSchema6to7(_ int) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var gk []byte
var nk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var delErr error
err := withNeedLocalBefore11(folder, false, func(f protocol.FileIntf) bool {
name := []byte(f.FileName())
gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if delErr != nil {
return false
}
svl, err := t.Get(gk)
if err != nil {
// If there is no global list, we hardly need it.
key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
if err != nil {
delErr = err
return false
}
delErr = t.Delete(key)
return delErr == nil
}
var fl VersionListDeprecated
err = fl.Unmarshal(svl)
if err != nil {
// This can't happen, but it's ignored everywhere else too,
// so lets not act on it.
return true
}
globalFV := FileVersionDeprecated{
Version: f.FileVersion(),
Invalid: f.IsInvalid(),
Deleted: f.IsDeleted(),
}
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !needDeprecated(globalFV, haveLocalFV, localFV.Version) {
key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
if err != nil {
delErr = err
return false
}
delErr = t.Delete(key)
}
return delErr == nil
}, t.readOnlyTransaction)
if delErr != nil {
return delErr
}
if err != nil {
return err
}
if err := t.Checkpoint(); err != nil {
return err
}
}
return t.Commit()
}
func (db *schemaUpdater) updateSchemaTo9(_ int) error {
// Loads and rewrites all files with blocks, to deduplicate block lists.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
if err := rewriteFiles(t); err != nil {
return err
}
db.recordTime(indirectGCTimeKey)
return t.Commit()
}
func rewriteFiles(t readWriteTransaction) error {
it, err := t.NewPrefixIterator([]byte{KeyTypeDevice})
if err != nil {
return err
}
defer it.Release()
for it.Next() {
intf, err := t.unmarshalTrunc(it.Value(), false)
if backend.IsNotFound(err) {
// Unmarshal error due to missing parts (block list), probably
// due to a bad migration in a previous RC. Drop this key, as
// getFile would anyway return this as a "not found" in the
// normal flow of things.
if err := t.Delete(it.Key()); err != nil {
return err
}
continue
} else if err != nil {
return err
}
fi := intf.(protocol.FileInfo)
if fi.Blocks == nil {
continue
}
if err := t.putFile(it.Key(), fi); err != nil {
return err
}
if err := t.Checkpoint(); err != nil {
return err
}
}
it.Release()
return it.Error()
}
func (db *schemaUpdater) updateSchemaTo10(_ int) error {
// Rewrites global lists to include a Deleted flag.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var buf []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
buf, err = t.keyer.GenerateGlobalVersionKey(buf, folder, nil)
if err != nil {
return err
}
buf = globalVersionKey(buf).WithoutName()
dbi, err := t.NewPrefixIterator(buf)
if err != nil {
return err
}
defer dbi.Release()
for dbi.Next() {
var vl VersionListDeprecated
if err := vl.Unmarshal(dbi.Value()); err != nil {
return err
}
changed := false
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
for i, fv := range vl.Versions {
buf, err = t.keyer.GenerateDeviceFileKey(buf, folder, fv.Device, name)
if err != nil {
return err
}
f, ok, err := t.getFileTrunc(buf, true)
if !ok {
return errEntryFromGlobalMissing
}
if err != nil {
return err
}
if f.IsDeleted() {
vl.Versions[i].Deleted = true
changed = true
}
}
if changed {
if err := t.Put(dbi.Key(), mustMarshal(&vl)); err != nil {
return err
}
if err := t.Checkpoint(); err != nil {
return err
}
}
}
dbi.Release()
}
// Trigger metadata recalc
if err := t.deleteKeyPrefix([]byte{KeyTypeFolderMeta}); err != nil {
return err
}
return t.Commit()
}
func (db *schemaUpdater) updateSchemaTo11(_ int) error {
// Populates block list map for every folder.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi protocol.FileIntf) bool {
f := fi.(FileInfoTruncated)
if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() || f.IsInvalid() || f.BlocksHash == nil {
return true
}
name := []byte(f.FileName())
dk, putErr = db.keyer.GenerateBlockListMapKey(dk, folder, f.BlocksHash, name)
if putErr != nil {
return false
}
if putErr = t.Put(dk, nil); putErr != nil {
return false
}
putErr = t.Checkpoint()
return putErr == nil
})
if putErr != nil {
return putErr
}
if err != nil {
return err
}
}
return t.Commit()
}
func (db *schemaUpdater) updateSchemaTo13(prev int) error {
// Loads and rewrites all files, to deduplicate version vectors.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
if prev < 12 {
if err := rewriteFiles(t); err != nil {
return err
}
}
if err := rewriteGlobals(t); err != nil {
return err
}
return t.Commit()
}
func (db *schemaUpdater) updateSchemaTo14(_ int) error {
// Checks for missing blocks and marks those entries as requiring a
// rehash/being invalid. The db is checked/repaired afterwards, i.e.
@@ -737,10 +167,11 @@ func (db *schemaUpdater) updateSchemaTo14(_ int) error {
}
defer it.Release()
for it.Next() {
var fi protocol.FileInfo
if err := fi.Unmarshal(it.Value()); err != nil {
var bepf bep.FileInfo
if err := proto.Unmarshal(it.Value(), &bepf); err != nil {
return err
}
fi := protocol.FileInfoFromDB(&bepf)
if len(fi.Blocks) > 0 || len(fi.BlocksHash) == 0 {
continue
}
@@ -808,12 +239,11 @@ func (db *schemaUpdater) migration17(prev int) error {
return db.updateLocalFiles(folder, fs, meta)
})
var innerErr error
err = t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(fi protocol.FileIntf) bool {
err = t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(fi protocol.FileInfo) bool {
if fi.IsInvalid() && fi.FileLocalFlags() == 0 {
f := fi.(protocol.FileInfo)
f.SetMustRescan()
f.Version = protocol.Vector{}
batch.Append(f)
fi.SetMustRescan()
fi.Version = protocol.Vector{}
batch.Append(fi)
innerErr = batch.FlushIfFull()
return innerErr == nil
}
@@ -839,263 +269,3 @@ func (db *schemaUpdater) dropAllIndexIDsMigration(_ int) error {
func (db *schemaUpdater) dropOutgoingIndexIDsMigration(_ int) error {
return db.dropOtherDeviceIndexIDs()
}
func rewriteGlobals(t readWriteTransaction) error {
it, err := t.NewPrefixIterator([]byte{KeyTypeGlobal})
if err != nil {
return err
}
defer it.Release()
for it.Next() {
var vl VersionListDeprecated
if err := vl.Unmarshal(it.Value()); err != nil {
// If we crashed during an earlier migration, some version
// lists might already be in the new format: Skip those.
var nvl VersionList
if nerr := nvl.Unmarshal(it.Value()); nerr == nil {
continue
}
return err
}
if len(vl.Versions) == 0 {
if err := t.Delete(it.Key()); err != nil {
return err
}
}
newVl := convertVersionList(vl)
if err := t.Put(it.Key(), mustMarshal(&newVl)); err != nil {
return err
}
if err := t.Checkpoint(); err != nil {
return err
}
}
return it.Error()
}
func convertVersionList(vl VersionListDeprecated) VersionList {
var newVl VersionList
var newPos, oldPos int
var lastVersion protocol.Vector
for _, fv := range vl.Versions {
if fv.Invalid {
break
}
oldPos++
if len(newVl.RawVersions) > 0 && lastVersion.Equal(fv.Version) {
newVl.RawVersions[newPos].Devices = append(newVl.RawVersions[newPos].Devices, fv.Device)
continue
}
newPos = len(newVl.RawVersions)
newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, false, fv.Deleted))
lastVersion = fv.Version
}
if oldPos == len(vl.Versions) {
return newVl
}
if len(newVl.RawVersions) == 0 {
fv := vl.Versions[oldPos]
newVl.RawVersions = []FileVersion{newFileVersion(fv.Device, fv.Version, true, fv.Deleted)}
oldPos++
}
newPos = 0
outer:
for _, fv := range vl.Versions[oldPos:] {
for _, nfv := range newVl.RawVersions[newPos:] {
switch nfv.Version.Compare(fv.Version) {
case protocol.Equal:
newVl.RawVersions[newPos].InvalidDevices = append(newVl.RawVersions[newPos].InvalidDevices, fv.Device)
continue outer
case protocol.Lesser:
newVl.insertAt(newPos, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
continue outer
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version is invalid, i.e. it looses anyway,
// no need to check/get the conflicting file.
}
newPos++
}
// Couldn't insert into any existing versions
newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
newPos++
}
return newVl
}
func getGlobalVersionsByKeyBefore11(key []byte, t readOnlyTransaction) (VersionListDeprecated, error) {
bs, err := t.Get(key)
if err != nil {
return VersionListDeprecated{}, err
}
var vl VersionListDeprecated
if err := vl.Unmarshal(bs); err != nil {
return VersionListDeprecated{}, err
}
return vl, nil
}
func withGlobalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer dbi.Release()
var dk []byte
for dbi.Next() {
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
var vl VersionListDeprecated
if err := vl.Unmarshal(dbi.Value()); err != nil {
return err
}
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
if err != nil {
return err
}
f, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
if err != nil {
return err
}
return dbi.Error()
}
func withNeedLocalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutName())
if err != nil {
return err
}
defer dbi.Release()
var keyBuf []byte
var f protocol.FileIntf
var ok bool
for dbi.Next() {
keyBuf, f, ok, err = getGlobalBefore11(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate, t)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func getGlobalBefore11(keyBuf, folder, file []byte, truncate bool, t readOnlyTransaction) ([]byte, protocol.FileIntf, bool, error) {
keyBuf, err := t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
if err != nil {
return nil, nil, false, err
}
bs, err := t.Get(keyBuf)
if backend.IsNotFound(err) {
return keyBuf, nil, false, nil
} else if err != nil {
return nil, nil, false, err
}
var vl VersionListDeprecated
if err := vl.Unmarshal(bs); err != nil {
return nil, nil, false, err
}
if len(vl.Versions) == 0 {
return nil, nil, false, nil
}
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
if err != nil {
return nil, nil, false, err
}
fi, ok, err := t.getFileTrunc(keyBuf, truncate)
if err != nil || !ok {
return keyBuf, nil, false, err
}
return keyBuf, fi, true, nil
}
func (vl *VersionListDeprecated) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range vl.Versions {
if i > 0 {
b.WriteString(", ")
}
copy(id[:], v.Device)
fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
}
b.WriteString("}")
return b.String()
}
func (vl *VersionListDeprecated) pop(device []byte) (FileVersionDeprecated, int) {
for i, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
return v, i
}
}
return FileVersionDeprecated{}, -1
}
func (vl *VersionListDeprecated) Get(device []byte) (FileVersionDeprecated, bool) {
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
return v, true
}
}
return FileVersionDeprecated{}, false
}
func (vl *VersionListDeprecated) insertAt(i int, v FileVersionDeprecated) {
vl.Versions = append(vl.Versions, FileVersionDeprecated{})
copy(vl.Versions[i+1:], vl.Versions[i:])
vl.Versions[i] = v
}
func needDeprecated(global FileVersionDeprecated, haveLocal bool, localVersion protocol.Vector) bool {
// We never need an invalid file.
if global.Invalid {
return false
}
// We don't need a deleted file if we don't have it.
if global.Deleted && !haveLocal {
return false
}
// We don't need the global file if we already have the same version.
if haveLocal && localVersion.GreaterEqual(global.Version) {
return false
}
return true
}
lib/db/set.go
+53 -24
View File
@@ -13,8 +13,10 @@
package db
import (
"bytes"
"fmt"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
@@ -33,7 +35,7 @@ type FileSet struct {
// The Iterator is called with either a protocol.FileInfo or a
// FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f protocol.FileIntf) bool
type Iterator func(f protocol.FileInfo) bool
func NewFileSet(folder string, db *Lowlevel) (*FileSet, error) {
select {
@@ -292,26 +294,24 @@ func (s *Snapshot) GetGlobal(file string) (protocol.FileInfo, bool) {
if !ok {
return protocol.FileInfo{}, false
}
f := fi.(protocol.FileInfo)
f.Name = osutil.NativeFilename(f.Name)
return f, true
fi.Name = osutil.NativeFilename(fi.Name)
return fi, true
}
func (s *Snapshot) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
func (s *Snapshot) GetGlobalTruncated(file string) (protocol.FileInfo, bool) {
opStr := fmt.Sprintf("%s GetGlobalTruncated(%v)", s.folder, file)
l.Debugf(opStr)
_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
if backend.IsClosed(err) {
return FileInfoTruncated{}, false
return protocol.FileInfo{}, false
} else if err != nil {
s.fatalError(err, opStr)
}
if !ok {
return FileInfoTruncated{}, false
return protocol.FileInfo{}, false
}
f := fi.(FileInfoTruncated)
f.Name = osutil.NativeFilename(f.Name)
return f, true
fi.Name = osutil.NativeFilename(fi.Name)
return fi, true
}
func (s *Snapshot) Availability(file string) []protocol.DeviceID {
@@ -326,16 +326,16 @@ func (s *Snapshot) Availability(file string) []protocol.DeviceID {
return av
}
func (s *Snapshot) DebugGlobalVersions(file string) VersionList {
func (s *Snapshot) DebugGlobalVersions(file string) *DebugVersionList {
opStr := fmt.Sprintf("%s DebugGlobalVersions(%v)", s.folder, file)
l.Debugf(opStr)
vl, err := s.t.getGlobalVersions(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
if backend.IsClosed(err) || backend.IsNotFound(err) {
return VersionList{}
return nil
} else if err != nil {
s.fatalError(err, opStr)
}
return vl
return &DebugVersionList{vl}
}
func (s *Snapshot) Sequence(device protocol.DeviceID) int64 {
@@ -503,17 +503,9 @@ func normalizeFilenamesAndDropDuplicates(fs []protocol.FileInfo) []protocol.File
}
func nativeFileIterator(fn Iterator) Iterator {
return func(fi protocol.FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = osutil.NativeFilename(f.Name)
return fn(f)
case FileInfoTruncated:
f.Name = osutil.NativeFilename(f.Name)
return fn(f)
default:
panic("unknown interface type")
}
return func(fi protocol.FileInfo) bool {
fi.Name = osutil.NativeFilename(fi.Name)
return fn(fi)
}
}
@@ -522,3 +514,40 @@ func fatalError(err error, opStr string, db *Lowlevel) {
l.Warnf("Fatal error: %v: %v", opStr, err)
panic(ldbPathRe.ReplaceAllString(err.Error(), "$1 x: "))
}
// DebugFileVersion is the database-internal representation of a file
// version, with a nicer string representation, used only by API debug
// methods.
type DebugVersionList struct {
*dbproto.VersionList
}
func (vl DebugVersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("[")
for i, v := range vl.Versions {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "{Version:%v, Deleted:%v, Devices:[", protocol.VectorFromWire(v.Version), v.Deleted)
for j, dev := range v.Devices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
b.WriteString("], Invalid:[")
for j, dev := range v.InvalidDevices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
fmt.Fprint(&b, "]}")
}
b.WriteString("]")
return b.String()
}
lib/db/set_test.go
+23 -27
View File
@@ -16,6 +16,7 @@ import (
"time"
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
@@ -48,21 +49,19 @@ func globalList(t testing.TB, s *db.FileSet) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
snap.WithGlobal(func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
snap.WithGlobal(func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
}
func globalListPrefixed(t testing.TB, s *db.FileSet, prefix string) []db.FileInfoTruncated {
var fs []db.FileInfoTruncated
func globalListPrefixed(t testing.TB, s *db.FileSet, prefix string) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
fs = append(fs, f)
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
@@ -72,21 +71,19 @@ func haveList(t testing.TB, s *db.FileSet, n protocol.DeviceID) []protocol.FileI
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
snap.WithHave(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
snap.WithHave(n, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
}
func haveListPrefixed(t testing.TB, s *db.FileSet, n protocol.DeviceID, prefix string) []db.FileInfoTruncated {
var fs []db.FileInfoTruncated
func haveListPrefixed(t testing.TB, s *db.FileSet, n protocol.DeviceID, prefix string) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
snap.WithPrefixedHaveTruncated(n, prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
fs = append(fs, f)
snap.WithPrefixedHaveTruncated(n, prefix, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
@@ -96,9 +93,8 @@ func needList(t testing.TB, s *db.FileSet, n protocol.DeviceID) []protocol.FileI
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
snap.WithNeed(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
snap.WithNeed(n, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
@@ -1011,9 +1007,9 @@ func TestWithHaveSequence(t *testing.T) {
i := 2
snap := snapshot(t, s)
defer snap.Release()
snap.WithHaveSequence(int64(i), func(fi protocol.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
snap.WithHaveSequence(int64(i), func(fi protocol.FileInfo) bool {
if !fi.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", fi, localHave[i-1])
}
i++
return true
@@ -1062,7 +1058,7 @@ loop:
default:
}
snap := snapshot(t, s)
snap.WithHaveSequence(prevSeq+1, func(fi protocol.FileIntf) bool {
snap.WithHaveSequence(prevSeq+1, func(fi protocol.FileInfo) bool {
if fi.SequenceNo() < prevSeq+1 {
t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo())
}
@@ -1540,8 +1536,8 @@ func TestSequenceIndex(t *testing.T) {
// Start a routine to walk the sequence index and inspect the result.
seen := make(map[string]protocol.FileIntf)
latest := make([]protocol.FileIntf, 0, len(local))
seen := make(map[string]protocol.FileInfo)
latest := make([]protocol.FileInfo, 0, len(local))
var seq int64
t0 := time.Now()
@@ -1552,7 +1548,7 @@ func TestSequenceIndex(t *testing.T) {
// update has happened since our last iteration.
latest = latest[:0]
snap := snapshot(t, s)
snap.WithHaveSequence(seq+1, func(f protocol.FileIntf) bool {
snap.WithHaveSequence(seq+1, func(f protocol.FileInfo) bool {
seen[f.FileName()] = f
latest = append(latest, f)
seq = f.SequenceNo()
@@ -1657,7 +1653,7 @@ func TestUpdateWithOneFileTwice(t *testing.T) {
snap := snapshot(t, s)
defer snap.Release()
count := 0
snap.WithHaveSequence(0, func(f protocol.FileIntf) bool {
snap.WithHaveSequence(0, func(_ protocol.FileInfo) bool {
count++
return true
})
lib/db/structs.go
+108 -269
View File
@@ -10,172 +10,52 @@ import (
"bytes"
"fmt"
"strings"
"time"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/protocol"
)
func (f FileInfoTruncated) String() string {
switch f.Type {
case protocol.FileInfoTypeDirectory:
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions)
case protocol.FileInfoTypeFile:
return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, BlockSize:%d}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.RawBlockSize)
case protocol.FileInfoTypeSymlink, protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile:
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, Deleted:%v, Invalid:%v, LocalFlags:0x%x, NoPermissions:%v, SymlinkTarget:%q}",
f.Name, f.Type, f.Sequence, f.Version, f.Deleted, f.RawInvalid, f.LocalFlags, f.NoPermissions, f.SymlinkTarget)
default:
panic("mystery file type detected")
type CountsSet struct {
Counts []Counts
Created int64 // unix nanos
}
type Counts struct {
Files int
Directories int
Symlinks int
Deleted int
Bytes int64
Sequence int64 // zero for the global state
DeviceID protocol.DeviceID // device ID for remote devices, or special values for local/global
LocalFlags uint32 // the local flag for this count bucket
}
func (c Counts) toWire() *dbproto.Counts {
return &dbproto.Counts{
Files: int32(c.Files),
Directories: int32(c.Directories),
Symlinks: int32(c.Symlinks),
Deleted: int32(c.Deleted),
Bytes: c.Bytes,
Sequence: c.Sequence,
DeviceId: c.DeviceID[:],
LocalFlags: c.LocalFlags,
}
}
func (f FileInfoTruncated) IsDeleted() bool {
return f.Deleted
}
func (f FileInfoTruncated) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0
}
func (f FileInfoTruncated) IsUnsupported() bool {
return f.LocalFlags&protocol.FlagLocalUnsupported != 0
}
func (f FileInfoTruncated) IsIgnored() bool {
return f.LocalFlags&protocol.FlagLocalIgnored != 0
}
func (f FileInfoTruncated) MustRescan() bool {
return f.LocalFlags&protocol.FlagLocalMustRescan != 0
}
func (f FileInfoTruncated) IsReceiveOnlyChanged() bool {
return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Type == protocol.FileInfoTypeDirectory
}
func (f FileInfoTruncated) IsSymlink() bool {
switch f.Type {
case protocol.FileInfoTypeSymlink, protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile:
return true
default:
return false
}
}
func (f FileInfoTruncated) ShouldConflict() bool {
return f.LocalFlags&protocol.LocalConflictFlags != 0
}
func (f FileInfoTruncated) HasPermissionBits() bool {
return !f.NoPermissions
}
func (f FileInfoTruncated) FileSize() int64 {
if f.Deleted {
return 0
}
if f.IsDirectory() || f.IsSymlink() {
return protocol.SyntheticDirectorySize
}
return f.Size
}
func (f FileInfoTruncated) BlockSize() int {
if f.RawBlockSize == 0 {
return protocol.MinBlockSize
}
return int(f.RawBlockSize)
}
func (f FileInfoTruncated) FileName() string {
return f.Name
}
func (f FileInfoTruncated) FileLocalFlags() uint32 {
return f.LocalFlags
}
func (f FileInfoTruncated) ModTime() time.Time {
return time.Unix(f.ModifiedS, int64(f.ModifiedNs))
}
func (f FileInfoTruncated) SequenceNo() int64 {
return f.Sequence
}
func (f FileInfoTruncated) FileVersion() protocol.Vector {
return f.Version
}
func (f FileInfoTruncated) FileType() protocol.FileInfoType {
return f.Type
}
func (f FileInfoTruncated) FilePermissions() uint32 {
return f.Permissions
}
func (f FileInfoTruncated) FileModifiedBy() protocol.ShortID {
return f.ModifiedBy
}
func (f FileInfoTruncated) PlatformData() protocol.PlatformData {
return f.Platform
}
func (f FileInfoTruncated) InodeChangeTime() time.Time {
return time.Unix(0, f.InodeChangeNs)
}
func (f FileInfoTruncated) FileBlocksHash() []byte {
return f.BlocksHash
}
func (f FileInfoTruncated) ConvertToIgnoredFileInfo() protocol.FileInfo {
file := f.copyToFileInfo()
file.SetIgnored()
return file
}
func (f FileInfoTruncated) ConvertToDeletedFileInfo(by protocol.ShortID) protocol.FileInfo {
file := f.copyToFileInfo()
file.SetDeleted(by)
return file
}
// ConvertDeletedToFileInfo converts a deleted truncated file info to a regular file info
func (f FileInfoTruncated) ConvertDeletedToFileInfo() protocol.FileInfo {
if !f.Deleted {
panic("ConvertDeletedToFileInfo must only be called on deleted items")
}
return f.copyToFileInfo()
}
// copyToFileInfo just copies all members of FileInfoTruncated to protocol.FileInfo
func (f FileInfoTruncated) copyToFileInfo() protocol.FileInfo {
return protocol.FileInfo{
Name: f.Name,
Size: f.Size,
ModifiedS: f.ModifiedS,
ModifiedBy: f.ModifiedBy,
Version: f.Version,
Sequence: f.Sequence,
SymlinkTarget: f.SymlinkTarget,
BlocksHash: f.BlocksHash,
Type: f.Type,
Permissions: f.Permissions,
ModifiedNs: f.ModifiedNs,
RawBlockSize: f.RawBlockSize,
LocalFlags: f.LocalFlags,
Deleted: f.Deleted,
RawInvalid: f.RawInvalid,
NoPermissions: f.NoPermissions,
func countsFromWire(w *dbproto.Counts) Counts {
return Counts{
Files: int(w.Files),
Directories: int(w.Directories),
Symlinks: int(w.Symlinks),
Deleted: int(w.Deleted),
Bytes: w.Bytes,
Sequence: w.Sequence,
DeviceID: protocol.DeviceID(w.DeviceId),
LocalFlags: w.LocalFlags,
}
}
@@ -187,7 +67,7 @@ func (c Counts) Add(other Counts) Counts {
Deleted: c.Deleted + other.Deleted,
Bytes: c.Bytes + other.Bytes,
Sequence: c.Sequence + other.Sequence,
DeviceID: protocol.EmptyDeviceID[:],
DeviceID: protocol.EmptyDeviceID,
LocalFlags: c.LocalFlags | other.LocalFlags,
}
}
@@ -197,7 +77,6 @@ func (c Counts) TotalItems() int {
}
func (c Counts) String() string {
dev, _ := protocol.DeviceIDFromBytes(c.DeviceID)
var flags strings.Builder
if c.LocalFlags&needFlag != 0 {
flags.WriteString("Need")
@@ -220,7 +99,7 @@ func (c Counts) String() string {
if flags.Len() == 0 {
flags.WriteString("---")
}
return fmt.Sprintf("{Device:%v, Files:%d, Dirs:%d, Symlinks:%d, Del:%d, Bytes:%d, Seq:%d, Flags:%s}", dev, c.Files, c.Directories, c.Symlinks, c.Deleted, c.Bytes, c.Sequence, flags.String())
return fmt.Sprintf("{Device:%v, Files:%d, Dirs:%d, Symlinks:%d, Del:%d, Bytes:%d, Seq:%d, Flags:%s}", c.DeviceID, c.Files, c.Directories, c.Symlinks, c.Deleted, c.Bytes, c.Sequence, flags.String())
}
// Equal compares the numbers only, not sequence/dev/flags.
@@ -228,80 +107,50 @@ func (c Counts) Equal(o Counts) bool {
return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes
}
func (vl VersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range vl.RawVersions {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "{Version:%v, Deleted:%v, Devices:{", v.Version, v.Deleted)
for j, dev := range v.Devices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
b.WriteString("}, Invalid:{")
for j, dev := range v.InvalidDevices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
fmt.Fprint(&b, "}}")
}
b.WriteString("}")
return b.String()
}
// update brings the VersionList up to date with file. It returns the updated
// VersionList, a device that has the global/newest version, a device that previously
// had the global/newest version, a boolean indicating if the global version has
// changed and if any error occurred (only possible in db interaction).
func (vl *VersionList) update(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (FileVersion, FileVersion, FileVersion, bool, bool, bool, error) {
if len(vl.RawVersions) == 0 {
func vlUpdate(vl *dbproto.VersionList, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (*dbproto.FileVersion, *dbproto.FileVersion, *dbproto.FileVersion, bool, bool, bool, error) {
if len(vl.Versions) == 0 {
nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
vl.RawVersions = append(vl.RawVersions, nv)
return nv, FileVersion{}, FileVersion{}, false, false, true, nil
vl.Versions = append(vl.Versions, nv)
return nv, nil, nil, false, false, true, nil
}
// Get the current global (before updating)
oldFV, haveOldGlobal := vl.GetGlobal()
oldFV = oldFV.copy()
oldFV, haveOldGlobal := vlGetGlobal(vl)
oldFV = fvCopy(oldFV)
// Remove ourselves first
removedFV, haveRemoved, _ := vl.pop(device)
removedFV, haveRemoved, _ := vlPop(vl, device)
// Find position and insert the file
err := vl.insert(folder, device, file, t)
err := vlInsert(vl, folder, device, file, t)
if err != nil {
return FileVersion{}, FileVersion{}, FileVersion{}, false, false, false, err
return nil, nil, nil, false, false, false, err
}
newFV, _ := vl.GetGlobal() // We just inserted something above, can't be empty
newFV, _ := vlGetGlobal(vl) // We just inserted something above, can't be empty
if !haveOldGlobal {
return newFV, FileVersion{}, removedFV, false, haveRemoved, true, nil
return newFV, nil, removedFV, false, haveRemoved, true, nil
}
globalChanged := true
if oldFV.IsInvalid() == newFV.IsInvalid() && oldFV.Version.Equal(newFV.Version) {
if fvIsInvalid(oldFV) == fvIsInvalid(newFV) && protocol.VectorFromWire(oldFV.Version).Equal(protocol.VectorFromWire(newFV.Version)) {
globalChanged = false
}
return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
}
func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) error {
func vlInsert(vl *dbproto.VersionList, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) error {
var added bool
var err error
i := 0
for ; i < len(vl.RawVersions); i++ {
for ; i < len(vl.Versions); i++ {
// Insert our new version
added, err = vl.checkInsertAt(i, folder, device, file, t)
added, err = vlCheckInsertAt(vl, i, folder, device, file, t)
if err != nil {
return err
}
@@ -309,80 +158,76 @@ func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t r
break
}
}
if i == len(vl.RawVersions) {
if i == len(vl.Versions) {
// Append to the end
vl.RawVersions = append(vl.RawVersions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
vl.Versions = append(vl.Versions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
}
return nil
}
func (vl *VersionList) insertAt(i int, v FileVersion) {
vl.RawVersions = append(vl.RawVersions, FileVersion{})
copy(vl.RawVersions[i+1:], vl.RawVersions[i:])
vl.RawVersions[i] = v
func vlInsertAt(vl *dbproto.VersionList, i int, v *dbproto.FileVersion) {
vl.Versions = append(vl.Versions, &dbproto.FileVersion{})
copy(vl.Versions[i+1:], vl.Versions[i:])
vl.Versions[i] = v
}
// pop removes the given device from the VersionList and returns the FileVersion
// before removing the device, whether it was found/removed at all and whether
// the global changed in the process.
func (vl *VersionList) pop(device []byte) (FileVersion, bool, bool) {
invDevice, i, j, ok := vl.findDevice(device)
func vlPop(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool, bool) {
invDevice, i, j, ok := vlFindDevice(vl, device)
if !ok {
return FileVersion{}, false, false
return nil, false, false
}
globalPos := vl.findGlobal()
globalPos := vlFindGlobal(vl)
if vl.RawVersions[i].deviceCount() == 1 {
fv := vl.RawVersions[i]
vl.popVersionAt(i)
fv := vl.Versions[i]
if fvDeviceCount(fv) == 1 {
vlPopVersionAt(vl, i)
return fv, true, globalPos == i
}
oldFV := vl.RawVersions[i].copy()
oldFV := fvCopy(fv)
if invDevice {
vl.RawVersions[i].InvalidDevices = popDeviceAt(vl.RawVersions[i].InvalidDevices, j)
vl.Versions[i].InvalidDevices = popDeviceAt(vl.Versions[i].InvalidDevices, j)
return oldFV, true, false
}
vl.RawVersions[i].Devices = popDeviceAt(vl.RawVersions[i].Devices, j)
vl.Versions[i].Devices = popDeviceAt(vl.Versions[i].Devices, j)
// If the last valid device of the previous global was removed above,
// the global changed.
return oldFV, true, len(vl.RawVersions[i].Devices) == 0 && globalPos == i
return oldFV, true, len(vl.Versions[i].Devices) == 0 && globalPos == i
}
// Get returns a FileVersion that contains the given device and whether it has
// been found at all.
func (vl *VersionList) Get(device []byte) (FileVersion, bool) {
_, i, _, ok := vl.findDevice(device)
func vlGet(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool) {
_, i, _, ok := vlFindDevice(vl, device)
if !ok {
return FileVersion{}, false
return &dbproto.FileVersion{}, false
}
return vl.RawVersions[i], true
return vl.Versions[i], true
}
// GetGlobal returns the current global FileVersion. The returned FileVersion
// may be invalid, if all FileVersions are invalid. Returns false only if
// VersionList is empty.
func (vl *VersionList) GetGlobal() (FileVersion, bool) {
i := vl.findGlobal()
func vlGetGlobal(vl *dbproto.VersionList) (*dbproto.FileVersion, bool) {
i := vlFindGlobal(vl)
if i == -1 {
return FileVersion{}, false
return nil, false
}
return vl.RawVersions[i], true
}
func (vl *VersionList) Empty() bool {
return len(vl.RawVersions) == 0
return vl.Versions[i], true
}
// findGlobal returns the first version that isn't invalid, or if all versions are
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
func (vl *VersionList) findGlobal() int {
for i, fv := range vl.RawVersions {
if !fv.IsInvalid() {
func vlFindGlobal(vl *dbproto.VersionList) int {
for i := range vl.Versions {
if !fvIsInvalid(vl.Versions[i]) {
return i
}
}
if len(vl.RawVersions) == 0 {
if len(vl.Versions) == 0 {
return -1
}
return 0
@@ -391,8 +236,8 @@ func (vl *VersionList) findGlobal() int {
// findDevice returns whether the device is in InvalidVersions or Versions and
// in InvalidDevices or Devices (true for invalid), the positions in the version
// and device slices and whether it has been found at all.
func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) {
for i, v := range vl.RawVersions {
func vlFindDevice(vl *dbproto.VersionList, device []byte) (bool, int, int, bool) {
for i, v := range vl.Versions {
if j := deviceIndex(v.Devices, device); j != -1 {
return false, i, j, true
}
@@ -403,30 +248,31 @@ func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) {
return false, -1, -1, false
}
func (vl *VersionList) popVersionAt(i int) {
vl.RawVersions = append(vl.RawVersions[:i], vl.RawVersions[i+1:]...)
func vlPopVersionAt(vl *dbproto.VersionList, i int) {
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
}
// checkInsertAt determines if the given device and associated file should be
// inserted into the FileVersion at position i or into a new FileVersion at
// position i.
func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
ordering := vl.RawVersions[i].Version.Compare(file.FileVersion())
func vlCheckInsertAt(vl *dbproto.VersionList, i int, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (bool, error) {
fv := vl.Versions[i]
ordering := protocol.VectorFromWire(fv.Version).Compare(file.FileVersion())
if ordering == protocol.Equal {
if !file.IsInvalid() {
vl.RawVersions[i].Devices = append(vl.RawVersions[i].Devices, device)
fv.Devices = append(fv.Devices, device)
} else {
vl.RawVersions[i].InvalidDevices = append(vl.RawVersions[i].InvalidDevices, device)
fv.InvalidDevices = append(fv.InvalidDevices, device)
}
return true, nil
}
existingDevice, _ := vl.RawVersions[i].FirstDevice()
insert, err := shouldInsertBefore(ordering, folder, existingDevice, vl.RawVersions[i].IsInvalid(), file, t)
existingDevice, _ := fvFirstDevice(fv)
insert, err := shouldInsertBefore(ordering, folder, existingDevice, fvIsInvalid(fv), file, t)
if err != nil {
return false, err
}
if insert {
vl.insertAt(i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
vlInsertAt(vl, i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
return true, nil
}
return false, nil
@@ -435,7 +281,7 @@ func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol
// shouldInsertBefore determines whether the file comes before an existing
// entry, given the version ordering (existing compared to new one), existing
// device and if the existing version is invalid.
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileInfo, t readOnlyTransaction) (bool, error) {
switch ordering {
case protocol.Lesser:
// The version at this point in the list is lesser
@@ -461,10 +307,7 @@ func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byt
if !ok {
return true, nil
}
if err != nil {
return false, err
}
if protocol.WinsConflict(file, of) {
if file.WinsConflict(of) {
return true, nil
}
}
@@ -484,9 +327,9 @@ func popDeviceAt(devices [][]byte, i int) [][]byte {
return append(devices[:i], devices[i+1:]...)
}
func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) FileVersion {
fv := FileVersion{
Version: version,
func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) *dbproto.FileVersion {
fv := &dbproto.FileVersion{
Version: version.ToWire(),
Deleted: deleted,
}
if invalid {
@@ -497,7 +340,7 @@ func newFileVersion(device []byte, version protocol.Vector, invalid, deleted boo
return fv
}
func (fv FileVersion) FirstDevice() ([]byte, bool) {
func fvFirstDevice(fv *dbproto.FileVersion) ([]byte, bool) {
if len(fv.Devices) != 0 {
return fv.Devices[0], true
}
@@ -507,18 +350,14 @@ func (fv FileVersion) FirstDevice() ([]byte, bool) {
return nil, false
}
func (fv FileVersion) IsInvalid() bool {
return len(fv.Devices) == 0
func fvIsInvalid(fv *dbproto.FileVersion) bool {
return fv == nil || len(fv.Devices) == 0
}
func (fv FileVersion) deviceCount() int {
func fvDeviceCount(fv *dbproto.FileVersion) int {
return len(fv.Devices) + len(fv.InvalidDevices)
}
func (fv FileVersion) copy() FileVersion {
n := fv
n.Version = fv.Version.Copy()
n.Devices = append([][]byte{}, fv.Devices...)
n.InvalidDevices = append([][]byte{}, fv.InvalidDevices...)
return n
func fvCopy(fv *dbproto.FileVersion) *dbproto.FileVersion {
return proto.Clone(fv).(*dbproto.FileVersion)
}
lib/db/structs.pb.go
-3478
View File
File diff suppressed because it is too large Load Diff
lib/db/testdata/.gitignore
-1
View File
@@ -1 +0,0 @@
index.db
lib/db/testdata/v0.14.45-update0to3.db.jsons
Vendored
-22
View File
@@ -1,22 +0,0 @@
{"k":"AAAAAAAAAAABL25vdGdvb2Q=","v":"Cggvbm90Z29vZEoHCgUIARDoB1ACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHw=="}
{"k":"AAAAAAAAAAABYQ==","v":"CgFhSgcKBQgBEOgHUAGCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f"}
{"k":"AAAAAAAAAAACYg==","v":"CgFiSgcKBQgBEOkHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIA=="}
{"k":"AAAAAAAAAAACYw==","v":"CgFjOAFKBwoFCAEQ6geCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
{"k":"AAAAAAAAAAACZA==","v":"CgFkSgcKBQgBEOsHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="}
{"k":"AAAAAAAAAAADYw==","v":"CgFjSgcKBQgBEOoHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="}
{"k":"AAAAAAAAAAADZA==","v":"CgFkOAFKBwoFCAEQ6weCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
{"k":"AAAAAAAAAAADaW52YWxpZA==","v":"CgdpbnZhbGlkOAFKBwoFCAEQ7AeCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
{"k":"AQAAAAAvbm90Z29vZA==","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
{"k":"AQAAAABi","v":"CisKBwoFCAEQ6QcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/At"}
{"k":"AQAAAABj","v":"CisKBwoFCAEQ6gcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcCi0KBwoFCAEQ6gcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtGAE="}
{"k":"AQAAAABk","v":"CisKBwoFCAEQ6wcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCi0KBwoFCAEQ6wcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="}
{"k":"AQAAAABpbnZhbGlk","v":"Ci0KBwoFCAEQ7AcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="}
{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHy9ub3Rnb29k","v":"AAAAAA=="}
{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2E=","v":"AAAAAA=="}
{"k":"BgAAAAAAAAAA","v":"VXBkYXRlU2NoZW1hMHRvMw=="}
{"k":"BwAAAAAAAAAA","v":""}
{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="}
{"k":"BwAAAAMAAAAA","v":"R5QD/970lrX15bycChUiHnAHMWRQn6MHYa9jCU9vlFw="}
{"k":"CQAAAAA=","v":"CicIAjACigEg//////////////////////////////////////////8KJwgFMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAKKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLQolCAGKASBHlAP/3vSWtfXlvJwKFSIecAcxZFCfowdhr2MJT2+UXBCyn6iaw4HGlxU="}
lib/db/testdata/v0.14.48-ignoredfiles.db.jsons
Vendored
-15
View File
@@ -1,15 +0,0 @@
{"k":"AAAAAAAAAAABYmFy","v":"CgNiYXJKBwoFCAEQ6QdQAg=="}
{"k":"AAAAAAAAAAABZm9v","v":"CgNmb284AUoHCgUIARDoB1AB"}
{"k":"AAAAAAAAAAACYmF6","v":"CgNiYXo4AUoHCgUIKhDoBw=="}
{"k":"AAAAAAAAAAACcXV1eA==","v":"CgRxdXV4SgcKBQgqEOoH"}
{"k":"AQAAAABiYXI=","v":"CisKBwoFCAEQ6QcSIP//////////////////////////////////////////"}
{"k":"AQAAAABiYXo=","v":"Ci0KBwoFCCoQ6AcSICoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAE="}
{"k":"AQAAAABmb28=","v":"Ci0KBwoFCAEQ6AcSIP//////////////////////////////////////////GAE="}
{"k":"AQAAAABxdXV4","v":"CisKBwoFCCoQ6gcSICoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
{"k":"BgAAAAAAAAAA","v":"dGVzdA=="}
{"k":"BwAAAAAAAAAA","v":""}
{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
{"k":"BwAAAAIAAAAA","v":"KgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}
{"k":"CQAAAAA=","v":"CicIATACigEg//////////////////////////////////////////8KJwgCMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAGKASAqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABCoudnCuM+vlxU="}
{"k":"CwAAAAAAAAAAAAAAAQ==","v":"AAAAAAAAAAABZm9v"}
{"k":"CwAAAAAAAAAAAAAAAg==","v":"AAAAAAAAAAABYmFy"}
lib/db/testdata/v1.4.0-updateTo10.json
Vendored
-24
View File
@@ -1,24 +0,0 @@
{"k":"AAAAAAAAAAABYQ==","v":"CgFhMAFKBwoFCAEQ6AdQAQ=="}
{"k":"AAAAAAAAAAABYg==","v":"CgFiSgcKBQgBEOgHUAKCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gkgEgC8XkepY1E4woWwAAyi81YItXr5CMuwY6mfvf2iLupTo="}
{"k":"AAAAAAAAAAABYw==","v":"CgFjMAFKBwoFCAEQ6AdQAw=="}
{"k":"AAAAAAAAAAACYQ==","v":"CgFhMAFKBwoFCAEQ6AdQAQ=="}
{"k":"AAAAAAAAAAACYg==","v":"CgFiMAFKFQoFCAEQ6AcKDAi5vtz687f5kQIQAVACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIJIBIAvF5HqWNROMKFsAAMovNWCLV6+QjLsGOpn739oi7qU6"}
{"k":"AAAAAAAAAAACYw==","v":"CgFjShUKBQgBEOgHCgwIub7c+vO3+ZECEAFQA4IBIhogAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh+SASBjDc0pZsQzZpESVEi7sltP9BKknHMtssirwbhYG9cQ3Q=="}
{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
{"k":"AQAAAABi","v":"CjkKFQoFCAEQ6AcKDAi5vtz687f5kQIQARIgAiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0KKwoHCgUIARDoBxIg//////////////////////////////////////////8="}
{"k":"AQAAAABj","v":"CjkKFQoFCAEQ6AcKDAi5vtz687f5kQIQARIgAiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0KKwoHCgUIARDoBxIg//////////////////////////////////////////8="}
{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2I=","v":"AAAAAA=="}
{"k":"AgAAAAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIGI=","v":"AAAAAQ=="}
{"k":"BgAAAAAAAAAA","v":"dGVzdA=="}
{"k":"BwAAAAAAAAAA","v":""}
{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="}
{"k":"CQAAAAA=","v":"CikIASACMAOKASD//////////////////////////////////////////wonCAEgAooBIPj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4CikIASACMAOKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLRDE7Jrik+mdhhY="}
{"k":"CmRiTWluU3luY3RoaW5nVmVyc2lvbg==","v":"djEuNC4w"}
{"k":"CmRiVmVyc2lvbg==","v":"AAAAAAAAAAk="}
{"k":"Cmxhc3RJbmRpcmVjdEdDVGltZQ==","v":"AAAAAF6yy/Q="}
{"k":"CwAAAAAAAAAAAAAAAQ==","v":"AAAAAAAAAAABYQ=="}
{"k":"CwAAAAAAAAAAAAAAAg==","v":"AAAAAAAAAAABYg=="}
{"k":"CwAAAAAAAAAAAAAAAw==","v":"AAAAAAAAAAABYw=="}
{"k":"DAAAAABi","v":""}
{"k":"DAAAAABj","v":""}
lib/db/transactions.go
+112 -105
View File
@@ -11,10 +11,15 @@ import (
"errors"
"fmt"
"google.golang.org/protobuf/proto"
"github.com/syncthing/syncthing/internal/gen/bep"
"github.com/syncthing/syncthing/internal/gen/dbproto"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sliceutil"
)
var (
@@ -63,48 +68,49 @@ func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool,
if err != nil || !ok {
return protocol.FileInfo{}, false, err
}
return f.(protocol.FileInfo), true, nil
return f, true, nil
}
func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileIntf, bool, error) {
func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileInfo, bool, error) {
bs, err := t.Get(key)
if backend.IsNotFound(err) {
return nil, false, nil
return protocol.FileInfo{}, false, nil
}
if err != nil {
return nil, false, err
return protocol.FileInfo{}, false, err
}
f, err := t.unmarshalTrunc(bs, trunc)
if backend.IsNotFound(err) {
return nil, false, nil
return protocol.FileInfo{}, false, nil
}
if err != nil {
return nil, false, err
return protocol.FileInfo{}, false, err
}
return f, true, nil
}
func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileIntf, error) {
func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileInfo, error) {
if trunc {
var tf FileInfoTruncated
err := tf.Unmarshal(bs)
var bfi dbproto.FileInfoTruncated
err := proto.Unmarshal(bs, &bfi)
if err != nil {
return nil, err
return protocol.FileInfo{}, err
}
if err := t.fillTruncated(&tf); err != nil {
return nil, err
if err := t.fillTruncated(&bfi); err != nil {
return protocol.FileInfo{}, err
}
return tf, nil
return protocol.FileInfoFromDBTruncated(&bfi), nil
}
var fi protocol.FileInfo
if err := fi.Unmarshal(bs); err != nil {
return nil, err
var bfi bep.FileInfo
err := proto.Unmarshal(bs, &bfi)
if err != nil {
return protocol.FileInfo{}, err
}
if err := t.fillFileInfo(&fi); err != nil {
return nil, err
if err := t.fillFileInfo(&bfi); err != nil {
return protocol.FileInfo{}, err
}
return fi, nil
return protocol.FileInfoFromDB(&bfi), nil
}
type blocksIndirectionError struct {
@@ -121,7 +127,7 @@ func (e *blocksIndirectionError) Unwrap() error {
// fillFileInfo follows the (possible) indirection of blocks and version
// vector and fills it out.
func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
func (t readOnlyTransaction) fillFileInfo(fi *bep.FileInfo) error {
var key []byte
if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
@@ -131,8 +137,8 @@ func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
if err != nil {
return &blocksIndirectionError{err}
}
var bl BlockList
if err := bl.Unmarshal(bs); err != nil {
var bl dbproto.BlockList
if err := proto.Unmarshal(bs, &bl); err != nil {
return err
}
fi.Blocks = bl.Blocks
@@ -144,11 +150,11 @@ func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
if err != nil {
return fmt.Errorf("filling Version: %w", err)
}
var v protocol.Vector
if err := v.Unmarshal(bs); err != nil {
var v bep.Vector
if err := proto.Unmarshal(bs, &v); err != nil {
return err
}
fi.Version = v
fi.Version = &v
}
return nil
@@ -156,7 +162,7 @@ func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
// fillTruncated follows the (possible) indirection of version vector and
// fills it.
func (t readOnlyTransaction) fillTruncated(fi *FileInfoTruncated) error {
func (t readOnlyTransaction) fillTruncated(fi *dbproto.FileInfoTruncated) error {
var key []byte
if len(fi.VersionHash) == 0 {
@@ -168,73 +174,72 @@ func (t readOnlyTransaction) fillTruncated(fi *FileInfoTruncated) error {
if err != nil {
return err
}
var v protocol.Vector
if err := v.Unmarshal(bs); err != nil {
var v bep.Vector
if err := proto.Unmarshal(bs, &v); err != nil {
return err
}
fi.Version = v
fi.Version = &v
return nil
}
func (t readOnlyTransaction) getGlobalVersions(keyBuf, folder, file []byte) (VersionList, error) {
func (t readOnlyTransaction) getGlobalVersions(keyBuf, folder, file []byte) (*dbproto.VersionList, error) {
var err error
keyBuf, err = t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
if err != nil {
return VersionList{}, err
return nil, err
}
return t.getGlobalVersionsByKey(keyBuf)
}
func (t readOnlyTransaction) getGlobalVersionsByKey(key []byte) (VersionList, error) {
func (t readOnlyTransaction) getGlobalVersionsByKey(key []byte) (*dbproto.VersionList, error) {
bs, err := t.Get(key)
if err != nil {
return VersionList{}, err
return nil, err
}
var vl VersionList
if err := vl.Unmarshal(bs); err != nil {
return VersionList{}, err
var vl dbproto.VersionList
if err := proto.Unmarshal(bs, &vl); err != nil {
return nil, err
}
return vl, nil
return &vl, nil
}
func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, protocol.FileIntf, bool, error) {
func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, protocol.FileInfo, bool, error) {
vl, err := t.getGlobalVersions(keyBuf, folder, file)
if backend.IsNotFound(err) {
return keyBuf, nil, false, nil
return keyBuf, protocol.FileInfo{}, false, nil
} else if err != nil {
return nil, nil, false, err
return nil, protocol.FileInfo{}, false, err
}
var fi protocol.FileIntf
keyBuf, fi, err = t.getGlobalFromVersionList(keyBuf, folder, file, truncate, vl)
keyBuf, fi, err := t.getGlobalFromVersionList(keyBuf, folder, file, truncate, vl)
return keyBuf, fi, true, err
}
func (t readOnlyTransaction) getGlobalFromVersionList(keyBuf, folder, file []byte, truncate bool, vl VersionList) ([]byte, protocol.FileIntf, error) {
fv, ok := vl.GetGlobal()
func (t readOnlyTransaction) getGlobalFromVersionList(keyBuf, folder, file []byte, truncate bool, vl *dbproto.VersionList) ([]byte, protocol.FileInfo, error) {
fv, ok := vlGetGlobal(vl)
if !ok {
return keyBuf, nil, errEmptyGlobal
return keyBuf, protocol.FileInfo{}, errEmptyGlobal
}
keyBuf, fi, err := t.getGlobalFromFileVersion(keyBuf, folder, file, truncate, fv)
return keyBuf, fi, err
}
func (t readOnlyTransaction) getGlobalFromFileVersion(keyBuf, folder, file []byte, truncate bool, fv FileVersion) ([]byte, protocol.FileIntf, error) {
dev, ok := fv.FirstDevice()
func (t readOnlyTransaction) getGlobalFromFileVersion(keyBuf, folder, file []byte, truncate bool, fv *dbproto.FileVersion) ([]byte, protocol.FileInfo, error) {
dev, ok := fvFirstDevice(fv)
if !ok {
return keyBuf, nil, errEmptyFileVersion
return keyBuf, protocol.FileInfo{}, errEmptyFileVersion
}
keyBuf, err := t.keyer.GenerateDeviceFileKey(keyBuf, folder, dev, file)
if err != nil {
return keyBuf, nil, err
return keyBuf, protocol.FileInfo{}, err
}
fi, ok, err := t.getFileTrunc(keyBuf, truncate)
if err != nil {
return keyBuf, nil, err
return keyBuf, protocol.FileInfo{}, err
}
if !ok {
return keyBuf, nil, errEntryFromGlobalMissing
return keyBuf, protocol.FileInfo{}, errEntryFromGlobalMissing
}
return keyBuf, fi, nil
}
@@ -357,13 +362,13 @@ func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, f
return nil
}
var vl VersionList
if err := vl.Unmarshal(dbi.Value()); err != nil {
var vl dbproto.VersionList
if err := proto.Unmarshal(dbi.Value(), &vl); err != nil {
return err
}
var f protocol.FileIntf
dk, f, err = t.getGlobalFromVersionList(dk, folder, name, truncate, vl)
var f protocol.FileInfo
dk, f, err = t.getGlobalFromVersionList(dk, folder, name, truncate, &vl)
if err != nil {
return err
}
@@ -432,7 +437,7 @@ func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.Devi
return nil, err
}
fv, ok := vl.GetGlobal()
fv, ok := vlGetGlobal(vl)
if !ok {
return nil, nil
}
@@ -472,32 +477,32 @@ func (t *readOnlyTransaction) withNeedIteratingGlobal(folder, device []byte, tru
return err
}
for dbi.Next() {
var vl VersionList
if err := vl.Unmarshal(dbi.Value()); err != nil {
var vl dbproto.VersionList
if err := proto.Unmarshal(dbi.Value(), &vl); err != nil {
return err
}
globalFV, ok := vl.GetGlobal()
globalFV, ok := vlGetGlobal(&vl)
if !ok {
return errEmptyGlobal
}
haveFV, have := vl.Get(device)
haveFV, have := vlGet(&vl, device)
if !Need(globalFV, have, haveFV.Version) {
if !Need(globalFV, have, protocol.VectorFromWire(haveFV.Version)) {
continue
}
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
var gf protocol.FileIntf
var gf protocol.FileInfo
dk, gf, err = t.getGlobalFromFileVersion(dk, folder, name, truncate, globalFV)
if err != nil {
return err
}
if shouldDebug() {
if globalDev, ok := globalFV.FirstDevice(); ok {
if globalDev, ok := fvFirstDevice(globalFV); ok {
globalID, _ := protocol.DeviceIDFromBytes(globalDev)
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v haveDeleted=%v globalV=%v globalDeleted=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, haveFV.Deleted, gf.FileVersion(), globalFV.Deleted, globalID)
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v haveDeleted=%v globalV=%v globalDeleted=%v globalDev=%v", folder, devID, name, have, fvIsInvalid(haveFV), haveFV.Version, haveFV.Deleted, gf.FileVersion(), globalFV.Deleted, globalID)
}
}
if !fn(gf) {
@@ -519,7 +524,7 @@ func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Ite
defer dbi.Release()
var keyBuf []byte
var f protocol.FileIntf
var f protocol.FileInfo
var ok bool
for dbi.Next() {
keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
@@ -589,7 +594,8 @@ func (t readWriteTransaction) putFile(fkey []byte, fi protocol.FileInfo) error {
bkey = t.keyer.GenerateBlockListKey(bkey, fi.BlocksHash)
if _, err := t.Get(bkey); backend.IsNotFound(err) {
// Marshal the block list and save it
blocksBs := mustMarshal(&BlockList{Blocks: fi.Blocks})
blocks := sliceutil.Map(fi.Blocks, protocol.BlockInfo.ToWire)
blocksBs := mustMarshal(&dbproto.BlockList{Blocks: blocks})
if err := t.Put(bkey, blocksBs); err != nil {
return err
}
@@ -605,7 +611,7 @@ func (t readWriteTransaction) putFile(fkey []byte, fi protocol.FileInfo) error {
bkey = t.keyer.GenerateVersionKey(bkey, fi.VersionHash)
if _, err := t.Get(bkey); backend.IsNotFound(err) {
// Marshal the version vector and save it
versionBs := mustMarshal(&fi.Version)
versionBs := mustMarshal(fi.Version.ToWire())
if err := t.Put(bkey, versionBs); err != nil {
return err
}
@@ -619,7 +625,7 @@ func (t readWriteTransaction) putFile(fkey []byte, fi protocol.FileInfo) error {
t.indirectionTracker.recordIndirectionHashesForFile(&fi)
fiBs := mustMarshal(&fi)
fiBs := mustMarshal(fi.ToWire(true))
return t.Put(fkey, fiBs)
}
@@ -638,8 +644,11 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
if err != nil && !backend.IsNotFound(err) {
return nil, err
}
if fl == nil {
fl = &dbproto.VersionList{}
}
globalFV, oldGlobalFV, removedFV, haveOldGlobal, haveRemoved, globalChanged, err := fl.update(folder, device, file, t.readOnlyTransaction)
globalFV, oldGlobalFV, removedFV, haveOldGlobal, haveRemoved, globalChanged, err := vlUpdate(fl, folder, device, file, t.readOnlyTransaction)
if err != nil {
return nil, err
}
@@ -647,20 +656,20 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
name := []byte(file.Name)
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
if err := t.Put(gk, mustMarshal(fl)); err != nil {
return nil, err
}
// Only load those from db if actually needed
var gotGlobal, gotOldGlobal bool
var global, oldGlobal protocol.FileIntf
var global, oldGlobal protocol.FileInfo
// Check the need of the device that was updated
// Must happen before updating global meta: If this is the first
// item from this device, it will be initialized with the global state.
needBefore := haveOldGlobal && Need(oldGlobalFV, haveRemoved, removedFV.Version)
needBefore := haveOldGlobal && Need(oldGlobalFV, haveRemoved, protocol.VectorFromWire(removedFV.GetVersion()))
needNow := Need(globalFV, true, file.Version)
if needBefore {
if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil {
@@ -709,7 +718,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
// Add the new global to the global size counter
if !gotGlobal {
if globalFV.Version.Equal(file.Version) {
if protocol.VectorFromWire(globalFV.Version).Equal(file.Version) {
// The inserted file is the global file
global = file
} else {
@@ -718,15 +727,15 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
return nil, err
}
}
gotGlobal = true
}
meta.addFile(protocol.GlobalDeviceID, global)
// check for local (if not already done before)
if !bytes.Equal(device, protocol.LocalDeviceID[:]) {
localFV, haveLocal := fl.Get(protocol.LocalDeviceID[:])
needBefore := haveOldGlobal && Need(oldGlobalFV, haveLocal, localFV.Version)
needNow := Need(globalFV, haveLocal, localFV.Version)
localFV, haveLocal := vlGet(fl, protocol.LocalDeviceID[:])
localVersion := protocol.VectorFromWire(localFV.Version)
needBefore := haveOldGlobal && Need(oldGlobalFV, haveLocal, localVersion)
needNow := Need(globalFV, haveLocal, localVersion)
if needBefore {
meta.removeNeeded(protocol.LocalDeviceID, oldGlobal)
if !needNow {
@@ -750,11 +759,12 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
// Already handled above
continue
}
fv, have := fl.Get(dev[:])
if haveOldGlobal && Need(oldGlobalFV, have, fv.Version) {
fv, have := vlGet(fl, dev[:])
fvVersion := protocol.VectorFromWire(fv.Version)
if haveOldGlobal && Need(oldGlobalFV, have, fvVersion) {
meta.removeNeeded(dev, oldGlobal)
}
if Need(globalFV, have, fv.Version) {
if Need(globalFV, have, fvVersion) {
meta.addNeeded(dev, global)
}
}
@@ -778,11 +788,12 @@ func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add b
return keyBuf, err
}
func Need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool {
func Need(global *dbproto.FileVersion, haveLocal bool, localVersion protocol.Vector) bool {
// We never need an invalid file or a file without a valid version (just
// another way of expressing "invalid", really, until we fix that
// part...).
if global.IsInvalid() || global.Version.IsEmpty() {
globalVersion := protocol.VectorFromWire(global.Version)
if fvIsInvalid(global) || globalVersion.IsEmpty() {
return false
}
// We don't need a deleted file if we don't have it.
@@ -790,7 +801,7 @@ func Need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool
return false
}
// We don't need the global file if we already have the same version.
if haveLocal && localVersion.GreaterEqual(global.Version) {
if haveLocal && localVersion.GreaterEqual(globalVersion) {
return false
}
return true
@@ -816,8 +827,8 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
return nil, err
}
oldGlobalFV, haveOldGlobal := fl.GetGlobal()
oldGlobalFV = oldGlobalFV.copy()
oldGlobalFV, haveOldGlobal := vlGetGlobal(fl)
oldGlobalFV = fvCopy(oldGlobalFV)
if !haveOldGlobal {
// Shouldn't ever happen, but doesn't hurt to handle.
@@ -825,18 +836,18 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
return keyBuf, t.Delete(gk)
}
removedFV, haveRemoved, globalChanged := fl.pop(device)
removedFV, haveRemoved, globalChanged := vlPop(fl, device)
if !haveRemoved {
// There is no version for the given device
return keyBuf, nil
}
var global protocol.FileIntf
var global protocol.FileInfo
var gotGlobal bool
globalFV, haveGlobal := fl.GetGlobal()
globalFV, haveGlobal := vlGetGlobal(fl)
// Add potential needs of the removed device
if haveGlobal && !globalFV.IsInvalid() && Need(globalFV, false, protocol.Vector{}) && !Need(oldGlobalFV, haveRemoved, removedFV.Version) {
if haveGlobal && !fvIsInvalid(globalFV) && Need(globalFV, false, protocol.Vector{}) && !Need(oldGlobalFV, haveRemoved, protocol.VectorFromWire(removedFV.Version)) {
keyBuf, global, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl)
if err != nil {
return nil, err
@@ -853,13 +864,13 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
// Global hasn't changed, abort early
if !globalChanged {
l.Debugf("new global after remove: %v", fl)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
if err := t.Put(gk, mustMarshal(fl)); err != nil {
return nil, err
}
return keyBuf, nil
}
var oldGlobal protocol.FileIntf
var oldGlobal protocol.FileInfo
keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, file, true, oldGlobalFV)
if err != nil {
return nil, err
@@ -868,11 +879,12 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
// Remove potential device needs
shouldRemoveNeed := func(dev protocol.DeviceID) bool {
fv, have := fl.Get(dev[:])
if !Need(oldGlobalFV, have, fv.Version) {
fv, have := vlGet(fl, dev[:])
fvVersion := protocol.VectorFromWire(fv.Version)
if !Need(oldGlobalFV, have, fvVersion) {
return false // Didn't need it before
}
return !haveGlobal || !Need(globalFV, have, fv.Version)
return !haveGlobal || !Need(globalFV, have, fvVersion)
}
if shouldRemoveNeed(protocol.LocalDeviceID) {
meta.removeNeeded(protocol.LocalDeviceID, oldGlobal)
@@ -890,7 +902,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
}
// Nothing left, i.e. nothing to add to the global counter below.
if fl.Empty() {
if len(fl.Versions) == 0 {
if err := t.Delete(gk); err != nil {
return nil, err
}
@@ -907,7 +919,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file
meta.addFile(protocol.GlobalDeviceID, global)
l.Debugf(`new global for "%s" after remove: %v`, file, fl)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
if err := t.Put(gk, mustMarshal(fl)); err != nil {
return nil, err
}
@@ -935,7 +947,7 @@ func (t readWriteTransaction) deleteKeyPrefixMatching(prefix []byte, match func(
return dbi.Error()
}
func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error {
func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(device []byte, f protocol.FileInfo) bool) error {
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, nil, nil)
if err != nil {
return err
@@ -957,11 +969,10 @@ func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(dev
continue
}
intf, err := t.unmarshalTrunc(dbi.Value(), true)
f, err := t.unmarshalTrunc(dbi.Value(), true)
if err != nil {
return err
}
f := intf.(FileInfoTruncated)
switch f.Name {
case "", ".", "..", "/": // A few obviously invalid filenames
@@ -988,12 +999,8 @@ func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(dev
return dbi.Error()
}
type marshaller interface {
Marshal() ([]byte, error)
}
func mustMarshal(f marshaller) []byte {
bs, err := f.Marshal()
func mustMarshal(f proto.Message) []byte {
bs, err := proto.Marshal(f)
if err != nil {
panic(err)
}
lib/db/util.go
+5 -2
View File
@@ -6,7 +6,10 @@
package db
import "github.com/syncthing/syncthing/lib/protocol"
import (
"github.com/syncthing/syncthing/lib/protocol"
"google.golang.org/protobuf/proto"
)
// How many files to send in each Index/IndexUpdate message.
const (
@@ -44,7 +47,7 @@ func (b *FileInfoBatch) Append(f protocol.FileInfo) {
b.infos = make([]protocol.FileInfo, 0, MaxBatchSizeFiles)
}
b.infos = append(b.infos, f)
b.size += f.ProtoSize()
b.size += proto.Size(f.ToWire(true))
}
func (b *FileInfoBatch) Full() bool {