felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

lib/db: Refactor to use global list by version (fixes #6372) (#6638)

Browse Source 
Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei
2020-05-30 09:50:23 +02:00
committed by GitHub
co-authored by Jakob Borg
parent 1eea076f5c
commit 1f8e6c55f6
21 changed files with 1540 additions and 504 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/stindex/idxck.go
+26 -17
View File
@@ -219,10 +219,10 @@ func idxck(ldb backend.Backend) (success bool) {
fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
success = false
}
for i, fv := range vl.Versions {
dev, ok := deviceToIDs[string(fv.Device)]
checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
dev, ok := deviceToIDs[string(device)]
if !ok {
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
success = false
}
fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
@@ -235,14 +235,26 @@ func idxck(ldb backend.Backend) (success bool) {
if fi.VersionHash != nil {
fiv = versions[string(fi.VersionHash)]
}
if !fiv.Equal(fv.Version) {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
if !fiv.Equal(version) {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
success = false
}
if fi.IsInvalid() != fv.Invalid {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
if fi.IsInvalid() != invalid {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
success = false
}
if fi.IsDeleted() != deleted {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
success = false
}
}
for i, fv := range vl.RawVersions {
for _, device := range fv.Devices {
checkGlobal(i, device, fv.Version, false, fv.Deleted)
}
for _, device := range fv.InvalidDevices {
checkGlobal(i, device, fv.Version, true, fv.Deleted)
}
}
// If we need this file we should have a need entry for it. False
@@ -251,7 +263,9 @@ func idxck(ldb backend.Backend) (success bool) {
if needsLocally(vl) {
_, ok := needs[gk]
if !ok {
dev := deviceToIDs[string(vl.Versions[0].Device)]
fv, _ := vl.GetGlobal()
devB, _ := fv.FirstDevice()
dev := deviceToIDs[string(devB)]
fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
if !fi.IsDeleted() && !fi.IsIgnored() {
fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
@@ -319,15 +333,10 @@ func idxck(ldb backend.Backend) (success bool) {
}
func needsLocally(vl db.VersionList) bool {
var lv *protocol.Vector
for _, fv := range vl.Versions {
if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
lv = &fv.Version
break
}
}
if lv == nil {
fv, ok := vl.Get(protocol.LocalDeviceID[:])
if !ok {
return true // proviosinally, it looks like we need the file
}
return !lv.GreaterEqual(vl.Versions[0].Version)
gfv, _ := vl.GetGlobal() // Can't not have a global if we got something above
return !fv.Version.GreaterEqual(gfv.Version)
}