Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors. Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei
Jakob Borg
+288
-70
@@ -12,7 +12,6 @@ package db
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
@@ -196,98 +195,317 @@ func (vl VersionList) String() string {
|
||||
var b bytes.Buffer
|
||||
var id protocol.DeviceID
|
||||
b.WriteString("{")
|
||||
for i, v := range vl.Versions {
|
||||
for i, v := range vl.RawVersions {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
copy(id[:], v.Device)
|
||||
fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
|
||||
fmt.Fprintf(&b, "{%v, {", v.Version)
|
||||
for j, dev := range v.Devices {
|
||||
if j > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
copy(id[:], dev)
|
||||
fmt.Fprint(&b, id.Short())
|
||||
}
|
||||
b.WriteString("}, {")
|
||||
for j, dev := range v.InvalidDevices {
|
||||
if j > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
copy(id[:], dev)
|
||||
fmt.Fprint(&b, id.Short())
|
||||
}
|
||||
fmt.Fprint(&b, "}}")
|
||||
}
|
||||
b.WriteString("}")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// update brings the VersionList up to date with file. It returns the updated
|
||||
// VersionList, a potentially removed old FileVersion and its index, as well as
|
||||
// the index where the new FileVersion was inserted.
|
||||
func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int, err error) {
|
||||
vl, removedFV, removedAt = vl.pop(device)
|
||||
|
||||
nv := FileVersion{
|
||||
Device: device,
|
||||
Version: file.Version,
|
||||
Invalid: file.IsInvalid(),
|
||||
Deleted: file.IsDeleted(),
|
||||
}
|
||||
i := 0
|
||||
if nv.Invalid {
|
||||
i = sort.Search(len(vl.Versions), func(j int) bool {
|
||||
return vl.Versions[j].Invalid
|
||||
})
|
||||
}
|
||||
for ; i < len(vl.Versions); i++ {
|
||||
switch vl.Versions[i].Version.Compare(file.Version) {
|
||||
case protocol.Equal:
|
||||
fallthrough
|
||||
|
||||
case protocol.Lesser:
|
||||
// The version at this point in the list is equal to or lesser
|
||||
// ("older") than us. We insert ourselves in front of it.
|
||||
vl = vl.insertAt(i, nv)
|
||||
return vl, removedFV, removedAt, i, nil
|
||||
|
||||
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
|
||||
// The version at this point is in conflict with us. We must pull
|
||||
// the actual file metadata to determine who wins. If we win, we
|
||||
// insert ourselves in front of the loser here. (The "Lesser" and
|
||||
// "Greater" in the condition above is just based on the device
|
||||
// IDs in the version vector, which is not the only thing we use
|
||||
// to determine the winner.)
|
||||
//
|
||||
// A surprise missing file entry here is counted as a win for us.
|
||||
if of, ok, err := t.getFile(folder, vl.Versions[i].Device, []byte(file.Name)); err != nil {
|
||||
return vl, removedFV, removedAt, i, err
|
||||
} else if !ok || file.WinsConflict(of) {
|
||||
vl = vl.insertAt(i, nv)
|
||||
return vl, removedFV, removedAt, i, nil
|
||||
}
|
||||
}
|
||||
// VersionList, a device that has the global/newest version, a device that previously
|
||||
// had the global/newest version, a boolean indicating if the global version has
|
||||
// changed and if any error occurred (only possible in db interaction).
|
||||
func (vl *VersionList) update(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (FileVersion, FileVersion, FileVersion, bool, bool, bool, error) {
|
||||
if len(vl.RawVersions) == 0 {
|
||||
nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
|
||||
vl.RawVersions = append(vl.RawVersions, nv)
|
||||
return nv, FileVersion{}, FileVersion{}, false, false, true, nil
|
||||
}
|
||||
|
||||
// We didn't find a position for an insert above, so append to the end.
|
||||
vl.Versions = append(vl.Versions, nv)
|
||||
// Get the current global (before updating)
|
||||
oldFV, haveOldGlobal := vl.GetGlobal()
|
||||
|
||||
return vl, removedFV, removedAt, len(vl.Versions) - 1, nil
|
||||
// Remove ourselves first
|
||||
removedFV, haveRemoved, _, err := vl.pop(folder, device, []byte(file.FileName()), t)
|
||||
if err == nil {
|
||||
// Find position and insert the file
|
||||
err = vl.insert(folder, device, file, t)
|
||||
}
|
||||
if err != nil {
|
||||
return FileVersion{}, FileVersion{}, FileVersion{}, false, false, false, err
|
||||
}
|
||||
|
||||
newFV, _ := vl.GetGlobal() // We just inserted something above, can't be empty
|
||||
|
||||
if !haveOldGlobal {
|
||||
return newFV, FileVersion{}, removedFV, false, haveRemoved, true, nil
|
||||
}
|
||||
|
||||
globalChanged := true
|
||||
if oldFV.IsInvalid() == newFV.IsInvalid() && oldFV.Version.Equal(newFV.Version) {
|
||||
globalChanged = false
|
||||
}
|
||||
|
||||
return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
|
||||
}
|
||||
|
||||
func (vl VersionList) insertAt(i int, v FileVersion) VersionList {
|
||||
vl.Versions = append(vl.Versions, FileVersion{})
|
||||
copy(vl.Versions[i+1:], vl.Versions[i:])
|
||||
vl.Versions[i] = v
|
||||
return vl
|
||||
func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) error {
|
||||
var added bool
|
||||
var err error
|
||||
i := 0
|
||||
for ; i < len(vl.RawVersions); i++ {
|
||||
// Insert our new version
|
||||
added, err = vl.checkInsertAt(i, folder, device, file, t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if added {
|
||||
break
|
||||
}
|
||||
}
|
||||
if i == len(vl.RawVersions) {
|
||||
// Append to the end
|
||||
vl.RawVersions = append(vl.RawVersions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vl *VersionList) insertAt(i int, v FileVersion) {
|
||||
vl.RawVersions = append(vl.RawVersions, FileVersion{})
|
||||
copy(vl.RawVersions[i+1:], vl.RawVersions[i:])
|
||||
vl.RawVersions[i] = v
|
||||
}
|
||||
|
||||
// pop returns the VersionList without the entry for the given device, as well
|
||||
// as the removed FileVersion and the position, where that FileVersion was.
|
||||
// If there is no FileVersion for the given device, the position is -1.
|
||||
func (vl VersionList) pop(device []byte) (VersionList, FileVersion, int) {
|
||||
for i, v := range vl.Versions {
|
||||
if bytes.Equal(v.Device, device) {
|
||||
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
|
||||
return vl, v, i
|
||||
}
|
||||
// as the removed FileVersion, whether it was found/removed at all and whether
|
||||
// the global changed in the process.
|
||||
func (vl *VersionList) pop(folder, device, name []byte, t readOnlyTransaction) (FileVersion, bool, bool, error) {
|
||||
invDevice, i, j, ok := vl.findDevice(device)
|
||||
if !ok {
|
||||
return FileVersion{}, false, false, nil
|
||||
}
|
||||
return vl, FileVersion{}, -1
|
||||
globalPos := vl.findGlobal()
|
||||
|
||||
if vl.RawVersions[i].deviceCount() == 1 {
|
||||
fv := vl.RawVersions[i]
|
||||
vl.popVersionAt(i)
|
||||
return fv, true, globalPos == i, nil
|
||||
}
|
||||
|
||||
if invDevice {
|
||||
vl.RawVersions[i].InvalidDevices = popDeviceAt(vl.RawVersions[i].InvalidDevices, j)
|
||||
} else {
|
||||
vl.RawVersions[i].Devices = popDeviceAt(vl.RawVersions[i].Devices, j)
|
||||
}
|
||||
// If the last valid device of the previous global was removed above,
|
||||
// the next entry is now the global entry (unless all entries are invalid).
|
||||
if len(vl.RawVersions[i].Devices) == 0 && globalPos == i {
|
||||
return vl.RawVersions[i], true, globalPos == vl.findGlobal(), nil
|
||||
}
|
||||
return vl.RawVersions[i], true, false, nil
|
||||
}
|
||||
|
||||
func (vl VersionList) Get(device []byte) (FileVersion, bool) {
|
||||
for _, v := range vl.Versions {
|
||||
if bytes.Equal(v.Device, device) {
|
||||
return v, true
|
||||
// Get returns a FileVersion that contains the given device and whether it has
|
||||
// been found at all.
|
||||
func (vl *VersionList) Get(device []byte) (FileVersion, bool) {
|
||||
_, i, _, ok := vl.findDevice(device)
|
||||
if !ok {
|
||||
return FileVersion{}, false
|
||||
}
|
||||
return vl.RawVersions[i], true
|
||||
}
|
||||
|
||||
// GetGlobal returns the current global FileVersion. The returned FileVersion
|
||||
// may be invalid, if all FileVersions are invalid. Returns false only if
|
||||
// VersionList is empty.
|
||||
func (vl *VersionList) GetGlobal() (FileVersion, bool) {
|
||||
i := vl.findGlobal()
|
||||
if i == -1 {
|
||||
return FileVersion{}, false
|
||||
}
|
||||
return vl.RawVersions[i], true
|
||||
}
|
||||
|
||||
func (vl *VersionList) Empty() bool {
|
||||
return len(vl.RawVersions) == 0
|
||||
}
|
||||
|
||||
// findGlobal returns the first version that isn't invalid, or if all versions are
|
||||
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
|
||||
func (vl *VersionList) findGlobal() int {
|
||||
for i, fv := range vl.RawVersions {
|
||||
if !fv.IsInvalid() {
|
||||
return i
|
||||
}
|
||||
}
|
||||
if len(vl.RawVersions) == 0 {
|
||||
return -1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
return FileVersion{}, false
|
||||
// findDevices returns whether the device is in InvalidVersions or Versions and
|
||||
// in InvalidDevices or Devices (true for invalid), the positions in the version
|
||||
// and device slices and whether it has been found at all.
|
||||
func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) {
|
||||
for i, v := range vl.RawVersions {
|
||||
if j := deviceIndex(v.Devices, device); j != -1 {
|
||||
return false, i, j, true
|
||||
}
|
||||
if j := deviceIndex(v.InvalidDevices, device); j != -1 {
|
||||
return true, i, j, true
|
||||
}
|
||||
}
|
||||
return false, -1, -1, false
|
||||
}
|
||||
|
||||
func (vl *VersionList) popVersion(version protocol.Vector) (FileVersion, bool) {
|
||||
i := vl.versionIndex(version)
|
||||
if i == -1 {
|
||||
return FileVersion{}, false
|
||||
}
|
||||
fv := vl.RawVersions[i]
|
||||
vl.popVersionAt(i)
|
||||
return fv, true
|
||||
}
|
||||
|
||||
func (vl *VersionList) versionIndex(version protocol.Vector) int {
|
||||
for i, v := range vl.RawVersions {
|
||||
if version.Equal(v.Version) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (vl *VersionList) popVersionAt(i int) {
|
||||
vl.RawVersions = append(vl.RawVersions[:i], vl.RawVersions[i+1:]...)
|
||||
}
|
||||
|
||||
// checkInsertAt determines if the given device and associated file should be
|
||||
// inserted into the FileVersion at position i or into a new FileVersion at
|
||||
// position i.
|
||||
func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
|
||||
ordering := vl.RawVersions[i].Version.Compare(file.FileVersion())
|
||||
if ordering == protocol.Equal {
|
||||
if !file.IsInvalid() {
|
||||
vl.RawVersions[i].Devices = append(vl.RawVersions[i].Devices, device)
|
||||
} else {
|
||||
vl.RawVersions[i].InvalidDevices = append(vl.RawVersions[i].InvalidDevices, device)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
existingDevice, _ := vl.RawVersions[i].FirstDevice()
|
||||
insert, err := shouldInsertBefore(ordering, folder, existingDevice, vl.RawVersions[i].IsInvalid(), file, t)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if insert {
|
||||
vl.insertAt(i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// shouldInsertBefore determines whether the file comes before an existing
|
||||
// entry, given the version ordering (existing compared to new one), existing
|
||||
// device and if the existing version is invalid.
|
||||
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
|
||||
switch ordering {
|
||||
case protocol.Lesser:
|
||||
// The version at this point in the list is lesser
|
||||
// ("older") than us. We insert ourselves in front of it.
|
||||
return true, nil
|
||||
|
||||
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
|
||||
// The version in conflict with us.
|
||||
// Check if we can shortcut due to one being invalid.
|
||||
if existingInvalid != file.IsInvalid() {
|
||||
return existingInvalid, nil
|
||||
}
|
||||
// We must pull the actual file metadata to determine who wins.
|
||||
// If we win, we insert ourselves in front of the loser here.
|
||||
// (The "Lesser" and "Greater" in the condition above is just
|
||||
// based on the device IDs in the version vector, which is not
|
||||
// the only thing we use to determine the winner.)
|
||||
of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// A surprise missing file entry here is counted as a win for us.
|
||||
if !ok {
|
||||
return true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if protocol.WinsConflict(file, of) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func deviceIndex(devices [][]byte, device []byte) int {
|
||||
for i, dev := range devices {
|
||||
if bytes.Equal(device, dev) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func popDeviceAt(devices [][]byte, i int) [][]byte {
|
||||
return append(devices[:i], devices[i+1:]...)
|
||||
}
|
||||
|
||||
func popDevice(devices [][]byte, device []byte) ([][]byte, bool) {
|
||||
i := deviceIndex(devices, device)
|
||||
if i == -1 {
|
||||
return devices, false
|
||||
}
|
||||
return popDeviceAt(devices, i), true
|
||||
}
|
||||
|
||||
func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) FileVersion {
|
||||
fv := FileVersion{
|
||||
Version: version,
|
||||
Deleted: deleted,
|
||||
}
|
||||
if invalid {
|
||||
fv.InvalidDevices = [][]byte{device}
|
||||
} else {
|
||||
fv.Devices = [][]byte{device}
|
||||
}
|
||||
return fv
|
||||
}
|
||||
|
||||
func (fv FileVersion) FirstDevice() ([]byte, bool) {
|
||||
if len(fv.Devices) != 0 {
|
||||
return fv.Devices[0], true
|
||||
}
|
||||
if len(fv.InvalidDevices) != 0 {
|
||||
return fv.InvalidDevices[0], true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (fv FileVersion) IsInvalid() bool {
|
||||
return len(fv.Devices) == 0
|
||||
}
|
||||
|
||||
func (fv FileVersion) deviceCount() int {
|
||||
return len(fv.Devices) + len(fv.InvalidDevices)
|
||||
}
|
||||
|
||||
type fileList []protocol.FileInfo
|
||||
|
||||
Reference in New Issue
Block a user