felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Transactionalize db.FileSet (fixes #5952) (#6239)

Browse Source
This commit is contained in:
Simon Frei
2020-01-21 18:23:08 +01:00
committed by Jakob Borg
parent d62a0cf692
commit 08f0e125ef
23 changed files with 957 additions and 891 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/model.go
+35 -202
View File
@@ -90,21 +90,14 @@ type Model interface {
GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)
LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated
DBSnapshot(folder string) (*db.Snapshot, error)
NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
FolderProgressBytesCompleted(folder string) int64
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability
GlobalSize(folder string) db.Counts
LocalSize(folder string) db.Counts
NeedSize(folder string) db.Counts
ReceiveOnlyChangedSize(folder string) db.Counts
CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool)
Completion(device protocol.DeviceID, folder string) FolderCompletion
ConnectionStats() map[string]interface{}
DeviceStatistics() (map[string]stats.DeviceStatistics, error)
@@ -764,7 +757,10 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
}
tot := rf.GlobalSize().Bytes
snap := rf.Snapshot()
defer snap.Release()
tot := snap.GlobalSize().Bytes
if tot == 0 {
// Folder is empty, so we have all of it
return FolderCompletion{
@@ -777,7 +773,7 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
m.pmut.RUnlock()
var need, items, fileNeed, downloaded, deletes int64
rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
snap.WithNeedTruncated(device, func(f db.FileIntf) bool {
ft := f.(db.FileInfoTruncated)
// If the file is deleted, we account it only in the deleted column.
@@ -822,84 +818,19 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
}
}
func addSizeOfFile(s *db.Counts, f db.FileIntf) {
switch {
case f.IsDeleted():
s.Deleted++
case f.IsDirectory():
s.Directories++
case f.IsSymlink():
s.Symlinks++
default:
s.Files++
}
s.Bytes += f.FileSize()
}
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *model) GlobalSize(folder string) db.Counts {
// DBSnapshot returns a snapshot of the database content relevant to the given folder.
func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if ok {
return rf.GlobalSize()
if !ok {
return nil, errFolderMissing
}
return db.Counts{}
return rf.Snapshot(), nil
}
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *model) LocalSize(folder string) db.Counts {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if ok {
return rf.LocalSize()
}
return db.Counts{}
}
// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *model) ReceiveOnlyChangedSize(folder string) db.Counts {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if ok {
return rf.ReceiveOnlyChangedSize()
}
return db.Counts{}
}
// NeedSize returns the number of currently needed files and their total size
// minus the amount that has already been downloaded.
func (m *model) NeedSize(folder string) db.Counts {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
cfg := m.folderCfgs[folder]
m.fmut.RUnlock()
var result db.Counts
if ok {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
addSizeOfFile(&result, f)
return true
})
}
result.Bytes -= m.progressEmitter.BytesCompleted(folder)
// This may happen if we are in progress of pulling files that were
// deleted globally after the pull started.
if result.Bytes < 0 {
result.Bytes = 0
}
l.Debugf("%v NeedSize(%q): %v", m, folder, result)
return result
func (m *model) FolderProgressBytesCompleted(folder string) int64 {
return m.progressEmitter.BytesCompleted(folder)
}
// NeedFolderFiles returns paginated list of currently needed files in
@@ -915,6 +846,8 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
return nil, nil, nil
}
snap := rf.Snapshot()
defer snap.Release()
var progress, queued, rest []db.FileInfoTruncated
var seen map[string]struct{}
@@ -929,14 +862,14 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))
for i, name := range progressNames {
if f, ok := rf.GetGlobalTruncated(name); ok {
if f, ok := snap.GetGlobalTruncated(name); ok {
progress[i] = f
seen[name] = struct{}{}
}
}
for i, name := range queuedNames {
if f, ok := rf.GetGlobalTruncated(name); ok {
if f, ok := snap.GetGlobalTruncated(name); ok {
queued[i] = f
seen[name] = struct{}{}
}
@@ -950,7 +883,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
}
rest = make([]db.FileInfoTruncated, 0, perpage)
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
@@ -970,77 +903,6 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
return progress, queued, rest
}
// LocalChangedFiles returns a paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
fcfg := m.folderCfgs[folder]
m.fmut.RUnlock()
if !ok {
return nil
}
if fcfg.Type != config.FolderTypeReceiveOnly {
return nil
}
if rf.ReceiveOnlyChangedSize().TotalItems() == 0 {
return nil
}
files := make([]db.FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if !f.IsReceiveOnlyChanged() {
return true
}
if skip > 0 {
skip--
return true
}
ft := f.(db.FileInfoTruncated)
files = append(files, ft)
get--
return get > 0
})
return files
}
// RemoteNeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
m.fmut.RLock()
m.pmut.RLock()
err := m.checkDeviceFolderConnectedLocked(device, folder)
rf := m.folderFiles[folder]
m.pmut.RUnlock()
m.fmut.RUnlock()
if err != nil {
return nil, err
}
files := make([]db.FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
if skip > 0 {
skip--
return true
}
files = append(files, f.(db.FileInfoTruncated))
get--
return get > 0
})
return files, nil
}
// Index is called when a new device is connected and we receive their full index.
// Implements the protocol.Model interface.
func (m *model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) error {
@@ -1752,7 +1614,9 @@ func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo
if !ok {
return protocol.FileInfo{}, false
}
return fs.Get(protocol.LocalDeviceID, file)
snap := fs.Snapshot()
defer snap.Release()
return snap.Get(protocol.LocalDeviceID, file)
}
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) {
@@ -1762,7 +1626,9 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
if !ok {
return protocol.FileInfo{}, false
}
return fs.GetGlobal(file)
snap := fs.Snapshot()
defer snap.Release()
return snap.GetGlobal(file)
}
// Connection returns the current connection for device, and a boolean whether a connection was found.
@@ -2075,7 +1941,9 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
var err error
var f protocol.FileInfo
s.fset.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
snap := s.fset.Snapshot()
defer snap.Release()
snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
if err = batch.flushIfFull(); err != nil {
return false
}
@@ -2347,45 +2215,6 @@ func (m *model) Revert(folder string) {
runner.Revert()
}
// CurrentSequence returns the change version for the given folder.
// This is guaranteed to increment if the contents of the local folder has
// changed.
func (m *model) CurrentSequence(folder string) (int64, bool) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
// The folder might not exist, since this can be called with a user
// specified folder name from the REST interface.
return 0, false
}
return fs.Sequence(protocol.LocalDeviceID), true
}
// RemoteSequence returns the change version for the given folder, as
// sent by remote peers. This is guaranteed to increment if the contents of
// the remote or global folder has changed.
func (m *model) RemoteSequence(folder string) (int64, bool) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
cfg := m.folderCfgs[folder]
m.fmut.RUnlock()
if !ok {
// The folder might not exist, since this can be called with a user
// specified folder name from the REST interface.
return 0, false
}
var ver int64
for _, device := range cfg.Devices {
ver += fs.Sequence(device.DeviceID)
}
return ver, true
}
func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} {
m.fmut.RLock()
files, ok := m.folderFiles[folder]
@@ -2402,7 +2231,9 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
prefix = prefix + sep
}
files.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
snap := files.Snapshot()
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
// Don't include the prefix itself.
@@ -2514,8 +2345,10 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc
}
var availabilities []Availability
snap := fs.Snapshot()
defer snap.Release()
next:
for _, device := range fs.Availability(file.Name) {
for _, device := range snap.Availability(file.Name) {
for _, pausedFolder := range m.remotePausedFolders[device] {
if pausedFolder == folder {
continue next