lib: Return error from db.FileSet.Snapshot (fixes #7419, ref #5907) (#7424)

This commit is contained in:
Simon Frei
2021-03-07 13:43:22 +01:00
committed by GitHub
parent c1d06d9501
commit 310fba4c12
25 changed files with 601 additions and 344 deletions
+80 -38
View File
@@ -27,6 +27,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
@@ -87,7 +88,7 @@ type syncRequest struct {
}
type puller interface {
pull() bool // true when successful and should not be retried
pull() (bool, error) // true when successful and should not be retried
}
func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *byteSemaphore, ver versioner.Versioner) folder {
@@ -164,16 +165,20 @@ func (f *folder) Serve(ctx context.Context) error {
initialCompleted := f.initialScanFinished
for {
var err error
select {
case <-f.ctx.Done():
close(f.done)
return nil
case <-f.pullScheduled:
f.pull()
_, err = f.pull()
case <-f.pullFailTimer.C:
if !f.pull() && f.pullPause < 60*f.pullBasePause() {
var success bool
success, err = f.pull()
if (err != nil || !success) && f.pullPause < 60*f.pullBasePause() {
// Back off from retrying to pull
f.pullPause *= 2
}
@@ -181,18 +186,19 @@ func (f *folder) Serve(ctx context.Context) error {
case <-initialCompleted:
// Initial scan has completed, we should do a pull
initialCompleted = nil // never hit this case again
f.pull()
_, err = f.pull()
case <-f.forcedRescanRequested:
f.handleForcedRescans()
err = f.handleForcedRescans()
case <-f.scanTimer.C:
l.Debugln(f, "Scanning due to timer")
f.scanTimerFired()
err = f.scanTimerFired()
case req := <-f.doInSyncChan:
l.Debugln(f, "Running something due to request")
req.err <- req.fn()
err = req.fn()
req.err <- err
case next := <-f.scanDelay:
l.Debugln(f, "Delaying scan")
@@ -200,16 +206,23 @@ func (f *folder) Serve(ctx context.Context) error {
case fsEvents := <-f.watchChan:
l.Debugln(f, "Scan due to watcher")
f.scanSubdirs(fsEvents)
err = f.scanSubdirs(fsEvents)
case <-f.restartWatchChan:
l.Debugln(f, "Restart watcher")
f.restartWatch()
err = f.restartWatch()
case <-f.versionCleanupTimer.C:
l.Debugln(f, "Doing version cleanup")
f.versionCleanupTimerFired()
}
if err != nil {
if svcutil.IsFatal(err) {
return err
}
f.setError(err)
}
}
}
@@ -307,7 +320,7 @@ func (f *folder) getHealthErrorWithoutIgnores() error {
return nil
}
func (f *folder) pull() (success bool) {
func (f *folder) pull() (success bool, err error) {
f.pullFailTimer.Stop()
select {
case <-f.pullFailTimer.C:
@@ -318,7 +331,7 @@ func (f *folder) pull() (success bool) {
case <-f.initialScanFinished:
default:
// Once the initial scan finished, a pull will be scheduled
return true
return true, nil
}
defer func() {
@@ -330,7 +343,10 @@ func (f *folder) pull() (success bool) {
// If there is nothing to do, don't even enter sync-waiting state.
abort := true
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return false, err
}
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
abort = false
return false
@@ -341,16 +357,16 @@ func (f *folder) pull() (success bool) {
f.errorsMut.Lock()
f.pullErrors = nil
f.errorsMut.Unlock()
return true
return true, nil
}
// Abort early (before acquiring a token) if there's a folder error
err := f.getHealthErrorWithoutIgnores()
f.setError(err)
err = f.getHealthErrorWithoutIgnores()
if err != nil {
l.Debugln("Skipping pull of", f.Description(), "due to folder error:", err)
return false
return false, err
}
f.setError(nil)
// Send only folder doesn't do any io, it only checks for out-of-sync
// items that differ in metadata and updates those.
@@ -358,8 +374,7 @@ func (f *folder) pull() (success bool) {
f.setState(FolderSyncWaiting)
if err := f.ioLimiter.takeWithContext(f.ctx, 1); err != nil {
f.setError(err)
return true
return true, err
}
defer f.ioLimiter.give(1)
}
@@ -374,23 +389,23 @@ func (f *folder) pull() (success bool) {
}
}()
err = f.getHealthErrorAndLoadIgnores()
f.setError(err)
if err != nil {
l.Debugln("Skipping pull of", f.Description(), "due to folder error:", err)
return false
return false, err
}
success = f.puller.pull()
success, err = f.puller.pull()
if success {
return true
if success && err == nil {
return true, nil
}
// Pulling failed, try again later.
delay := f.pullPause + time.Since(startTime)
l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), util.NiceDurationString(delay))
f.pullFailTimer.Reset(delay)
return false
return false, err
}
func (f *folder) scanSubdirs(subDirs []string) error {
@@ -399,7 +414,6 @@ func (f *folder) scanSubdirs(subDirs []string) error {
oldHash := f.ignores.Hash()
err := f.getHealthErrorAndLoadIgnores()
f.setError(err)
if err != nil {
// If there is a health error we set it as the folder error. We do not
// clear the folder error if there is no health error, as there might be
@@ -407,6 +421,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// we do not use the CheckHealth() convenience function here.
return err
}
f.setError(nil)
// Check on the way out if the ignore patterns changed as part of scanning
// this folder. If they did we should schedule a pull of the folder so that
@@ -443,7 +458,10 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// Clean the list of subitems to ensure that we start at a known
// directory, and don't scan subdirectories of things we've already
// scanned.
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return err
}
subDirs = unifySubs(subDirs, func(file string) bool {
_, ok := snap.Get(protocol.LocalDeviceID, file)
return ok
@@ -560,7 +578,10 @@ func (f *folder) scanSubdirsBatchAppendFunc(batch *fileInfoBatch) batchAppendFun
func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *fileInfoBatch, batchAppend batchAppendFunc) (int, error) {
changes := 0
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return changes, err
}
defer snap.Release()
// If we return early e.g. due to a folder health error, the scan needs
@@ -629,7 +650,10 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *fileInfoB
var toIgnore []db.FileInfoTruncated
ignoredParent := ""
changes := 0
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return 0, err
}
defer snap.Release()
for _, sub := range subDirs {
@@ -821,7 +845,7 @@ func (f *folder) findRename(snap *db.Snapshot, file protocol.FileInfo, alreadyUs
return nf, found
}
func (f *folder) scanTimerFired() {
func (f *folder) scanTimerFired() error {
err := f.scanSubdirs(nil)
select {
@@ -836,6 +860,8 @@ func (f *folder) scanTimerFired() {
}
f.Reschedule()
return err
}
func (f *folder) versionCleanupTimerFired() {
@@ -884,10 +910,10 @@ func (f *folder) scheduleWatchRestart() {
// restartWatch should only ever be called synchronously. If you want to use
// this asynchronously, you should probably use scheduleWatchRestart instead.
func (f *folder) restartWatch() {
func (f *folder) restartWatch() error {
f.stopWatch()
f.startWatch()
f.scanSubdirs(nil)
return f.scanSubdirs(nil)
}
// startWatch should only ever be called synchronously. If you want to use
@@ -1166,7 +1192,7 @@ func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events
}
}
func (f *folder) handleForcedRescans() {
func (f *folder) handleForcedRescans() error {
f.forcedRescanPathsMut.Lock()
paths := make([]string, 0, len(f.forcedRescanPaths))
for path := range f.forcedRescanPaths {
@@ -1175,7 +1201,7 @@ func (f *folder) handleForcedRescans() {
f.forcedRescanPaths = make(map[string]struct{})
f.forcedRescanPathsMut.Unlock()
if len(paths) == 0 {
return
return nil
}
batch := newFileInfoBatch(func(fs []protocol.FileInfo) error {
@@ -1183,10 +1209,16 @@ func (f *folder) handleForcedRescans() {
return nil
})
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
for _, path := range paths {
_ = batch.flushIfFull()
if err := batch.flushIfFull(); err != nil {
return err
}
fi, ok := snap.Get(protocol.LocalDeviceID, path)
if !ok {
@@ -1196,11 +1228,21 @@ func (f *folder) handleForcedRescans() {
batch.append(fi)
}
snap.Release()
if err = batch.flush(); err != nil {
return err
}
_ = batch.flush()
return f.scanSubdirs(paths)
}
_ = f.scanSubdirs(paths)
// dbSnapshots gets a snapshot from the fileset, and wraps any error
// in a svcutil.FatalErr.
func (f *folder) dbSnapshot() (*db.Snapshot, error) {
snap, err := f.fset.Snapshot()
if err != nil {
return nil, svcutil.AsFatalErr(err, svcutil.ExitError)
}
return snap, nil
}
// The exists function is expected to return true for all known paths
+8 -7
View File
@@ -32,10 +32,10 @@ func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.M
}
func (f *receiveEncryptedFolder) Revert() {
f.doInSync(func() error { f.revert(); return nil })
f.doInSync(f.revert)
}
func (f *receiveEncryptedFolder) revert() {
func (f *receiveEncryptedFolder) revert() error {
l.Infof("Reverting unexpected items in folder %v (receive-encrypted)", f.Description())
f.setState(FolderScanning)
@@ -46,7 +46,10 @@ func (f *receiveEncryptedFolder) revert() {
return nil
})
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
var iterErr error
var dirs []string
@@ -85,12 +88,10 @@ func (f *receiveEncryptedFolder) revert() {
f.revertHandleDirs(dirs, snap)
if iterErr == nil {
iterErr = batch.flush()
}
if iterErr != nil {
l.Infoln("Failed to delete unexpected items:", iterErr)
return iterErr
}
return batch.flush()
}
func (f *receiveEncryptedFolder) revertHandleDirs(dirs []string, snap *db.Snapshot) {
+8 -3
View File
@@ -63,10 +63,10 @@ func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
}
func (f *receiveOnlyFolder) Revert() {
f.doInSync(func() error { f.revert(); return nil })
f.doInSync(f.revert)
}
func (f *receiveOnlyFolder) revert() {
func (f *receiveOnlyFolder) revert() error {
l.Infof("Reverting folder %v", f.Description)
f.setState(FolderScanning)
@@ -84,7 +84,10 @@ func (f *receiveOnlyFolder) revert() {
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
fi := intf.(protocol.FileInfo)
@@ -161,6 +164,8 @@ func (f *receiveOnlyFolder) revert() {
// pull by itself. Make sure we schedule one so that we start
// downloading files.
f.SchedulePull()
return nil
}
// deleteQueue handles deletes by delegating to a handler and queuing
+1 -1
View File
@@ -384,7 +384,7 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) {
// Do the same changes on the remote
files := make([]protocol.FileInfo, 0, 2)
snap := f.fset.Snapshot()
snap := fsetSnapshot(t, f.fset)
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
if n := fi.FileName(); n != file && n != knownFile {
return true
+13 -6
View File
@@ -36,11 +36,14 @@ func (f *sendOnlyFolder) PullErrors() []FileError {
}
// pull checks need for files that only differ by metadata (no changes on disk)
func (f *sendOnlyFolder) pull() bool {
func (f *sendOnlyFolder) pull() (bool, error) {
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return false, err
}
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
@@ -83,14 +86,14 @@ func (f *sendOnlyFolder) pull() bool {
f.updateLocalsFromPulling(batch)
}
return true
return true, nil
}
func (f *sendOnlyFolder) Override() {
f.doInSync(func() error { f.override(); return nil })
f.doInSync(f.override)
}
func (f *sendOnlyFolder) override() {
func (f *sendOnlyFolder) override() error {
l.Infoln("Overriding global state on folder", f.Description())
f.setState(FolderScanning)
@@ -98,7 +101,10 @@ func (f *sendOnlyFolder) override() {
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return err
}
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
need := fi.(protocol.FileInfo)
@@ -130,4 +136,5 @@ func (f *sendOnlyFolder) override() {
if len(batch) > 0 {
f.updateLocalsFromScanning(batch)
}
return nil
}
+21 -14
View File
@@ -156,7 +156,7 @@ func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
// pull returns true if it manages to get all needed items from peers, i.e. get
// the device in sync with the global state.
func (f *sendReceiveFolder) pull() bool {
func (f *sendReceiveFolder) pull() (bool, error) {
l.Debugf("%v pulling", f)
scanChan := make(chan string)
@@ -173,10 +173,11 @@ func (f *sendReceiveFolder) pull() bool {
f.pullErrors = nil
f.errorsMut.Unlock()
var err error
for tries := 0; tries < maxPullerIterations; tries++ {
select {
case <-f.ctx.Done():
return false
return false, f.ctx.Err()
default:
}
@@ -184,7 +185,10 @@ func (f *sendReceiveFolder) pull() bool {
// it to FolderSyncing during the last iteration.
f.setState(FolderSyncPreparing)
changed = f.pullerIteration(scanChan)
changed, err = f.pullerIteration(scanChan)
if err != nil {
return false, err
}
l.Debugln(f, "changed", changed, "on try", tries+1)
@@ -219,19 +223,22 @@ func (f *sendReceiveFolder) pull() bool {
})
}
return changed == 0
return changed == 0, nil
}
// pullerIteration runs a single puller iteration for the given folder and
// returns the number items that should have been synced (even those that
// might have failed). One puller iteration handles all files currently
// flagged as needed in the folder.
func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) (int, error) {
f.errorsMut.Lock()
f.tempPullErrors = make(map[string]string)
f.errorsMut.Unlock()
snap := f.fset.Snapshot()
snap, err := f.dbSnapshot()
if err != nil {
return 0, err
}
defer snap.Release()
pullChan := make(chan pullBlockState)
@@ -265,7 +272,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
pullWg.Add(1)
go func() {
// pullerRoutine finishes when pullChan is closed
f.pullerRoutine(pullChan, finisherChan)
f.pullerRoutine(snap, pullChan, finisherChan)
pullWg.Done()
}()
@@ -300,7 +307,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
f.queue.Reset()
return changed
return changed, nil
}
func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
@@ -582,7 +589,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
// that don't result in a conflict.
case err == nil && !info.IsDir():
// Check that it is what we have in the database.
curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
err = errors.Wrap(err, "handling dir")
f.newPullError(file.Name, err)
@@ -766,7 +773,7 @@ func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, s
return err
}
// Check that it is what we have in the database.
curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
return err
}
@@ -1427,7 +1434,7 @@ func (f *sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) e
return nil
}
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
func (f *sendReceiveFolder) pullerRoutine(snap *db.Snapshot, in <-chan pullBlockState, out chan<- *sharedPullerState) {
requestLimiter := newByteSemaphore(f.PullerMaxPendingKiB * 1024)
wg := sync.NewWaitGroup()
@@ -1458,13 +1465,13 @@ func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *
defer wg.Done()
defer requestLimiter.give(bytes)
f.pullBlock(state, out)
f.pullBlock(state, snap, out)
}()
}
wg.Wait()
}
func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPullerState) {
func (f *sendReceiveFolder) pullBlock(state pullBlockState, snap *db.Snapshot, out chan<- *sharedPullerState) {
// Get an fd to the temporary file. Technically we don't need it until
// after fetching the block, but if we run into an error here there is
// no point in issuing the request to the network.
@@ -1483,7 +1490,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
}
var lastError error
candidates := f.model.Availability(f.folderID, state.file, state.block)
candidates := f.model.availabilityInSnapshot(f.FolderConfiguration, snap, state.file, state.block)
for {
select {
case <-f.ctx.Done():
+25 -24
View File
@@ -135,7 +135,7 @@ func TestHandleFile(t *testing.T) {
copyChan := make(chan copyBlocksState, 1)
f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
// Receive the results
toCopy := <-copyChan
@@ -181,7 +181,7 @@ func TestHandleFileWithTemp(t *testing.T) {
copyChan := make(chan copyBlocksState, 1)
f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
// Receive the results
toCopy := <-copyChan
@@ -245,7 +245,7 @@ func TestCopierFinder(t *testing.T) {
go f.copierRoutine(copyChan, pullChan, finisherChan)
defer close(copyChan)
f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
f.handleFile(requiredFile, fsetSnapshot(t, f.fset), copyChan)
timeout := time.After(10 * time.Second)
pulls := make([]pullBlockState, 4)
@@ -379,7 +379,7 @@ func TestWeakHash(t *testing.T) {
// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
fo.WeakHashThresholdPct = 101
fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan)
fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan)
var pulls []pullBlockState
timeout := time.After(10 * time.Second)
@@ -408,7 +408,7 @@ func TestWeakHash(t *testing.T) {
// Test 2 - using weak hash, expectPulls blocks pulled.
fo.WeakHashThresholdPct = -1
fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan)
fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan)
pulls = pulls[:0]
for len(pulls) < expectPulls {
@@ -489,7 +489,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState, 1)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := f.fset.Snapshot()
snap := fsetSnapshot(t, f.fset)
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
@@ -589,13 +589,13 @@ func TestDeregisterOnFailInPull(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := f.fset.Snapshot()
snap := fsetSnapshot(t, f.fset)
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
pullWg := sync.NewWaitGroup()
pullWg.Add(1)
go func() {
f.pullerRoutine(pullChan, finisherBufferChan)
f.pullerRoutine(snap, pullChan, finisherBufferChan)
pullWg.Done()
}()
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
@@ -696,7 +696,7 @@ func TestIssue3164(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(file, f.fset.Snapshot(), dbUpdateChan, make(chan string))
f.deleteDir(file, fsetSnapshot(t, f.fset), dbUpdateChan, make(chan string))
if _, err := ffs.Stat("issue3164"); !fs.IsNotExist(err) {
t.Fatal(err)
@@ -828,7 +828,7 @@ func TestCopyOwner(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string)
defer close(dbUpdateChan)
f.handleDir(dir, f.fset.Snapshot(), dbUpdateChan, scanChan)
f.handleDir(dir, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // empty the channel for later
case toScan := <-scanChan:
@@ -858,7 +858,7 @@ func TestCopyOwner(t *testing.T) {
// but it's the way data is passed around. When the database update
// comes the finisher is done.
snap := f.fset.Snapshot()
snap := fsetSnapshot(t, f.fset)
finisherChan := make(chan *sharedPullerState)
copierChan, copyWg := startCopier(f, nil, finisherChan)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, nil)
@@ -926,7 +926,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleDir(file, f.fset.Snapshot(), dbUpdateChan, scanChan)
f.handleDir(file, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@@ -959,7 +959,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleSymlink(file, f.fset.Snapshot(), dbUpdateChan, scanChan)
f.handleSymlink(file, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@@ -1001,7 +1001,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
fi.Version = fi.Version.Update(device1.Short())
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteFile(fi, f.fset.Snapshot(), dbUpdateChan, scanChan)
f.deleteFile(fi, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
select {
case f := <-scanChan:
t.Fatalf("Received %v on scanChan", f)
@@ -1031,7 +1031,7 @@ func TestPullCtxCancel(t *testing.T) {
var cancel context.CancelFunc
f.ctx, cancel = context.WithCancel(context.Background())
go f.pullerRoutine(pullChan, finisherChan)
go f.pullerRoutine(fsetSnapshot(t, f.fset), pullChan, finisherChan)
defer close(pullChan)
emptyState := func() pullBlockState {
@@ -1077,7 +1077,7 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(fi, f.fset.Snapshot(), dbUpdateChan, scanChan)
f.deleteDir(fi, fsetSnapshot(t, f.fset), dbUpdateChan, scanChan)
if _, err := ffs.Stat(dir); fs.IsNotExist(err) {
t.Error("directory has been deleted")
@@ -1226,7 +1226,7 @@ func TestPullTempFileCaseConflict(t *testing.T) {
fd.Close()
}
f.handleFile(file, f.fset.Snapshot(), copyChan)
f.handleFile(file, fsetSnapshot(t, f.fset), copyChan)
cs := <-copyChan
if _, err := cs.tempFile(); err != nil {
@@ -1252,7 +1252,7 @@ func TestPullCaseOnlyRename(t *testing.T) {
must(t, f.scanSubdirs(nil))
cur, ok := m.CurrentFolderFile(f.ID, name)
cur, ok := m.testCurrentFolderFile(f.ID, name)
if !ok {
t.Fatal("file missing")
}
@@ -1266,7 +1266,7 @@ func TestPullCaseOnlyRename(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 2)
scanChan := make(chan string, 2)
snap := f.fset.Snapshot()
snap := fsetSnapshot(t, f.fset)
defer snap.Release()
if err := f.renameFile(cur, deleted, confl, snap, dbUpdateChan, scanChan); err != nil {
t.Error(err)
@@ -1293,7 +1293,7 @@ func TestPullSymlinkOverExistingWindows(t *testing.T) {
must(t, f.scanSubdirs(nil))
file, ok := m.CurrentFolderFile(f.ID, name)
file, ok := m.testCurrentFolderFile(f.ID, name)
if !ok {
t.Fatal("file missing")
}
@@ -1301,11 +1301,12 @@ func TestPullSymlinkOverExistingWindows(t *testing.T) {
scanChan := make(chan string)
changed := f.pullerIteration(scanChan)
changed, err := f.pullerIteration(scanChan)
must(t, err)
if changed != 1 {
t.Error("Expected one change in pull, got", changed)
}
if file, ok := m.CurrentFolderFile(f.ID, name); !ok {
if file, ok := m.testCurrentFolderFile(f.ID, name); !ok {
t.Error("symlink entry missing")
} else if !file.IsUnsupported() {
t.Error("symlink entry isn't marked as unsupported")
@@ -1341,7 +1342,7 @@ func TestPullDeleteCaseConflict(t *testing.T) {
t.Error("Missing db update for file")
}
snap := f.fset.Snapshot()
snap := fsetSnapshot(t, f.fset)
defer snap.Release()
f.deleteDir(fi, snap, dbUpdateChan, scanChan)
select {
@@ -1371,7 +1372,7 @@ func TestPullDeleteIgnoreChildDir(t *testing.T) {
scanChan := make(chan string, 2)
err := f.deleteDirOnDisk(parent, f.fset.Snapshot(), scanChan)
err := f.deleteDirOnDisk(parent, fsetSnapshot(t, f.fset), scanChan)
if err == nil {
t.Error("no error")
}
+10 -5
View File
@@ -96,7 +96,7 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
// For API backwards compatibility (SyncTrayzor needs it) an empty folder
// summary is returned for not running folders, an error might actually be
// more appropriate
if err != nil && err != ErrFolderPaused && err != errFolderNotRunning {
if err != nil && err != ErrFolderPaused && err != ErrFolderNotRunning {
return nil, err
}
@@ -348,9 +348,14 @@ func (c *folderSummaryService) sendSummary(ctx context.Context, folder string) {
// Get completion percentage of this folder for the
// remote device.
comp := c.model.Completion(devCfg.DeviceID, folder).Map()
comp["folder"] = folder
comp["device"] = devCfg.DeviceID.String()
c.evLogger.Log(events.FolderCompletion, comp)
comp, err := c.model.Completion(devCfg.DeviceID, folder)
if err != nil {
l.Debugf("Error getting completion for folder %v, device %v: %v", folder, devCfg.DeviceID, err)
continue
}
ev := comp.Map()
ev["folder"] = folder
ev["device"] = devCfg.DeviceID.String()
c.evLogger.Log(events.FolderCompletion, ev)
}
}
+4 -1
View File
@@ -131,7 +131,10 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
var err error
var f protocol.FileInfo
snap := s.fset.Snapshot()
snap, err := s.fset.Snapshot()
if err != nil {
return svcutil.AsFatalErr(err, svcutil.ExitError)
}
defer snap.Release()
previousWasDelete := false
snap.WithHaveSequence(s.prevSequence+1, func(fi protocol.FileIntf) bool {
+56 -36
View File
@@ -22,7 +22,7 @@ type Model struct {
arg1 protocol.Connection
arg2 protocol.Hello
}
AvailabilityStub func(string, protocol.FileInfo, protocol.BlockInfo) []model.Availability
AvailabilityStub func(string, protocol.FileInfo, protocol.BlockInfo) ([]model.Availability, error)
availabilityMutex sync.RWMutex
availabilityArgsForCall []struct {
arg1 string
@@ -31,9 +31,11 @@ type Model struct {
}
availabilityReturns struct {
result1 []model.Availability
result2 error
}
availabilityReturnsOnCall map[int]struct {
result1 []model.Availability
result2 error
}
BringToFrontStub func(string, string)
bringToFrontMutex sync.RWMutex
@@ -59,7 +61,7 @@ type Model struct {
clusterConfigReturnsOnCall map[int]struct {
result1 error
}
CompletionStub func(protocol.DeviceID, string) model.FolderCompletion
CompletionStub func(protocol.DeviceID, string) (model.FolderCompletion, error)
completionMutex sync.RWMutex
completionArgsForCall []struct {
arg1 protocol.DeviceID
@@ -67,9 +69,11 @@ type Model struct {
}
completionReturns struct {
result1 model.FolderCompletion
result2 error
}
completionReturnsOnCall map[int]struct {
result1 model.FolderCompletion
result2 error
}
ConnectionStub func(protocol.DeviceID) (protocol.Connection, bool)
connectionMutex sync.RWMutex
@@ -94,7 +98,7 @@ type Model struct {
connectionStatsReturnsOnCall map[int]struct {
result1 map[string]interface{}
}
CurrentFolderFileStub func(string, string) (protocol.FileInfo, bool)
CurrentFolderFileStub func(string, string) (protocol.FileInfo, bool, error)
currentFolderFileMutex sync.RWMutex
currentFolderFileArgsForCall []struct {
arg1 string
@@ -103,12 +107,14 @@ type Model struct {
currentFolderFileReturns struct {
result1 protocol.FileInfo
result2 bool
result3 error
}
currentFolderFileReturnsOnCall map[int]struct {
result1 protocol.FileInfo
result2 bool
result3 error
}
CurrentGlobalFileStub func(string, string) (protocol.FileInfo, bool)
CurrentGlobalFileStub func(string, string) (protocol.FileInfo, bool, error)
currentGlobalFileMutex sync.RWMutex
currentGlobalFileArgsForCall []struct {
arg1 string
@@ -117,10 +123,12 @@ type Model struct {
currentGlobalFileReturns struct {
result1 protocol.FileInfo
result2 bool
result3 error
}
currentGlobalFileReturnsOnCall map[int]struct {
result1 protocol.FileInfo
result2 bool
result3 error
}
CurrentIgnoresStub func(string) ([]string, []string, error)
currentIgnoresMutex sync.RWMutex
@@ -577,7 +585,7 @@ func (fake *Model) AddConnectionArgsForCall(i int) (protocol.Connection, protoco
return argsForCall.arg1, argsForCall.arg2
}
func (fake *Model) Availability(arg1 string, arg2 protocol.FileInfo, arg3 protocol.BlockInfo) []model.Availability {
func (fake *Model) Availability(arg1 string, arg2 protocol.FileInfo, arg3 protocol.BlockInfo) ([]model.Availability, error) {
fake.availabilityMutex.Lock()
ret, specificReturn := fake.availabilityReturnsOnCall[len(fake.availabilityArgsForCall)]
fake.availabilityArgsForCall = append(fake.availabilityArgsForCall, struct {
@@ -593,9 +601,9 @@ func (fake *Model) Availability(arg1 string, arg2 protocol.FileInfo, arg3 protoc
return stub(arg1, arg2, arg3)
}
if specificReturn {
return ret.result1
return ret.result1, ret.result2
}
return fakeReturns.result1
return fakeReturns.result1, fakeReturns.result2
}
func (fake *Model) AvailabilityCallCount() int {
@@ -604,7 +612,7 @@ func (fake *Model) AvailabilityCallCount() int {
return len(fake.availabilityArgsForCall)
}
func (fake *Model) AvailabilityCalls(stub func(string, protocol.FileInfo, protocol.BlockInfo) []model.Availability) {
func (fake *Model) AvailabilityCalls(stub func(string, protocol.FileInfo, protocol.BlockInfo) ([]model.Availability, error)) {
fake.availabilityMutex.Lock()
defer fake.availabilityMutex.Unlock()
fake.AvailabilityStub = stub
@@ -617,27 +625,30 @@ func (fake *Model) AvailabilityArgsForCall(i int) (string, protocol.FileInfo, pr
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
}
func (fake *Model) AvailabilityReturns(result1 []model.Availability) {
func (fake *Model) AvailabilityReturns(result1 []model.Availability, result2 error) {
fake.availabilityMutex.Lock()
defer fake.availabilityMutex.Unlock()
fake.AvailabilityStub = nil
fake.availabilityReturns = struct {
result1 []model.Availability
}{result1}
result2 error
}{result1, result2}
}
func (fake *Model) AvailabilityReturnsOnCall(i int, result1 []model.Availability) {
func (fake *Model) AvailabilityReturnsOnCall(i int, result1 []model.Availability, result2 error) {
fake.availabilityMutex.Lock()
defer fake.availabilityMutex.Unlock()
fake.AvailabilityStub = nil
if fake.availabilityReturnsOnCall == nil {
fake.availabilityReturnsOnCall = make(map[int]struct {
result1 []model.Availability
result2 error
})
}
fake.availabilityReturnsOnCall[i] = struct {
result1 []model.Availability
}{result1}
result2 error
}{result1, result2}
}
func (fake *Model) BringToFront(arg1 string, arg2 string) {
@@ -768,7 +779,7 @@ func (fake *Model) ClusterConfigReturnsOnCall(i int, result1 error) {
}{result1}
}
func (fake *Model) Completion(arg1 protocol.DeviceID, arg2 string) model.FolderCompletion {
func (fake *Model) Completion(arg1 protocol.DeviceID, arg2 string) (model.FolderCompletion, error) {
fake.completionMutex.Lock()
ret, specificReturn := fake.completionReturnsOnCall[len(fake.completionArgsForCall)]
fake.completionArgsForCall = append(fake.completionArgsForCall, struct {
@@ -783,9 +794,9 @@ func (fake *Model) Completion(arg1 protocol.DeviceID, arg2 string) model.FolderC
return stub(arg1, arg2)
}
if specificReturn {
return ret.result1
return ret.result1, ret.result2
}
return fakeReturns.result1
return fakeReturns.result1, fakeReturns.result2
}
func (fake *Model) CompletionCallCount() int {
@@ -794,7 +805,7 @@ func (fake *Model) CompletionCallCount() int {
return len(fake.completionArgsForCall)
}
func (fake *Model) CompletionCalls(stub func(protocol.DeviceID, string) model.FolderCompletion) {
func (fake *Model) CompletionCalls(stub func(protocol.DeviceID, string) (model.FolderCompletion, error)) {
fake.completionMutex.Lock()
defer fake.completionMutex.Unlock()
fake.CompletionStub = stub
@@ -807,27 +818,30 @@ func (fake *Model) CompletionArgsForCall(i int) (protocol.DeviceID, string) {
return argsForCall.arg1, argsForCall.arg2
}
func (fake *Model) CompletionReturns(result1 model.FolderCompletion) {
func (fake *Model) CompletionReturns(result1 model.FolderCompletion, result2 error) {
fake.completionMutex.Lock()
defer fake.completionMutex.Unlock()
fake.CompletionStub = nil
fake.completionReturns = struct {
result1 model.FolderCompletion
}{result1}
result2 error
}{result1, result2}
}
func (fake *Model) CompletionReturnsOnCall(i int, result1 model.FolderCompletion) {
func (fake *Model) CompletionReturnsOnCall(i int, result1 model.FolderCompletion, result2 error) {
fake.completionMutex.Lock()
defer fake.completionMutex.Unlock()
fake.CompletionStub = nil
if fake.completionReturnsOnCall == nil {
fake.completionReturnsOnCall = make(map[int]struct {
result1 model.FolderCompletion
result2 error
})
}
fake.completionReturnsOnCall[i] = struct {
result1 model.FolderCompletion
}{result1}
result2 error
}{result1, result2}
}
func (fake *Model) Connection(arg1 protocol.DeviceID) (protocol.Connection, bool) {
@@ -947,7 +961,7 @@ func (fake *Model) ConnectionStatsReturnsOnCall(i int, result1 map[string]interf
}{result1}
}
func (fake *Model) CurrentFolderFile(arg1 string, arg2 string) (protocol.FileInfo, bool) {
func (fake *Model) CurrentFolderFile(arg1 string, arg2 string) (protocol.FileInfo, bool, error) {
fake.currentFolderFileMutex.Lock()
ret, specificReturn := fake.currentFolderFileReturnsOnCall[len(fake.currentFolderFileArgsForCall)]
fake.currentFolderFileArgsForCall = append(fake.currentFolderFileArgsForCall, struct {
@@ -962,9 +976,9 @@ func (fake *Model) CurrentFolderFile(arg1 string, arg2 string) (protocol.FileInf
return stub(arg1, arg2)
}
if specificReturn {
return ret.result1, ret.result2
return ret.result1, ret.result2, ret.result3
}
return fakeReturns.result1, fakeReturns.result2
return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3
}
func (fake *Model) CurrentFolderFileCallCount() int {
@@ -973,7 +987,7 @@ func (fake *Model) CurrentFolderFileCallCount() int {
return len(fake.currentFolderFileArgsForCall)
}
func (fake *Model) CurrentFolderFileCalls(stub func(string, string) (protocol.FileInfo, bool)) {
func (fake *Model) CurrentFolderFileCalls(stub func(string, string) (protocol.FileInfo, bool, error)) {
fake.currentFolderFileMutex.Lock()
defer fake.currentFolderFileMutex.Unlock()
fake.CurrentFolderFileStub = stub
@@ -986,17 +1000,18 @@ func (fake *Model) CurrentFolderFileArgsForCall(i int) (string, string) {
return argsForCall.arg1, argsForCall.arg2
}
func (fake *Model) CurrentFolderFileReturns(result1 protocol.FileInfo, result2 bool) {
func (fake *Model) CurrentFolderFileReturns(result1 protocol.FileInfo, result2 bool, result3 error) {
fake.currentFolderFileMutex.Lock()
defer fake.currentFolderFileMutex.Unlock()
fake.CurrentFolderFileStub = nil
fake.currentFolderFileReturns = struct {
result1 protocol.FileInfo
result2 bool
}{result1, result2}
result3 error
}{result1, result2, result3}
}
func (fake *Model) CurrentFolderFileReturnsOnCall(i int, result1 protocol.FileInfo, result2 bool) {
func (fake *Model) CurrentFolderFileReturnsOnCall(i int, result1 protocol.FileInfo, result2 bool, result3 error) {
fake.currentFolderFileMutex.Lock()
defer fake.currentFolderFileMutex.Unlock()
fake.CurrentFolderFileStub = nil
@@ -1004,15 +1019,17 @@ func (fake *Model) CurrentFolderFileReturnsOnCall(i int, result1 protocol.FileIn
fake.currentFolderFileReturnsOnCall = make(map[int]struct {
result1 protocol.FileInfo
result2 bool
result3 error
})
}
fake.currentFolderFileReturnsOnCall[i] = struct {
result1 protocol.FileInfo
result2 bool
}{result1, result2}
result3 error
}{result1, result2, result3}
}
func (fake *Model) CurrentGlobalFile(arg1 string, arg2 string) (protocol.FileInfo, bool) {
func (fake *Model) CurrentGlobalFile(arg1 string, arg2 string) (protocol.FileInfo, bool, error) {
fake.currentGlobalFileMutex.Lock()
ret, specificReturn := fake.currentGlobalFileReturnsOnCall[len(fake.currentGlobalFileArgsForCall)]
fake.currentGlobalFileArgsForCall = append(fake.currentGlobalFileArgsForCall, struct {
@@ -1027,9 +1044,9 @@ func (fake *Model) CurrentGlobalFile(arg1 string, arg2 string) (protocol.FileInf
return stub(arg1, arg2)
}
if specificReturn {
return ret.result1, ret.result2
return ret.result1, ret.result2, ret.result3
}
return fakeReturns.result1, fakeReturns.result2
return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3
}
func (fake *Model) CurrentGlobalFileCallCount() int {
@@ -1038,7 +1055,7 @@ func (fake *Model) CurrentGlobalFileCallCount() int {
return len(fake.currentGlobalFileArgsForCall)
}
func (fake *Model) CurrentGlobalFileCalls(stub func(string, string) (protocol.FileInfo, bool)) {
func (fake *Model) CurrentGlobalFileCalls(stub func(string, string) (protocol.FileInfo, bool, error)) {
fake.currentGlobalFileMutex.Lock()
defer fake.currentGlobalFileMutex.Unlock()
fake.CurrentGlobalFileStub = stub
@@ -1051,17 +1068,18 @@ func (fake *Model) CurrentGlobalFileArgsForCall(i int) (string, string) {
return argsForCall.arg1, argsForCall.arg2
}
func (fake *Model) CurrentGlobalFileReturns(result1 protocol.FileInfo, result2 bool) {
func (fake *Model) CurrentGlobalFileReturns(result1 protocol.FileInfo, result2 bool, result3 error) {
fake.currentGlobalFileMutex.Lock()
defer fake.currentGlobalFileMutex.Unlock()
fake.CurrentGlobalFileStub = nil
fake.currentGlobalFileReturns = struct {
result1 protocol.FileInfo
result2 bool
}{result1, result2}
result3 error
}{result1, result2, result3}
}
func (fake *Model) CurrentGlobalFileReturnsOnCall(i int, result1 protocol.FileInfo, result2 bool) {
func (fake *Model) CurrentGlobalFileReturnsOnCall(i int, result1 protocol.FileInfo, result2 bool, result3 error) {
fake.currentGlobalFileMutex.Lock()
defer fake.currentGlobalFileMutex.Unlock()
fake.CurrentGlobalFileStub = nil
@@ -1069,12 +1087,14 @@ func (fake *Model) CurrentGlobalFileReturnsOnCall(i int, result1 protocol.FileIn
fake.currentGlobalFileReturnsOnCall = make(map[int]struct {
result1 protocol.FileInfo
result2 bool
result3 error
})
}
fake.currentGlobalFileReturnsOnCall[i] = struct {
result1 protocol.FileInfo
result2 bool
}{result1, result2}
result3 error
}{result1, result2, result3}
}
func (fake *Model) CurrentIgnores(arg1 string) ([]string, []string, error) {
+85 -46
View File
@@ -98,11 +98,11 @@ type Model interface {
LocalChangedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, error)
FolderProgressBytesCompleted(folder string) int64
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error)
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error)
Completion(device protocol.DeviceID, folder string) FolderCompletion
Completion(device protocol.DeviceID, folder string) (FolderCompletion, error)
ConnectionStats() map[string]interface{}
DeviceStatistics() (map[protocol.DeviceID]stats.DeviceStatistics, error)
FolderStatistics() (map[string]stats.FolderStatistics, error)
@@ -179,8 +179,8 @@ var (
errDeviceIgnored = errors.New("device is ignored")
errDeviceRemoved = errors.New("device has been removed")
ErrFolderPaused = errors.New("folder is paused")
errFolderNotRunning = errors.New("folder is not running")
errFolderMissing = errors.New("no such folder")
ErrFolderNotRunning = errors.New("folder is not running")
ErrFolderMissing = errors.New("no such folder")
errNetworkNotAllowed = errors.New("network not allowed")
errNoVersioner = errors.New("folder has no versioner")
// errors about why a connection is closed
@@ -875,7 +875,7 @@ func (comp FolderCompletion) Map() map[string]interface{} {
// (including the local device) or explicitly protocol.LocalDeviceID. An
// empty folder string means the aggregate of all folders shared with the
// given device.
func (m *model) Completion(device protocol.DeviceID, folder string) FolderCompletion {
func (m *model) Completion(device protocol.DeviceID, folder string) (FolderCompletion, error) {
// The user specifically asked for our own device ID. Internally that is
// known as protocol.LocalDeviceID so translate.
if device == m.id {
@@ -891,21 +891,29 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
var comp FolderCompletion
for _, fcfg := range m.cfg.FolderList() {
if device == protocol.LocalDeviceID || fcfg.SharedWith(device) {
comp.add(m.folderCompletion(device, fcfg.ID))
folderComp, err := m.folderCompletion(device, fcfg.ID)
if err != nil {
return FolderCompletion{}, err
}
comp.add(folderComp)
}
}
return comp
return comp, nil
}
func (m *model) folderCompletion(device protocol.DeviceID, folder string) FolderCompletion {
func (m *model) folderCompletion(device protocol.DeviceID, folder string) (FolderCompletion, error) {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
err := m.checkFolderRunningLocked(folder)
rf := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
if err != nil {
return FolderCompletion{}, err
}
snap := rf.Snapshot()
snap, err := rf.Snapshot()
if err != nil {
return FolderCompletion{}, err
}
defer snap.Release()
m.pmut.RLock()
@@ -922,7 +930,7 @@ func (m *model) folderCompletion(device protocol.DeviceID, folder string) Folder
comp := newFolderCompletion(snap.GlobalSize(), need, snap.Sequence(device))
l.Debugf("%v Completion(%s, %q): %v", m, device, folder, comp.Map())
return comp
return comp, nil
}
// DBSnapshot returns a snapshot of the database content relevant to the given folder.
@@ -934,7 +942,7 @@ func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) {
if err != nil {
return nil, err
}
return rf.Snapshot(), nil
return rf.Snapshot()
}
func (m *model) FolderProgressBytesCompleted(folder string) int64 {
@@ -951,10 +959,13 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
m.fmut.RUnlock()
if !rfOk {
return nil, nil, nil, errFolderMissing
return nil, nil, nil, ErrFolderMissing
}
snap := rf.Snapshot()
snap, err := rf.Snapshot()
if err != nil {
return nil, nil, nil, err
}
defer snap.Release()
var progress, queued, rest []db.FileInfoTruncated
var seen map[string]struct{}
@@ -1018,10 +1029,13 @@ func (m *model) RemoteNeedFolderFiles(folder string, device protocol.DeviceID, p
m.fmut.RUnlock()
if !ok {
return nil, errFolderMissing
return nil, ErrFolderMissing
}
snap := rf.Snapshot()
snap, err := rf.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
files := make([]db.FileInfoTruncated, 0, perpage)
@@ -1043,10 +1057,13 @@ func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db.
m.fmut.RUnlock()
if !ok {
return nil, errFolderMissing
return nil, ErrFolderMissing
}
snap := rf.Snapshot()
snap, err := rf.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
if snap.ReceiveOnlyChangedSize().TotalItems() == 0 {
@@ -1120,7 +1137,7 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot
if cfg, ok := m.cfg.Folder(folder); !ok || !cfg.SharedWith(deviceID) {
l.Infof("%v for unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", op, folder, deviceID)
return errors.Wrap(errFolderMissing, folder)
return errors.Wrap(ErrFolderMissing, folder)
} else if cfg.Paused {
l.Debugf("%v for paused folder (ID %q) sent from device %q.", op, folder, deviceID)
return errors.Wrap(ErrFolderPaused, folder)
@@ -1133,7 +1150,7 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot
if !existing {
l.Infof("%v for nonexistent folder %q", op, folder)
return errors.Wrap(errFolderMissing, folder)
return errors.Wrap(ErrFolderMissing, folder)
}
if running {
@@ -1930,7 +1947,11 @@ func newLimitedRequestResponse(size int, limiters ...*byteSemaphore) *requestRes
}
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32) {
cf, ok := m.CurrentFolderFile(folder, name)
cf, ok, err := m.CurrentFolderFile(folder, name)
if err != nil {
l.Debugf("%v recheckFile: %s: %q / %q: current file error: %v", m, deviceID, folder, name, err)
return
}
if !ok {
l.Debugf("%v recheckFile: %s: %q / %q: no current file", m, deviceID, folder, name)
return
@@ -1976,28 +1997,36 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, off
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
}
func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
return protocol.FileInfo{}, false
return protocol.FileInfo{}, false, ErrFolderMissing
}
snap := fs.Snapshot()
defer snap.Release()
return snap.Get(protocol.LocalDeviceID, file)
snap, err := fs.Snapshot()
if err != nil {
return protocol.FileInfo{}, false, err
}
f, ok := snap.Get(protocol.LocalDeviceID, file)
snap.Release()
return f, ok, nil
}
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) {
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
return protocol.FileInfo{}, false
return protocol.FileInfo{}, false, ErrFolderMissing
}
snap := fs.Snapshot()
defer snap.Release()
return snap.GetGlobal(file)
snap, err := fs.Snapshot()
if err != nil {
return protocol.FileInfo{}, false, err
}
f, ok := snap.GetGlobal(file)
snap.Release()
return f, ok, nil
}
// Connection returns the current connection for device, and a boolean whether a connection was found.
@@ -2552,7 +2581,7 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly
files, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
return nil, errFolderMissing
return nil, ErrFolderMissing
}
root := &TreeEntry{
@@ -2565,9 +2594,11 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly
prefix = prefix + sep
}
snap := files.Snapshot()
snap, err := files.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
var err error
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
@@ -2661,7 +2692,7 @@ func (m *model) RestoreFolderVersions(folder string, versions map[string]time.Ti
return restoreErrors, nil
}
func (m *model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
func (m *model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error) {
// The slightly unusual locking sequence here is because we need to hold
// pmut for the duration (as the value returned from foldersFiles can
// get heavily modified on Close()), but also must acquire fmut before
@@ -2675,17 +2706,25 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc
m.fmut.RUnlock()
if !ok {
return nil
return nil, ErrFolderMissing
}
var availabilities []Availability
snap := fs.Snapshot()
snap, err := fs.Snapshot()
if err != nil {
return nil, err
}
defer snap.Release()
return m.availabilityInSnapshot(cfg, snap, file, block), nil
}
func (m *model) availabilityInSnapshot(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
var availabilities []Availability
for _, device := range snap.Availability(file.Name) {
if _, ok := m.remotePausedFolders[device]; !ok {
continue
}
if _, ok := m.remotePausedFolders[device][folder]; ok {
if _, ok := m.remotePausedFolders[device][cfg.ID]; ok {
continue
}
_, ok := m.conn[device]
@@ -2695,7 +2734,7 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc
}
for _, device := range cfg.Devices {
if m.deviceDownloads[device.DeviceID].Has(folder, file.Name, file.Version, int(block.Offset/int64(file.BlockSize()))) {
if m.deviceDownloads[device.DeviceID].Has(cfg.ID, file.Name, file.Version, int(block.Offset/int64(file.BlockSize()))) {
availabilities = append(availabilities, Availability{ID: device.DeviceID, FromTemporary: true})
}
}
@@ -2960,12 +2999,12 @@ func (m *model) checkFolderRunningLocked(folder string) error {
}
if cfg, ok := m.cfg.Folder(folder); !ok {
return errFolderMissing
return ErrFolderMissing
} else if cfg.Paused {
return ErrFolderPaused
}
return errFolderNotRunning
return ErrFolderNotRunning
}
// PendingDevices lists unknown devices that tried to connect.
+30 -30
View File
@@ -2300,7 +2300,7 @@ func TestIssue3496(t *testing.T) {
fs := m.folderFiles["default"]
m.fmut.RUnlock()
var localFiles []protocol.FileInfo
snap := fs.Snapshot()
snap := fsetSnapshot(t, fs)
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
localFiles = append(localFiles, i.(protocol.FileInfo))
return true
@@ -2329,7 +2329,7 @@ func TestIssue3496(t *testing.T) {
// Check that the completion percentage for us makes sense
comp := m.Completion(protocol.LocalDeviceID, "default")
comp := m.testCompletion(protocol.LocalDeviceID, "default")
if comp.NeedBytes > comp.GlobalBytes {
t.Errorf("Need more bytes than exist, not possible: %d > %d", comp.NeedBytes, comp.GlobalBytes)
}
@@ -2393,7 +2393,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
files.Update(device1, []protocol.FileInfo{file})
files.Update(device2, []protocol.FileInfo{file})
avail := m.Availability("default", file, file.Blocks[0])
avail := m.testAvailability("default", file, file.Blocks[0])
if len(avail) != 0 {
t.Errorf("should not be available, no connections")
}
@@ -2403,7 +2403,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
// !!! This is not what I'd expect to happen, as we don't even know if the peer has the original index !!!
avail = m.Availability("default", file, file.Blocks[0])
avail = m.testAvailability("default", file, file.Blocks[0])
if len(avail) != 2 {
t.Errorf("should have two available")
}
@@ -2423,7 +2423,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
m.ClusterConfig(device1, cc)
m.ClusterConfig(device2, cc)
avail = m.Availability("default", file, file.Blocks[0])
avail = m.testAvailability("default", file, file.Blocks[0])
if len(avail) != 2 {
t.Errorf("should have two available")
}
@@ -2431,7 +2431,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
m.Closed(newFakeConnection(device1, m), errDeviceUnknown)
m.Closed(newFakeConnection(device2, m), errDeviceUnknown)
avail = m.Availability("default", file, file.Blocks[0])
avail = m.testAvailability("default", file, file.Blocks[0])
if len(avail) != 0 {
t.Errorf("should have no available")
}
@@ -2446,7 +2446,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
ccp.Folders[0].Paused = true
m.ClusterConfig(device1, ccp)
avail = m.Availability("default", file, file.Blocks[0])
avail = m.testAvailability("default", file, file.Blocks[0])
if len(avail) != 1 {
t.Errorf("should have one available")
}
@@ -2479,12 +2479,12 @@ func TestIssue2571(t *testing.T) {
m.ScanFolder("default")
if dir, ok := m.CurrentFolderFile("default", "toLink"); !ok {
if dir, ok := m.testCurrentFolderFile("default", "toLink"); !ok {
t.Fatalf("Dir missing in db")
} else if !dir.IsSymlink() {
t.Errorf("Dir wasn't changed to symlink")
}
if file, ok := m.CurrentFolderFile("default", filepath.Join("toLink", "a")); !ok {
if file, ok := m.testCurrentFolderFile("default", filepath.Join("toLink", "a")); !ok {
t.Fatalf("File missing in db")
} else if !file.Deleted {
t.Errorf("File below symlink has not been marked as deleted")
@@ -2517,7 +2517,7 @@ func TestIssue4573(t *testing.T) {
m.ScanFolder("default")
if file, ok := m.CurrentFolderFile("default", file); !ok {
if file, ok := m.testCurrentFolderFile("default", file); !ok {
t.Fatalf("File missing in db")
} else if file.Deleted {
t.Errorf("Inaccessible file has been marked as deleted.")
@@ -2577,7 +2577,7 @@ func TestInternalScan(t *testing.T) {
m.ScanFolder("default")
for path, cond := range testCases {
if f, ok := m.CurrentFolderFile("default", path); !ok {
if f, ok := m.testCurrentFolderFile("default", path); !ok {
t.Fatalf("%v missing in db", path)
} else if cond(f) {
t.Errorf("Incorrect db entry for %v", path)
@@ -2638,14 +2638,14 @@ func TestRemoveDirWithContent(t *testing.T) {
m := setupModel(t, defaultCfgWrapper)
defer cleanupModel(m)
dir, ok := m.CurrentFolderFile("default", "dirwith")
dir, ok := m.testCurrentFolderFile("default", "dirwith")
if !ok {
t.Fatalf("Can't get dir \"dirwith\" after initial scan")
}
dir.Deleted = true
dir.Version = dir.Version.Update(device1.Short()).Update(device1.Short())
file, ok := m.CurrentFolderFile("default", content)
file, ok := m.testCurrentFolderFile("default", content)
if !ok {
t.Fatalf("Can't get file \"%v\" after initial scan", content)
}
@@ -2657,11 +2657,11 @@ func TestRemoveDirWithContent(t *testing.T) {
// Is there something we could trigger on instead of just waiting?
timeout := time.NewTimer(5 * time.Second)
for {
dir, ok := m.CurrentFolderFile("default", "dirwith")
dir, ok := m.testCurrentFolderFile("default", "dirwith")
if !ok {
t.Fatalf("Can't get dir \"dirwith\" after index update")
}
file, ok := m.CurrentFolderFile("default", content)
file, ok := m.testCurrentFolderFile("default", content)
if !ok {
t.Fatalf("Can't get file \"%v\" after index update", content)
}
@@ -2713,11 +2713,11 @@ func TestIssue4475(t *testing.T) {
created := false
for {
if !created {
if _, ok := m.CurrentFolderFile("default", fileName); ok {
if _, ok := m.testCurrentFolderFile("default", fileName); ok {
created = true
}
} else {
dir, ok := m.CurrentFolderFile("default", "delDir")
dir, ok := m.testCurrentFolderFile("default", "delDir")
if !ok {
t.Fatalf("can't get dir from db")
}
@@ -2954,7 +2954,7 @@ func TestPausedFolders(t *testing.T) {
t.Errorf("Expected folder paused error, received: %v", err)
}
if err := m.ScanFolder("nonexistent"); err != errFolderMissing {
if err := m.ScanFolder("nonexistent"); err != ErrFolderMissing {
t.Errorf("Expected missing folder error, received: %v", err)
}
}
@@ -3035,7 +3035,7 @@ func TestIssue5002(t *testing.T) {
t.Error(err)
}
file, ok := m.CurrentFolderFile("default", "foo")
file, ok := m.testCurrentFolderFile("default", "foo")
if !ok {
t.Fatal("test file should exist")
}
@@ -3054,7 +3054,7 @@ func TestParentOfUnignored(t *testing.T) {
m.SetIgnores("default", []string{"!quux", "*"})
if parent, ok := m.CurrentFolderFile("default", "baz"); !ok {
if parent, ok := m.testCurrentFolderFile("default", "baz"); !ok {
t.Errorf(`Directory "baz" missing in db`)
} else if parent.IsIgnored() {
t.Errorf(`Directory "baz" is ignored`)
@@ -3222,7 +3222,7 @@ func TestModTimeWindow(t *testing.T) {
// Get current version
fi, ok := m.CurrentFolderFile("default", name)
fi, ok := m.testCurrentFolderFile("default", name)
if !ok {
t.Fatal("File missing")
}
@@ -3237,7 +3237,7 @@ func TestModTimeWindow(t *testing.T) {
// No change due to within window
fi, _ = m.CurrentFolderFile("default", name)
fi, _ = m.testCurrentFolderFile("default", name)
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
@@ -3251,7 +3251,7 @@ func TestModTimeWindow(t *testing.T) {
// Version should have updated
fi, _ = m.CurrentFolderFile("default", name)
fi, _ = m.testCurrentFolderFile("default", name)
if fi.Version.Compare(v) != protocol.Greater {
t.Fatalf("Got result %v, expected %v", fi.Version.Compare(v), protocol.Greater)
}
@@ -3368,8 +3368,8 @@ func TestFolderAPIErrors(t *testing.T) {
if err := method(fcfg.ID); err != ErrFolderPaused {
t.Errorf(`Expected "%v", got "%v" (method no %v)`, ErrFolderPaused, err, i)
}
if err := method("notexisting"); err != errFolderMissing {
t.Errorf(`Expected "%v", got "%v" (method no %v)`, errFolderMissing, err, i)
if err := method("notexisting"); err != ErrFolderMissing {
t.Errorf(`Expected "%v", got "%v" (method no %v)`, ErrFolderMissing, err, i)
}
}
}
@@ -3776,7 +3776,7 @@ func TestScanDeletedROChangedOnSR(t *testing.T) {
must(t, writeFile(ffs, name, []byte(name), 0644))
m.ScanFolders()
file, ok := m.CurrentFolderFile(fcfg.ID, name)
file, ok := m.testCurrentFolderFile(fcfg.ID, name)
if !ok {
t.Fatal("file missing in db")
}
@@ -3927,7 +3927,7 @@ func TestIssue6961(t *testing.T) {
pauseFolder(t, wcfg, fcfg.ID, true)
pauseFolder(t, wcfg, fcfg.ID, false)
if comp := m.Completion(device2, fcfg.ID); comp.NeedDeletes != 0 {
if comp := m.testCompletion(device2, fcfg.ID); comp.NeedDeletes != 0 {
t.Error("Expected 0 needed deletes, got", comp.NeedDeletes)
} else {
t.Log(comp)
@@ -3946,7 +3946,7 @@ func TestCompletionEmptyGlobal(t *testing.T) {
files[0].Deleted = true
files[0].Version = files[0].Version.Update(device1.Short())
m.IndexUpdate(device1, fcfg.ID, files)
comp := m.Completion(protocol.LocalDeviceID, fcfg.ID)
comp := m.testCompletion(protocol.LocalDeviceID, fcfg.ID)
if comp.CompletionPct != 95 {
t.Error("Expected completion of 95%, got", comp.CompletionPct)
}
@@ -3978,13 +3978,13 @@ func TestNeedMetaAfterIndexReset(t *testing.T) {
files[0].Sequence = seq
m.IndexUpdate(device1, fcfg.ID, files)
if comp := m.Completion(device2, fcfg.ID); comp.NeedItems != 1 {
if comp := m.testCompletion(device2, fcfg.ID); comp.NeedItems != 1 {
t.Error("Expected one needed item for device2, got", comp.NeedItems)
}
// Pretend we had an index reset on device 1
m.Index(device1, fcfg.ID, files)
if comp := m.Completion(device2, fcfg.ID); comp.NeedItems != 1 {
if comp := m.testCompletion(device2, fcfg.ID); comp.NeedItems != 1 {
t.Error("Expected one needed item for device2, got", comp.NeedItems)
}
}
+29
View File
@@ -152,6 +152,7 @@ func setupModel(t testing.TB, w config.Wrapper) *testModel {
type testModel struct {
*model
t testing.TB
cancel context.CancelFunc
evCancel context.CancelFunc
stopped chan struct{}
@@ -171,6 +172,7 @@ func newModel(t testing.TB, cfg config.Wrapper, id protocol.DeviceID, clientName
model: m,
evCancel: cancel,
stopped: make(chan struct{}),
t: t,
}
}
@@ -184,6 +186,24 @@ func (m *testModel) ServeBackground() {
<-m.started
}
func (m *testModel) testAvailability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
av, err := m.model.Availability(folder, file, block)
must(m.t, err)
return av
}
func (m *testModel) testCurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
f, ok, err := m.model.CurrentFolderFile(folder, file)
must(m.t, err)
return f, ok
}
func (m *testModel) testCompletion(device protocol.DeviceID, folder string) FolderCompletion {
comp, err := m.Completion(protocol.LocalDeviceID, "default")
must(m.t, err)
return comp
}
func cleanupModel(m *testModel) {
if m.cancel != nil {
m.cancel()
@@ -277,6 +297,15 @@ func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {
return snap
}
func fsetSnapshot(t *testing.T, fset *db.FileSet) *db.Snapshot {
t.Helper()
snap, err := fset.Snapshot()
if err != nil {
t.Fatal(err)
}
return snap
}
// Reach in and update the ignore matcher to one that always does
// reloads when asked to, instead of checking file mtimes. This is
// because we will be changing the files on disk often enough that the