felixfoertsch/syncthing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

all: Implement suture v4-api (#6947)

Browse Source
This commit is contained in:
Simon Frei
2020-11-17 13:19:04 +01:00
committed by GitHub
parent e8fc465ea8
commit 9524b51708
65 changed files with 617 additions and 693 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/model/fakeconns_test.go
+2 -2
View File
@@ -31,7 +31,7 @@ type fakeConnection struct {
files []protocol.FileInfo
fileData map[string][]byte
folder string
model *model
model *testModel
indexFn func(context.Context, string, []protocol.FileInfo)
requestFn func(ctx context.Context, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
closeFn func(error)
@@ -201,7 +201,7 @@ func (f *fakeConnection) sendIndexUpdate() {
f.model.IndexUpdate(f.id, f.folder, f.files)
}
func addFakeConn(m *model, dev protocol.DeviceID) *fakeConnection {
func addFakeConn(m *testModel, dev protocol.DeviceID) *fakeConnection {
fc := &fakeConnection{id: dev, model: m}
m.AddConnection(fc, protocol.Hello{})
lib/model/folder.go
+2 -5
View File
@@ -31,12 +31,9 @@ import (
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/watchaggregator"
"github.com/thejerf/suture"
)
type folder struct {
suture.Service
stateTracker
config.FolderConfiguration
*stats.FolderStatisticsReference
@@ -135,7 +132,7 @@ func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg conf
return f
}
func (f *folder) serve(ctx context.Context) {
func (f *folder) Serve(ctx context.Context) error {
atomic.AddInt32(&f.model.foldersRunning, 1)
defer atomic.AddInt32(&f.model.foldersRunning, -1)
@@ -168,7 +165,7 @@ func (f *folder) serve(ctx context.Context) {
select {
case <-f.ctx.Done():
close(f.done)
return
return nil
case <-f.pullScheduled:
f.pull()
lib/model/folder_recvonly_test.go
+2 -1
View File
@@ -441,7 +441,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
return knownFiles
}
func setupROFolder(t *testing.T) (*model, *receiveOnlyFolder) {
func setupROFolder(t *testing.T) (*testModel, *receiveOnlyFolder) {
t.Helper()
w := createTmpWrapper(defaultCfg)
@@ -455,6 +455,7 @@ func setupROFolder(t *testing.T) (*model, *receiveOnlyFolder) {
m := newModel(w, myID, "syncthing", "dev", db.NewLowlevel(backend.OpenMemory()), nil)
m.ServeBackground()
<-m.started
must(t, m.ScanFolder("ro"))
m.fmut.RLock()
lib/model/folder_sendonly.go
-2
View File
@@ -13,7 +13,6 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
)
@@ -30,7 +29,6 @@ func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher,
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil),
}
f.folder.puller = f
f.folder.Service = util.AsService(f.serve, f.String())
return f
}
lib/model/folder_sendrecv.go
-2
View File
@@ -28,7 +28,6 @@ import (
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
)
@@ -140,7 +139,6 @@ func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
writeLimiter: newByteSemaphore(cfg.MaxConcurrentWrites),
}
f.folder.puller = f
f.folder.Service = util.AsService(f.serve, f.String())
if f.Copiers == 0 {
f.Copiers = defaultCopiers
lib/model/folder_sendrecv_test.go
+5 -4
View File
@@ -91,10 +91,12 @@ func createFile(t *testing.T, name string, fs fs.Filesystem) protocol.FileInfo {
}
// Sets up a folder and model, but makes sure the services aren't actually running.
func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFolder) {
func setupSendReceiveFolder(files ...protocol.FileInfo) (*testModel, *sendReceiveFolder) {
w, fcfg := tmpDefaultWrapper()
// Initialise model and stop immediately.
model := setupModel(w)
model.Supervisor.Stop()
model.cancel()
<-model.stopped
f := model.folderRunners[fcfg.ID].(*sendReceiveFolder)
f.tempPullErrors = make(map[string]string)
f.ctx = context.Background()
@@ -107,8 +109,7 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
return model, f
}
func cleanupSRFolder(f *sendReceiveFolder, m *model) {
m.evLogger.Stop()
func cleanupSRFolder(f *sendReceiveFolder, m *testModel) {
os.Remove(m.cfg.ConfigPath())
os.RemoveAll(f.Filesystem().URI())
}
lib/model/folder_summary.go
+7 -9
View File
@@ -12,7 +12,7 @@ import (
"strings"
"time"
"github.com/thejerf/suture"
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
@@ -52,9 +52,7 @@ type folderSummaryService struct {
func NewFolderSummaryService(cfg config.Wrapper, m Model, id protocol.DeviceID, evLogger events.Logger) FolderSummaryService {
service := &folderSummaryService{
Supervisor: suture.New("folderSummaryService", suture.Spec{
PassThroughPanics: true,
}),
Supervisor: suture.New("folderSummaryService", util.Spec()),
cfg: cfg,
model: m,
id: id,
@@ -169,7 +167,7 @@ func (c *folderSummaryService) OnEventRequest() {
// listenForUpdates subscribes to the event bus and makes note of folders that
// need their data recalculated.
func (c *folderSummaryService) listenForUpdates(ctx context.Context) {
func (c *folderSummaryService) listenForUpdates(ctx context.Context) error {
sub := c.evLogger.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged | events.DownloadProgress)
defer sub.Unsubscribe()
@@ -180,7 +178,7 @@ func (c *folderSummaryService) listenForUpdates(ctx context.Context) {
case ev := <-sub.C():
c.processUpdate(ev)
case <-ctx.Done():
return
return ctx.Err()
}
}
}
@@ -261,7 +259,7 @@ func (c *folderSummaryService) processUpdate(ev events.Event) {
// calculateSummaries periodically recalculates folder summaries and
// completion percentage, and sends the results on the event bus.
func (c *folderSummaryService) calculateSummaries(ctx context.Context) {
func (c *folderSummaryService) calculateSummaries(ctx context.Context) error {
const pumpInterval = 2 * time.Second
pump := time.NewTimer(pumpInterval)
@@ -272,7 +270,7 @@ func (c *folderSummaryService) calculateSummaries(ctx context.Context) {
for _, folder := range c.foldersToHandle() {
select {
case <-ctx.Done():
return
return ctx.Err()
default:
}
c.sendSummary(ctx, folder)
@@ -288,7 +286,7 @@ func (c *folderSummaryService) calculateSummaries(ctx context.Context) {
c.sendSummary(ctx, folder)
case <-ctx.Done():
return
return ctx.Err()
}
}
}
lib/model/indexsender.go
+8 -9
View File
@@ -12,17 +12,15 @@ import (
"sync"
"time"
"github.com/thejerf/suture"
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
)
type indexSender struct {
suture.Service
conn protocol.Connection
folder string
folderIsReceiveEncrypted bool
@@ -36,7 +34,7 @@ type indexSender struct {
resumeChan chan *db.FileSet
}
func (s *indexSender) serve(ctx context.Context) {
func (s *indexSender) Serve(ctx context.Context) error {
var err error
l.Debugf("Starting indexSender for %s to %s at %s (slv=%d)", s.folder, s.conn.ID(), s.conn, s.prevSequence)
@@ -59,9 +57,9 @@ func (s *indexSender) serve(ctx context.Context) {
for err == nil {
select {
case <-ctx.Done():
return
return ctx.Err()
case <-s.connClosed:
return
return nil
default:
}
@@ -72,9 +70,9 @@ func (s *indexSender) serve(ctx context.Context) {
if s.fset.Sequence(protocol.LocalDeviceID) <= s.prevSequence {
select {
case <-ctx.Done():
return
return ctx.Err()
case <-s.connClosed:
return
return nil
case <-evChan:
case <-ticker.C:
case <-s.pauseChan:
@@ -95,6 +93,8 @@ func (s *indexSender) serve(ctx context.Context) {
// time to batch them up a little.
time.Sleep(250 * time.Millisecond)
}
return nil
}
// Complete implements the suture.IsCompletable interface. When Serve terminates
@@ -333,7 +333,6 @@ func (r *indexSenderRegistry) startLocked(folderID string, fset *db.FileSet, sta
pauseChan: make(chan struct{}),
resumeChan: make(chan *db.FileSet),
}
is.Service = util.AsService(is.serve, is.String())
is.token = r.sup.Add(is)
r.indexSenders[folderID] = is
}
lib/model/model.go
+20 -24
View File
@@ -22,7 +22,7 @@ import (
"unicode"
"github.com/pkg/errors"
"github.com/thejerf/suture"
"github.com/thejerf/suture/v4"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
@@ -36,6 +36,7 @@ import (
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
)
@@ -46,6 +47,7 @@ const (
)
type service interface {
suture.Service
BringToFront(string)
Override()
Revert()
@@ -53,8 +55,6 @@ type service interface {
SchedulePull() // something relevant changed, we should try a pull
Jobs(page, perpage int) ([]string, []string, int) // In progress, Queued, skipped
Scan(subs []string) error
Serve()
Stop()
Errors() []FileError
WatchError() error
ScheduleForceRescan(path string)
@@ -154,7 +154,9 @@ type model struct {
remotePausedFolders map[protocol.DeviceID]map[string]struct{} // deviceID -> folders
indexSenders map[protocol.DeviceID]*indexSenderRegistry
foldersRunning int32 // for testing only
// for testing only
foldersRunning int32
started chan struct{}
}
type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, fs.Filesystem, events.Logger, *byteSemaphore) service
@@ -192,13 +194,12 @@ var (
// where it sends index information to connected peers and responds to requests
// for file data without altering the local folder in any way.
func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger) Model {
spec := util.Spec()
spec.EventHook = func(e suture.Event) {
l.Debugln(e)
}
m := &model{
Supervisor: suture.New("model", suture.Spec{
Log: func(line string) {
l.Debugln(line)
},
PassThroughPanics: true,
}),
Supervisor: suture.New("model", spec),
// constructor parameters
cfg: cfg,
@@ -237,26 +238,20 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
deviceDownloads: make(map[protocol.DeviceID]*deviceDownloadState),
remotePausedFolders: make(map[protocol.DeviceID]map[string]struct{}),
indexSenders: make(map[protocol.DeviceID]*indexSenderRegistry),
// for testing only
started: make(chan struct{}),
}
for devID := range cfg.Devices() {
m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID.String())
}
m.Add(m.progressEmitter)
m.Add(util.AsService(m.serve, m.String()))
return m
}
func (m *model) Serve() {
m.onServe()
m.Supervisor.Serve()
}
func (m *model) ServeBackground() {
m.onServe()
m.Supervisor.ServeBackground()
}
func (m *model) onServe() {
func (m *model) serve(ctx context.Context) error {
// Add and start folders
cacheIgnoredFiles := m.cfg.Options().CacheIgnoredFiles
for _, folderCfg := range m.cfg.Folders() {
@@ -267,11 +262,11 @@ func (m *model) onServe() {
m.newFolder(folderCfg, cacheIgnoredFiles)
}
m.cfg.Subscribe(m)
}
func (m *model) Stop() {
close(m.started)
<-ctx.Done()
m.cfg.Unsubscribe(m)
m.Supervisor.Stop()
m.pmut.RLock()
closed := make([]chan struct{}, 0, len(m.conn))
for id, conn := range m.conn {
@@ -282,6 +277,7 @@ func (m *model) Stop() {
for _, c := range closed {
<-c
}
return nil
}
// StartDeadlockDetector starts a deadlock detector on the models locks which
lib/model/model_test.go
+3 -3
View File
@@ -120,7 +120,7 @@ func createTmpWrapper(cfg config.Configuration) config.Wrapper {
return wrapper
}
func newState(cfg config.Configuration) *model {
func newState(cfg config.Configuration) *testModel {
wcfg := createTmpWrapper(cfg)
m := setupModel(wcfg)
@@ -1396,7 +1396,7 @@ func TestAutoAcceptEnc(t *testing.T) {
}
}
func changeIgnores(t *testing.T, m *model, expected []string) {
func changeIgnores(t *testing.T, m *testModel, expected []string) {
arrEqual := func(a, b []string) bool {
if len(a) != len(b) {
return false
@@ -4205,7 +4205,7 @@ func TestNeedMetaAfterIndexReset(t *testing.T) {
func TestCcCheckEncryption(t *testing.T) {
w, fcfg := tmpDefaultWrapper()
m := setupModel(w)
m.Stop()
m.cancel()
defer cleanupModel(m)
pw := "foo"
lib/model/progressemitter.go
+2 -8
View File
@@ -11,18 +11,13 @@ import (
"fmt"
"time"
"github.com/thejerf/suture"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
)
type ProgressEmitter struct {
suture.Service
cfg config.Wrapper
registry map[string]map[string]*sharedPullerState // folder: name: puller
interval time.Duration
@@ -60,7 +55,6 @@ func NewProgressEmitter(cfg config.Wrapper, evLogger events.Logger) *ProgressEmi
evLogger: evLogger,
mut: sync.NewMutex(),
}
t.Service = util.AsService(t.serve, t.String())
t.CommitConfiguration(config.Configuration{}, cfg.RawCopy())
@@ -69,7 +63,7 @@ func NewProgressEmitter(cfg config.Wrapper, evLogger events.Logger) *ProgressEmi
// serve starts the progress emitter which starts emitting DownloadProgress
// events as the progress happens.
func (t *ProgressEmitter) serve(ctx context.Context) {
func (t *ProgressEmitter) Serve(ctx context.Context) error {
t.cfg.Subscribe(t)
defer t.cfg.Unsubscribe(t)
@@ -79,7 +73,7 @@ func (t *ProgressEmitter) serve(ctx context.Context) {
select {
case <-ctx.Done():
l.Debugln("progress emitter: stopping")
return
return nil
case <-t.timer.C:
t.mut.Lock()
l.Debugln("progress emitter: timer - looking after", len(t.registry))
lib/model/progressemitter_test.go
+7 -6
View File
@@ -53,9 +53,10 @@ func expectTimeout(w events.Subscription, t *testing.T) {
}
func TestProgressEmitter(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
evLogger := events.NewLogger()
go evLogger.Serve()
defer evLogger.Stop()
go evLogger.Serve(ctx)
defer cancel()
w := evLogger.Subscribe(events.DownloadProgress)
@@ -66,8 +67,7 @@ func TestProgressEmitter(t *testing.T) {
})
p := NewProgressEmitter(c, evLogger)
go p.Serve()
defer p.Stop()
go p.Serve(ctx)
p.interval = 0
expectTimeout(w, t)
@@ -118,9 +118,10 @@ func TestSendDownloadProgressMessages(t *testing.T) {
fc := &fakeConnection{}
ctx, cancel := context.WithCancel(context.Background())
evLogger := events.NewLogger()
go evLogger.Serve()
defer evLogger.Stop()
go evLogger.Serve(ctx)
defer cancel()
p := NewProgressEmitter(c, evLogger)
p.temporaryIndexSubscribe(fc, []string{"folder", "folder2"})
lib/model/testutils_test.go
+41 -14
View File
@@ -7,6 +7,7 @@
package model
import (
"context"
"io/ioutil"
"os"
"testing"
@@ -95,13 +96,13 @@ func testFolderConfigFake() config.FolderConfiguration {
return cfg
}
func setupModelWithConnection() (*model, *fakeConnection, config.FolderConfiguration) {
func setupModelWithConnection() (*testModel, *fakeConnection, config.FolderConfiguration) {
w, fcfg := tmpDefaultWrapper()
m, fc := setupModelWithConnectionFromWrapper(w)
return m, fc, fcfg
}
func setupModelWithConnectionFromWrapper(w config.Wrapper) (*model, *fakeConnection) {
func setupModelWithConnectionFromWrapper(w config.Wrapper) (*testModel, *fakeConnection) {
m := setupModel(w)
fc := addFakeConn(m, device1)
@@ -112,31 +113,57 @@ func setupModelWithConnectionFromWrapper(w config.Wrapper) (*model, *fakeConnect
return m, fc
}
func setupModel(w config.Wrapper) *model {
func setupModel(w config.Wrapper) *testModel {
db := db.NewLowlevel(backend.OpenMemory())
m := newModel(w, myID, "syncthing", "dev", db, nil)
m.ServeBackground()
<-m.started
m.ScanFolders()
return m
}
func newModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) *model {
evLogger := events.NewLogger()
m := NewModel(cfg, id, clientName, clientVersion, ldb, protectedFiles, evLogger).(*model)
go evLogger.Serve()
return m
type testModel struct {
*model
cancel context.CancelFunc
evCancel context.CancelFunc
stopped chan struct{}
}
func cleanupModel(m *model) {
m.Stop()
func newModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) *testModel {
evLogger := events.NewLogger()
m := NewModel(cfg, id, clientName, clientVersion, ldb, protectedFiles, evLogger).(*model)
ctx, cancel := context.WithCancel(context.Background())
go evLogger.Serve(ctx)
return &testModel{
model: m,
evCancel: cancel,
stopped: make(chan struct{}),
}
}
func (m *testModel) ServeBackground() {
ctx, cancel := context.WithCancel(context.Background())
m.cancel = cancel
go func() {
m.model.Serve(ctx)
close(m.stopped)
}()
<-m.started
}
func cleanupModel(m *testModel) {
if m.cancel != nil {
m.cancel()
<-m.stopped
}
m.evCancel()
m.db.Close()
m.evLogger.Stop()
os.Remove(m.cfg.ConfigPath())
}
func cleanupModelAndRemoveDir(m *model, dir string) {
func cleanupModelAndRemoveDir(m *testModel, dir string) {
cleanupModel(m)
os.RemoveAll(dir)
}
@@ -223,7 +250,7 @@ func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {
// reloads when asked to, instead of checking file mtimes. This is
// because we will be changing the files on disk often enough that the
// mtimes will be unreliable to determine change status.
func folderIgnoresAlwaysReload(m *model, fcfg config.FolderConfiguration) {
func folderIgnoresAlwaysReload(m *testModel, fcfg config.FolderConfiguration) {
m.removeFolder(fcfg)
fset := db.NewFileSet(fcfg.ID, fcfg.Filesystem(), m.db)
ignores := ignore.New(fcfg.Filesystem(), ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
@@ -250,7 +277,7 @@ func basicClusterConfig(local, remote protocol.DeviceID, folders ...string) prot
return cc
}
func localIndexUpdate(m *model, folder string, fs []protocol.FileInfo) {
func localIndexUpdate(m *testModel, folder string, fs []protocol.FileInfo) {
m.fmut.RLock()
fset := m.folderFiles[folder]
m.fmut.RUnlock()