all: Support multiple device connections (fixes #141) (#8918)

This adds the ability to have multiple concurrent connections to a single device. This is primarily useful when the network has multiple physical links for aggregated bandwidth. A single connection will never see a higher rate than a single link can give, but multiple connections are load-balanced over multiple links. It is also incidentally useful for older multi-core CPUs, where bandwidth could be limited by the TLS performance of a single CPU core -- using multiple connections achieves concurrency in the required crypto calculations... Co-authored-by: Simon Frei <freisim93@gmail.com> Co-authored-by: tomasz1986 <twilczynski@naver.com> Co-authored-by: bt90 <btom1990@googlemail.com>
2023-09-06 12:52:01 +02:00
parent 38bbdebffa
commit c6334e61aa
41 changed files with 1640 additions and 933 deletions
@@ -37,6 +37,7 @@ import (
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/semaphore"
 	"github.com/syncthing/syncthing/lib/stats"
@@ -108,6 +109,7 @@ type Model interface {
 	DeviceStatistics() (map[protocol.DeviceID]stats.DeviceStatistics, error)
 	FolderStatistics() (map[string]stats.FolderStatistics, error)
 	UsageReportingStats(report *contract.Report, version int, preview bool)
+	ConnectedTo(remoteID protocol.DeviceID) bool

 	PendingDevices() (map[protocol.DeviceID]db.ObservedDevice, error)
 	PendingFolders(device protocol.DeviceID) (map[string]db.PendingFolder, error)
@@ -124,8 +126,6 @@ type model struct {
 	// constructor parameters
 	cfg            config.Wrapper
 	id             protocol.DeviceID
-	clientName     string
-	clientVersion  string
 	db             *db.Lowlevel
 	protectedFiles []string
 	evLogger       events.Logger
@@ -143,6 +143,7 @@ type model struct {
 	fatalChan       chan error
 	started         chan struct{}
 	keyGen          *protocol.KeyGenerator
+	promotionTimer  *time.Timer

 	// fields protected by fmut
 	fmut                           sync.RWMutex
@@ -158,9 +159,11 @@ type model struct {

 	// fields protected by pmut
 	pmut                sync.RWMutex
-	conn                map[protocol.DeviceID]protocol.Connection
+	connections         map[string]protocol.Connection // connection ID -> connection
+	deviceConnIDs       map[protocol.DeviceID][]string // device -> connection IDs (invariant: if the key exists, the value is len >= 1, with the primary connection at the start of the slice)
+	promotedConnID      map[protocol.DeviceID]string   // device -> latest promoted connection ID
 	connRequestLimiters map[protocol.DeviceID]*semaphore.Semaphore
-	closed              map[protocol.DeviceID]chan struct{}
+	closed              map[string]chan struct{} // connection ID -> closed channel
 	helloMessages       map[protocol.DeviceID]protocol.Hello
 	deviceDownloads     map[protocol.DeviceID]*deviceDownloadState
 	remoteFolderStates  map[protocol.DeviceID]map[string]remoteFolderState // deviceID -> folders
@@ -179,13 +182,11 @@ var folderFactories = make(map[config.FolderType]folderFactory)
 var (
 	errDeviceUnknown    = errors.New("unknown device")
 	errDevicePaused     = errors.New("device is paused")
-	errDeviceRemoved    = errors.New("device has been removed")
 	ErrFolderPaused     = errors.New("folder is paused")
 	ErrFolderNotRunning = errors.New("folder is not running")
 	ErrFolderMissing    = errors.New("no such folder")
 	errNoVersioner      = errors.New("folder has no versioner")
 	// errors about why a connection is closed
-	errReplacingConnection                = errors.New("replacing connection")
 	errStopped                            = errors.New("Syncthing is being stopped")
 	errEncryptionInvConfigLocal           = errors.New("can't encrypt outgoing data because local data is encrypted (folder-type receive-encrypted)")
 	errEncryptionInvConfigRemote          = errors.New("remote has encrypted data and encrypts that data for us - this is impossible")
@@ -203,7 +204,7 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger, keyGen *protocol.KeyGenerator) Model {
+func NewModel(cfg config.Wrapper, id protocol.DeviceID, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger, keyGen *protocol.KeyGenerator) Model {
 	spec := svcutil.SpecWithDebugLogger(l)
 	m := &model{
 		Supervisor: suture.New("model", spec),
@@ -211,8 +212,6 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
 		// constructor parameters
 		cfg:            cfg,
 		id:             id,
-		clientName:     clientName,
-		clientVersion:  clientVersion,
 		db:             ldb,
 		protectedFiles: protectedFiles,
 		evLogger:       evLogger,
@@ -226,6 +225,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
 		fatalChan:            make(chan error),
 		started:              make(chan struct{}),
 		keyGen:               keyGen,
+		promotionTimer:       time.NewTimer(0),

 		// fields protected by fmut
 		fmut:                           sync.NewRWMutex(),
@@ -240,16 +240,19 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio

 		// fields protected by pmut
 		pmut:                sync.NewRWMutex(),
-		conn:                make(map[protocol.DeviceID]protocol.Connection),
+		connections:         make(map[string]protocol.Connection),
+		deviceConnIDs:       make(map[protocol.DeviceID][]string),
+		promotedConnID:      make(map[protocol.DeviceID]string),
 		connRequestLimiters: make(map[protocol.DeviceID]*semaphore.Semaphore),
-		closed:              make(map[protocol.DeviceID]chan struct{}),
+		closed:              make(map[string]chan struct{}),
 		helloMessages:       make(map[protocol.DeviceID]protocol.Hello),
 		deviceDownloads:     make(map[protocol.DeviceID]*deviceDownloadState),
 		remoteFolderStates:  make(map[protocol.DeviceID]map[string]remoteFolderState),
 		indexHandlers:       newServiceMap[protocol.DeviceID, *indexHandlerRegistry](evLogger),
 	}
-	for devID := range cfg.Devices() {
+	for devID, cfg := range cfg.Devices() {
 		m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID)
+		m.setConnRequestLimitersPLocked(cfg)
 	}
 	m.Add(m.folderRunners)
 	m.Add(m.progressEmitter)
@@ -272,11 +275,18 @@ func (m *model) serve(ctx context.Context) error {

 	close(m.started)

-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-m.fatalChan:
-		return svcutil.AsFatalErr(err, svcutil.ExitError)
+	for {
+		select {
+		case <-ctx.Done():
+			l.Debugln(m, "context closed, stopping", ctx.Err())
+			return ctx.Err()
+		case err := <-m.fatalChan:
+			l.Debugln(m, "fatal error, stopping", err)
+			return svcutil.AsFatalErr(err, svcutil.ExitError)
+		case <-m.promotionTimer.C:
+			l.Debugln("promotion timer fired")
+			m.promoteConnections()
+		}
 	}
 }

@@ -303,9 +313,9 @@ func (m *model) initFolders(cfg config.Configuration) error {

 func (m *model) closeAllConnectionsAndWait() {
 	m.pmut.RLock()
-	closed := make([]chan struct{}, 0, len(m.conn))
-	for id, conn := range m.conn {
-		closed = append(closed, m.closed[id])
+	closed := make([]chan struct{}, 0, len(m.connections))
+	for connID, conn := range m.connections {
+		closed = append(closed, m.closed[connID])
 		go conn.Close(errStopped)
 	}
 	m.pmut.RUnlock()
@@ -635,7 +645,7 @@ func (m *model) UsageReportingStats(report *contract.Report, version int, previe

 		// Transport stats
 		m.pmut.RLock()
-		for _, conn := range m.conn {
+		for _, conn := range m.connections {
 			report.TransportStats[conn.Transport()]++
 		}
 		m.pmut.RUnlock()
@@ -699,22 +709,28 @@ func (m *model) UsageReportingStats(report *contract.Report, version int, previe
 	}
 }

-type ConnectionInfo struct {
-	protocol.Statistics
+type ConnectionStats struct {
+	protocol.Statistics // Total for primary + secondaries
+
 	Connected     bool   `json:"connected"`
 	Paused        bool   `json:"paused"`
-	Address       string `json:"address"`
 	ClientVersion string `json:"clientVersion"`
-	Type          string `json:"type"`
-	IsLocal       bool   `json:"isLocal"`
-	Crypto        string `json:"crypto"`
+
+	Address string `json:"address"` // mirror values from Primary, for compatibility with <1.24.0
+	Type    string `json:"type"`    // mirror values from Primary, for compatibility with <1.24.0
+	IsLocal bool   `json:"isLocal"` // mirror values from Primary, for compatibility with <1.24.0
+	Crypto  string `json:"crypto"`  // mirror values from Primary, for compatibility with <1.24.0
+
+	Primary   ConnectionInfo   `json:"primary,omitempty"`
+	Secondary []ConnectionInfo `json:"secondary,omitempty"`
 }

-// NumConnections returns the current number of active connected devices.
-func (m *model) NumConnections() int {
-	m.pmut.RLock()
-	defer m.pmut.RUnlock()
-	return len(m.conn)
+type ConnectionInfo struct {
+	protocol.Statistics
+	Address string `json:"address"`
+	Type    string `json:"type"`
+	IsLocal bool   `json:"isLocal"`
+	Crypto  string `json:"crypto"`
 }

 // ConnectionStats returns a map with connection statistics for each device.
@@ -724,29 +740,59 @@ func (m *model) ConnectionStats() map[string]interface{} {

 	res := make(map[string]interface{})
 	devs := m.cfg.Devices()
-	conns := make(map[string]ConnectionInfo, len(devs))
+	conns := make(map[string]ConnectionStats, len(devs))
 	for device, deviceCfg := range devs {
+		if device == m.id {
+			continue
+		}
 		hello := m.helloMessages[device]
 		versionString := hello.ClientVersion
 		if hello.ClientName != "syncthing" {
 			versionString = hello.ClientName + " " + hello.ClientVersion
 		}
-		ci := ConnectionInfo{
-			ClientVersion: strings.TrimSpace(versionString),
+		connIDs, ok := m.deviceConnIDs[device]
+		cs := ConnectionStats{
+			Connected:     ok,
 			Paused:        deviceCfg.Paused,
+			ClientVersion: strings.TrimSpace(versionString),
 		}
-		if conn, ok := m.conn[device]; ok {
-			ci.Type = conn.Type()
-			ci.IsLocal = conn.IsLocal()
-			ci.Crypto = conn.Crypto()
-			ci.Connected = ok
-			ci.Statistics = conn.Statistics()
-			if addr := conn.RemoteAddr(); addr != nil {
-				ci.Address = addr.String()
+		if ok {
+			conn := m.connections[connIDs[0]]
+
+			cs.Primary.Type = conn.Type()
+			cs.Primary.IsLocal = conn.IsLocal()
+			cs.Primary.Crypto = conn.Crypto()
+			cs.Primary.Statistics = conn.Statistics()
+			cs.Primary.Address = conn.RemoteAddr().String()
+
+			cs.Type = cs.Primary.Type
+			cs.IsLocal = cs.Primary.IsLocal
+			cs.Crypto = cs.Primary.Crypto
+			cs.Address = cs.Primary.Address
+			cs.Statistics = cs.Primary.Statistics
+
+			for _, connID := range connIDs[1:] {
+				conn = m.connections[connID]
+				sec := ConnectionInfo{
+					Statistics: conn.Statistics(),
+					Address:    conn.RemoteAddr().String(),
+					Type:       conn.Type(),
+					IsLocal:    conn.IsLocal(),
+					Crypto:     conn.Crypto(),
+				}
+				if sec.At.After(cs.At) {
+					cs.At = sec.At
+				}
+				if sec.StartedAt.Before(cs.StartedAt) {
+					cs.StartedAt = sec.StartedAt
+				}
+				cs.InBytesTotal += sec.InBytesTotal
+				cs.OutBytesTotal += sec.OutBytesTotal
+				cs.Secondary = append(cs.Secondary, sec)
 			}
 		}

-		conns[device.String()] = ci
+		conns[device.String()] = cs
 	}

 	res["connections"] = conns
@@ -1138,17 +1184,16 @@ func (m *model) handleIndex(conn protocol.Connection, folder string, fs []protoc
 	}

 	m.pmut.RLock()
-	indexHandler, ok := m.indexHandlers.Get(deviceID)
+	indexHandler, ok := m.getIndexHandlerPRLocked(conn)
 	m.pmut.RUnlock()
 	if !ok {
-		// This should be impossible, as an index handler always exists for an
-		// open connection, and this method can't be called on a closed
-		// connection
+		// This should be impossible, as an index handler is registered when
+		// we send a cluster config, and that is what triggers index
+		// sending.
 		m.evLogger.Log(events.Failure, "index sender does not exist for connection on which indexes were received")
 		l.Debugf("%v for folder (ID %q) sent from device %q: missing index handler", op, folder, deviceID)
-		return fmt.Errorf("index handler missing: %s", folder)
+		return fmt.Errorf("%s: %w", folder, ErrFolderNotRunning)
 	}
-
 	return indexHandler.ReceiveIndex(folder, fs, update, op)
 }

@@ -1161,24 +1206,26 @@ type ClusterConfigReceivedEventData struct {
 }

 func (m *model) ClusterConfig(conn protocol.Connection, cm protocol.ClusterConfig) error {
+	deviceID := conn.DeviceID()
+
+	if cm.Secondary {
+		// No handling of secondary connection ClusterConfigs; they merely
+		// indicate the connection is ready to start.
+		l.Debugf("Skipping secondary ClusterConfig from %v at %s", deviceID.Short(), conn)
+		return nil
+	}
+
 	// Check the peer device's announced folders against our own. Emits events
 	// for folders that we don't expect (unknown or not shared).
 	// Also, collect a list of folders we do share, and if he's interested in
 	// temporary indexes, subscribe the connection.

-	deviceID := conn.DeviceID()
-	l.Debugf("Handling ClusterConfig from %v", deviceID.Short())
-
-	m.pmut.RLock()
-	indexHandlerRegistry, ok := m.indexHandlers.Get(deviceID)
-	m.pmut.RUnlock()
-	if !ok {
-		panic("bug: ClusterConfig called on closed or nonexistent connection")
-	}
+	l.Debugf("Handling ClusterConfig from %v at %s", deviceID.Short(), conn)
+	indexHandlerRegistry := m.ensureIndexHandler(conn)

 	deviceCfg, ok := m.cfg.Device(deviceID)
 	if !ok {
-		l.Debugln("Device disappeared from config while processing cluster-config")
+		l.Debugf("Device %s disappeared from config while processing cluster-config", deviceID.Short())
 		return errDeviceUnknown
 	}

@@ -1198,11 +1245,11 @@ func (m *model) ClusterConfig(conn protocol.Connection, cm protocol.ClusterConfi
 			}
 		}
 		if info.remote.ID == protocol.EmptyDeviceID {
-			l.Infof("Device %v sent cluster-config without the device info for the remote on folder %v", deviceID, folder.Description())
+			l.Infof("Device %v sent cluster-config without the device info for the remote on folder %v", deviceID.Short(), folder.Description())
 			return errMissingRemoteInClusterConfig
 		}
 		if info.local.ID == protocol.EmptyDeviceID {
-			l.Infof("Device %v sent cluster-config without the device info for us locally on folder %v", deviceID, folder.Description())
+			l.Infof("Device %v sent cluster-config without the device info for us locally on folder %v", deviceID.Short(), folder.Description())
 			return errMissingLocalInClusterConfig
 		}
 		ccDeviceInfos[folder.ID] = info
@@ -1260,12 +1307,16 @@ func (m *model) ClusterConfig(conn protocol.Connection, cm protocol.ClusterConfi
 	})

 	if len(tempIndexFolders) > 0 {
+		var connOK bool
+		var conn protocol.Connection
 		m.pmut.RLock()
-		conn, ok := m.conn[deviceID]
+		if connIDs, connIDOK := m.deviceConnIDs[deviceID]; connIDOK {
+			conn, connOK = m.connections[connIDs[0]]
+		}
 		m.pmut.RUnlock()
 		// In case we've got ClusterConfig, and the connection disappeared
 		// from infront of our nose.
-		if ok {
+		if connOK {
 			m.progressEmitter.temporaryIndexSubscribe(conn, tempIndexFolders)
 		}
 	}
@@ -1291,6 +1342,57 @@ func (m *model) ClusterConfig(conn protocol.Connection, cm protocol.ClusterConfi
 	return nil
 }

+func (m *model) ensureIndexHandler(conn protocol.Connection) *indexHandlerRegistry {
+	deviceID := conn.DeviceID()
+	connID := conn.ConnectionID()
+
+	m.pmut.Lock()
+	defer m.pmut.Unlock()
+
+	indexHandlerRegistry, ok := m.indexHandlers.Get(deviceID)
+	if ok && indexHandlerRegistry.conn.ConnectionID() == connID {
+		// This is an existing and proper index handler for this connection.
+		return indexHandlerRegistry
+	}
+
+	if ok {
+		// A handler exists, but it's for another connection than the one we
+		// now got a ClusterConfig on. This should be unusual as it means
+		// the other side has decided to start using a new primary
+		// connection but we haven't seen it close yet. Ideally it will
+		// close shortly by itself...
+		l.Infof("Abandoning old index handler for %s (%s) in favour of %s", deviceID.Short(), indexHandlerRegistry.conn.ConnectionID(), connID)
+		m.indexHandlers.RemoveAndWait(deviceID, 0)
+	}
+
+	// Create a new index handler for this device.
+	indexHandlerRegistry = newIndexHandlerRegistry(conn, m.deviceDownloads[deviceID], m.evLogger)
+	for id, fcfg := range m.folderCfgs {
+		l.Debugln("Registering folder", id, "for", deviceID.Short())
+		runner, _ := m.folderRunners.Get(id)
+		indexHandlerRegistry.RegisterFolderState(fcfg, m.folderFiles[id], runner)
+	}
+	m.indexHandlers.Add(deviceID, indexHandlerRegistry)
+
+	return indexHandlerRegistry
+}
+
+func (m *model) getIndexHandlerPRLocked(conn protocol.Connection) (*indexHandlerRegistry, bool) {
+	// Reads from index handlers, which requires pmut to be read locked
+
+	deviceID := conn.DeviceID()
+	connID := conn.ConnectionID()
+
+	indexHandlerRegistry, ok := m.indexHandlers.Get(deviceID)
+	if ok && indexHandlerRegistry.conn.ConnectionID() == connID {
+		// This is an existing and proper index handler for this connection.
+		return indexHandlerRegistry, true
+	}
+
+	// There is no index handler, or it's not registered for this connection.
+	return nil, false
+}
+
 func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.DeviceConfiguration, ccDeviceInfos map[string]*clusterConfigDeviceInfo, indexHandlers *indexHandlerRegistry) ([]string, map[string]remoteFolderState, error) {
 	var folderDevice config.FolderDeviceConfiguration
 	tempIndexFolders := make([]string, 0, len(folders))
@@ -1539,8 +1641,8 @@ func (m *model) sendClusterConfig(ids []protocol.DeviceID) {
 	ccConns := make([]protocol.Connection, 0, len(ids))
 	m.pmut.RLock()
 	for _, id := range ids {
-		if conn, ok := m.conn[id]; ok {
-			ccConns = append(ccConns, conn)
+		if connIDs, ok := m.deviceConnIDs[id]; ok {
+			ccConns = append(ccConns, m.connections[connIDs[0]])
 		}
 	}
 	m.pmut.RUnlock()
@@ -1777,33 +1879,62 @@ func (m *model) introduceDevice(device protocol.Device, introducerCfg config.Dev

 // Closed is called when a connection has been closed
 func (m *model) Closed(conn protocol.Connection, err error) {
-	device := conn.DeviceID()
+	connID := conn.ConnectionID()
+	deviceID := conn.DeviceID()
+
 	m.pmut.Lock()
-	conn, ok := m.conn[device]
+	conn, ok := m.connections[connID]
 	if !ok {
 		m.pmut.Unlock()
 		return
 	}

-	delete(m.conn, device)
-	delete(m.connRequestLimiters, device)
-	delete(m.helloMessages, device)
-	delete(m.deviceDownloads, device)
-	delete(m.remoteFolderStates, device)
-	closed := m.closed[device]
-	delete(m.closed, device)
-	wait := m.indexHandlers.RemoveAndWaitChan(device, 0)
+	closed := m.closed[connID]
+	delete(m.closed, connID)
+	delete(m.connections, connID)
+
+	removedIsPrimary := m.promotedConnID[deviceID] == connID
+	remainingConns := without(m.deviceConnIDs[deviceID], connID)
+	var wait <-chan error
+	if removedIsPrimary {
+		m.progressEmitter.temporaryIndexUnsubscribe(conn)
+		if idxh, ok := m.indexHandlers.Get(deviceID); ok && idxh.conn.ConnectionID() == connID {
+			wait = m.indexHandlers.RemoveAndWaitChan(deviceID, 0)
+		}
+		m.scheduleConnectionPromotion()
+	}
+	if len(remainingConns) == 0 {
+		// All device connections closed
+		delete(m.deviceConnIDs, deviceID)
+		delete(m.promotedConnID, deviceID)
+		delete(m.connRequestLimiters, deviceID)
+		delete(m.helloMessages, deviceID)
+		delete(m.remoteFolderStates, deviceID)
+		delete(m.deviceDownloads, deviceID)
+	} else {
+		// Some connections remain
+		m.deviceConnIDs[deviceID] = remainingConns
+	}
+
 	m.pmut.Unlock()
-	<-wait
+	if wait != nil {
+		<-wait
+	}

-	m.progressEmitter.temporaryIndexUnsubscribe(conn)
-	m.deviceDidClose(device, time.Since(conn.EstablishedAt()))
+	m.fmut.RLock()
+	m.deviceDidCloseFRLocked(deviceID, time.Since(conn.EstablishedAt()))
+	m.fmut.RUnlock()

-	l.Infof("Connection to %s at %s closed: %v", device, conn, err)
-	m.evLogger.Log(events.DeviceDisconnected, map[string]string{
-		"id":    device.String(),
-		"error": err.Error(),
-	})
+	k := map[bool]string{false: "secondary", true: "primary"}[removedIsPrimary]
+	l.Infof("Lost %s connection to %s at %s: %v (%d remain)", k, deviceID.Short(), conn, err, len(remainingConns))
+
+	if len(remainingConns) == 0 {
+		l.Infof("Connection to %s at %s closed: %v", deviceID.Short(), conn, err)
+		m.evLogger.Log(events.DeviceDisconnected, map[string]string{
+			"id":    deviceID.String(),
+			"error": err.Error(),
+		})
+	}
 	close(closed)
 }

@@ -1852,36 +1983,36 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i
 	if !ok {
 		// The folder might be already unpaused in the config, but not yet
 		// in the model.
-		l.Debugf("Request from %s for file %s in unstarted folder %q", deviceID, name, folder)
+		l.Debugf("Request from %s for file %s in unstarted folder %q", deviceID.Short(), name, folder)
 		return nil, protocol.ErrGeneric
 	}

 	if !folderCfg.SharedWith(deviceID) {
-		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
+		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID.Short(), name, folder)
 		return nil, protocol.ErrGeneric
 	}
 	if folderCfg.Paused {
-		l.Debugf("Request from %s for file %s in paused folder %q", deviceID, name, folder)
+		l.Debugf("Request from %s for file %s in paused folder %q", deviceID.Short(), name, folder)
 		return nil, protocol.ErrGeneric
 	}

 	// Make sure the path is valid and in canonical form
 	if name, err = fs.Canonicalize(name); err != nil {
-		l.Debugf("Request from %s in folder %q for invalid filename %s", deviceID, folder, name)
+		l.Debugf("Request from %s in folder %q for invalid filename %s", deviceID.Short(), folder, name)
 		return nil, protocol.ErrGeneric
 	}

 	if deviceID != protocol.LocalDeviceID {
-		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, size, fromTemporary)
+		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID.Short(), folder, name, offset, size, fromTemporary)
 	}

 	if fs.IsInternal(name) {
-		l.Debugf("%v REQ(in) for internal file: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) for internal file: %s: %q / %q o=%d s=%d", m, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrInvalid
 	}

 	if folderIgnores.Match(name).IsIgnored() {
-		l.Debugf("%v REQ(in) for ignored file: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) for ignored file: %s: %q / %q o=%d s=%d", m, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrInvalid
 	}

@@ -1908,7 +2039,7 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i
 	folderFs := folderCfg.Filesystem(nil)

 	if err := osutil.TraversesSymlink(folderFs, filepath.Dir(name)); err != nil {
-		l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrNoSuchFile
 	}

@@ -1920,7 +2051,7 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i
 		if info, err := folderFs.Lstat(tempFn); err != nil || !info.IsRegular() {
 			// Reject reads for anything that doesn't exist or is something
 			// other than a regular file.
-			l.Debugf("%v REQ(in) failed stating temp file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
+			l.Debugf("%v REQ(in) failed stating temp file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID.Short(), folder, name, offset, size)
 			return nil, protocol.ErrNoSuchFile
 		}
 		_, err := readOffsetIntoBuf(folderFs, tempFn, offset, res.data)
@@ -1934,13 +2065,13 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i
 	if info, err := folderFs.Lstat(name); err != nil || !info.IsRegular() {
 		// Reject reads for anything that doesn't exist or is something
 		// other than a regular file.
-		l.Debugf("%v REQ(in) failed stating file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) failed stating file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrNoSuchFile
 	}

 	n, err := readOffsetIntoBuf(folderFs, name, offset, res.data)
 	if fs.IsNotExist(err) {
-		l.Debugf("%v REQ(in) file doesn't exist: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) file doesn't exist: %s: %q / %q o=%d s=%d", m, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrNoSuchFile
 	} else if err == io.EOF {
 		// Read beyond end of file. This might indicate a problem, or it
@@ -1949,13 +2080,13 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i
 		// next step take care of it, by only hashing the part we actually
 		// managed to read.
 	} else if err != nil {
-		l.Debugf("%v REQ(in) failed reading file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) failed reading file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrGeneric
 	}

 	if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(hash) > 0 && !scanner.Validate(res.data[:n], hash, weakHash) {
 		m.recheckFile(deviceID, folder, name, offset, hash, weakHash)
-		l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
+		l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID.Short(), folder, name, offset, size)
 		return nil, protocol.ErrNoSuchFile
 	}

@@ -2074,15 +2205,12 @@ func (m *model) GetMtimeMapping(folder string, file string) (fs.MtimeMapping, er
 	return fs.GetMtimeMapping(fcfg.Filesystem(ffs), file)
 }

-// Connection returns the current connection for device, and a boolean whether a connection was found.
-func (m *model) Connection(deviceID protocol.DeviceID) (protocol.Connection, bool) {
+// Connection returns if we are connected to the given device.
+func (m *model) ConnectedTo(deviceID protocol.DeviceID) bool {
 	m.pmut.RLock()
-	cn, ok := m.conn[deviceID]
+	_, ok := m.deviceConnIDs[deviceID]
 	m.pmut.RUnlock()
-	if ok {
-		m.deviceWasSeen(deviceID)
-	}
-	return cn, ok
+	return ok
 }

 // LoadIgnores loads or refreshes the ignore patterns from disk, if the
@@ -2200,74 +2328,29 @@ func (m *model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
 	return nil
 }

-// GetHello is called when we are about to connect to some remote device.
-func (m *model) GetHello(id protocol.DeviceID) protocol.HelloIntf {
-	name := ""
-	if _, ok := m.cfg.Device(id); ok {
-		// Set our name (from the config of our device ID) only if we already know about the other side device ID.
-		if myCfg, ok := m.cfg.Device(m.id); ok {
-			name = myCfg.Name
-		}
-	}
-	return &protocol.Hello{
-		DeviceName:    name,
-		ClientName:    m.clientName,
-		ClientVersion: m.clientVersion,
-	}
-}
-
 // AddConnection adds a new peer connection to the model. An initial index will
 // be sent to the connected peer, thereafter index updates whenever the local
 // folder changes.
 func (m *model) AddConnection(conn protocol.Connection, hello protocol.Hello) {
 	deviceID := conn.DeviceID()
-	device, ok := m.cfg.Device(deviceID)
+	deviceCfg, ok := m.cfg.Device(deviceID)
 	if !ok {
 		l.Infoln("Trying to add connection to unknown device")
 		return
 	}

-	// The slightly unusual locking sequence here is because we must acquire
-	// fmut before pmut. (The locks can be *released* in any order.)
-	m.fmut.RLock()
-	m.pmut.Lock()
-	if oldConn, ok := m.conn[deviceID]; ok {
-		l.Infoln("Replacing old connection", oldConn, "with", conn, "for", deviceID)
-		// There is an existing connection to this device that we are
-		// replacing. We must close the existing connection and wait for the
-		// close to complete before adding the new connection. We do the
-		// actual close without holding pmut as the connection will call
-		// back into Closed() for the cleanup.
-		closed := m.closed[deviceID]
-		m.fmut.RUnlock()
-		m.pmut.Unlock()
-		oldConn.Close(errReplacingConnection)
-		<-closed
-		// Again, lock fmut before pmut.
-		m.fmut.RLock()
-		m.pmut.Lock()
-	}
-
-	m.conn[deviceID] = conn
+	connID := conn.ConnectionID()
 	closed := make(chan struct{})
-	m.closed[deviceID] = closed
-	m.deviceDownloads[deviceID] = newDeviceDownloadState()
-	indexRegistry := newIndexHandlerRegistry(conn, m.deviceDownloads[deviceID], m.evLogger)
-	for id, fcfg := range m.folderCfgs {
-		runner, _ := m.folderRunners.Get(id)
-		indexRegistry.RegisterFolderState(fcfg, m.folderFiles[id], runner)
-	}
-	m.indexHandlers.Add(deviceID, indexRegistry)
-	m.fmut.RUnlock()
-	// 0: default, <0: no limiting
-	switch {
-	case device.MaxRequestKiB > 0:
-		m.connRequestLimiters[deviceID] = semaphore.New(1024 * device.MaxRequestKiB)
-	case device.MaxRequestKiB == 0:
-		m.connRequestLimiters[deviceID] = semaphore.New(1024 * defaultPullerPendingKiB)
-	}

+	m.pmut.Lock()
+
+	m.connections[connID] = conn
+	m.closed[connID] = closed
 	m.helloMessages[deviceID] = hello
+	m.deviceConnIDs[deviceID] = append(m.deviceConnIDs[deviceID], connID)
+	if m.deviceDownloads[deviceID] == nil {
+		m.deviceDownloads[deviceID] = newDeviceDownloadState()
+	}

 	event := map[string]string{
 		"id":            deviceID.String(),
@@ -2284,17 +2367,15 @@ func (m *model) AddConnection(conn protocol.Connection, hello protocol.Hello) {

 	m.evLogger.Log(events.DeviceConnected, event)

-	l.Infof(`Device %s client is "%s %s" named "%s" at %s`, deviceID, hello.ClientName, hello.ClientVersion, hello.DeviceName, conn)
+	if len(m.deviceConnIDs[deviceID]) == 1 {
+		l.Infof(`Device %s client is "%s %s" named "%s" at %s`, deviceID.Short(), hello.ClientName, hello.ClientVersion, hello.DeviceName, conn)
+	} else {
+		l.Infof(`Additional connection (+%d) for device %s at %s`, len(m.deviceConnIDs[deviceID])-1, deviceID.Short(), conn)
+	}

-	conn.Start()
 	m.pmut.Unlock()

-	// Acquires fmut, so has to be done outside of pmut.
-	cm, passwords := m.generateClusterConfig(deviceID)
-	conn.SetFolderPasswords(passwords)
-	conn.ClusterConfig(cm)
-
-	if (device.Name == "" || m.cfg.Options().OverwriteRemoteDevNames) && hello.DeviceName != "" {
+	if (deviceCfg.Name == "" || m.cfg.Options().OverwriteRemoteDevNames) && hello.DeviceName != "" {
 		m.cfg.Modify(func(cfg *config.Configuration) {
 			for i := range cfg.Devices {
 				if cfg.Devices[i].DeviceID == deviceID {
@@ -2308,26 +2389,78 @@ func (m *model) AddConnection(conn protocol.Connection, hello protocol.Hello) {
 	}

 	m.deviceWasSeen(deviceID)
+	m.scheduleConnectionPromotion()
+}
+
+func (m *model) scheduleConnectionPromotion() {
+	// Keeps deferring to prevent multiple executions in quick succession,
+	// e.g. if multiple connections to a single device are closed.
+	m.promotionTimer.Reset(time.Second)
+}
+
+// promoteConnections checks for devices that have connections, but where
+// the primary connection hasn't started index handlers etc. yet, and
+// promotes the primary connection to be the index handling one. This should
+// be called after adding new connections, and after closing a primary
+// device connection.
+func (m *model) promoteConnections() {
+	m.fmut.RLock() // for generateClusterConfigFRLocked
+	defer m.fmut.RUnlock()
+
+	m.pmut.Lock() // for most other things
+	defer m.pmut.Unlock()
+
+	for deviceID, connIDs := range m.deviceConnIDs {
+		cm, passwords := m.generateClusterConfigFRLocked(deviceID)
+		if m.promotedConnID[deviceID] != connIDs[0] {
+			// The previously promoted connection is not the current
+			// primary; we should promote the primary connection to be the
+			// index handling one. We do this by sending a ClusterConfig on
+			// it, which will cause the other side to start sending us index
+			// messages there. (On our side, we manage index handlers based
+			// on where we get ClusterConfigs from the peer.)
+			conn := m.connections[connIDs[0]]
+			l.Debugf("Promoting connection to %s at %s", deviceID.Short(), conn)
+			if conn.Statistics().StartedAt.IsZero() {
+				conn.SetFolderPasswords(passwords)
+				conn.Start()
+			}
+			conn.ClusterConfig(cm)
+			m.promotedConnID[deviceID] = connIDs[0]
+		}
+
+		// Make sure any other new connections also get started, and that
+		// they get a secondary-marked ClusterConfig.
+		for _, connID := range connIDs[1:] {
+			conn := m.connections[connID]
+			if conn.Statistics().StartedAt.IsZero() {
+				conn.SetFolderPasswords(passwords)
+				conn.Start()
+				conn.ClusterConfig(protocol.ClusterConfig{Secondary: true})
+			}
+		}
+	}
 }

 func (m *model) DownloadProgress(conn protocol.Connection, folder string, updates []protocol.FileDownloadProgressUpdate) error {
+	deviceID := conn.DeviceID()
+
 	m.fmut.RLock()
 	cfg, ok := m.folderCfgs[folder]
 	m.fmut.RUnlock()

-	device := conn.DeviceID()
-	if !ok || cfg.DisableTempIndexes || !cfg.SharedWith(device) {
+	if !ok || cfg.DisableTempIndexes || !cfg.SharedWith(deviceID) {
 		return nil
 	}

 	m.pmut.RLock()
-	downloads := m.deviceDownloads[device]
+	downloads := m.deviceDownloads[deviceID]
 	m.pmut.RUnlock()
 	downloads.Update(folder, updates)
 	state := downloads.GetBlockCounts(folder)

 	m.evLogger.Log(events.RemoteDownloadProgress, map[string]interface{}{
-		"device": device.String(),
+		"device": deviceID.String(),
 		"folder": folder,
 		"state":  state,
 	})
@@ -2344,27 +2477,47 @@ func (m *model) deviceWasSeen(deviceID protocol.DeviceID) {
 	}
 }

-func (m *model) deviceDidClose(deviceID protocol.DeviceID, duration time.Duration) {
-	m.fmut.RLock()
-	sr, ok := m.deviceStatRefs[deviceID]
-	m.fmut.RUnlock()
-	if ok {
+func (m *model) deviceDidCloseFRLocked(deviceID protocol.DeviceID, duration time.Duration) {
+	if sr, ok := m.deviceStatRefs[deviceID]; ok {
 		_ = sr.LastConnectionDuration(duration)
 	}
 }

 func (m *model) requestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
-	m.pmut.RLock()
-	nc, ok := m.conn[deviceID]
-	m.pmut.RUnlock()
-
-	if !ok {
-		return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
+	conn, connOK := m.requestConnectionForDevice(deviceID)
+	if !connOK {
+		return nil, fmt.Errorf("requestGlobal: no connection to device: %s", deviceID.Short())
 	}

-	l.Debugf("%v REQ(out): %s: %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
+	l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
+	return conn.Request(ctx, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
+}

-	return nc.Request(ctx, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
+// requestConnectionForDevice returns a connection to the given device, to
+// be used for sending a request. If there is only one device connection,
+// this is the one to use. If there are multiple then we avoid the first
+// ("primary") connection, which is dedicated to index data, and pick a
+// random one of the others.
+func (m *model) requestConnectionForDevice(deviceID protocol.DeviceID) (protocol.Connection, bool) {
+	m.pmut.RLock()
+	defer m.pmut.RUnlock()
+
+	connIDs, ok := m.deviceConnIDs[deviceID]
+	if !ok {
+		return nil, false
+	}
+
+	// If there is an entry in deviceConns, it always contains at least one
+	// connection.
+	connID := connIDs[0]
+	if len(connIDs) > 1 {
+		// Pick a random connection of the non-primary ones
+		idx := rand.Intn(len(connIDs)-1) + 1
+		connID = connIDs[idx]
+	}
+
+	conn, connOK := m.connections[connID]
+	return conn, connOK
 }

 func (m *model) ScanFolders() map[string]error {
@@ -2455,11 +2608,13 @@ func (m *model) numHashers(folder string) int {
 // generateClusterConfig returns a ClusterConfigMessage that is correct and the
 // set of folder passwords for the given peer device
 func (m *model) generateClusterConfig(device protocol.DeviceID) (protocol.ClusterConfig, map[string]string) {
-	var message protocol.ClusterConfig
-
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
+	return m.generateClusterConfigFRLocked(device)
+}

+func (m *model) generateClusterConfigFRLocked(device protocol.DeviceID) (protocol.ClusterConfig, map[string]string) {
+	var message protocol.ClusterConfig
 	folders := m.cfg.FolderList()
 	passwords := make(map[string]string, len(folders))
 	for _, folderCfg := range folders {
@@ -2771,7 +2926,7 @@ func (m *model) availabilityInSnapshotPRlocked(cfg config.FolderConfiguration, s
 		if state := m.remoteFolderStates[device][cfg.ID]; state != remoteFolderValid {
 			continue
 		}
-		_, ok := m.conn[device]
+		_, ok := m.deviceConnIDs[device]
 		if ok {
 			availabilities = append(availabilities, Availability{ID: device, FromTemporary: false})
 		}
@@ -2904,13 +3059,6 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 		}
 	}

-	// Removing a device. We actually don't need to do anything.
-	// Because folder config has changed (since the device lists do not match)
-	// Folders for that had device got "restarted", which involves killing
-	// connections to all devices that we were sharing the folder with.
-	// At some point model.Close() will get called for that device which will
-	// clean residue device state that is not part of any folder.
-
 	// Pausing a device, unpausing is handled by the connection service.
 	fromDevices := from.DeviceMap()
 	toDevices := to.DeviceMap()
@@ -2941,7 +3089,14 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 			l.Infoln("Resuming", deviceID)
 			m.evLogger.Log(events.DeviceResumed, map[string]string{"device": deviceID.String()})
 		}
+
+		if toCfg.MaxRequestKiB != fromCfg.MaxRequestKiB {
+			m.pmut.Lock()
+			m.setConnRequestLimitersPLocked(toCfg)
+			m.pmut.Unlock()
+		}
 	}
+
 	// Clean up after removed devices
 	removedDevices := make([]protocol.DeviceID, 0, len(fromDevices))
 	m.fmut.Lock()
@@ -2955,14 +3110,18 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 	m.pmut.RLock()
 	for _, id := range closeDevices {
 		delete(clusterConfigDevices, id)
-		if conn, ok := m.conn[id]; ok {
-			go conn.Close(errDevicePaused)
+		if conns, ok := m.deviceConnIDs[id]; ok {
+			for _, connID := range conns {
+				go m.connections[connID].Close(errDevicePaused)
+			}
 		}
 	}
 	for _, id := range removedDevices {
 		delete(clusterConfigDevices, id)
-		if conn, ok := m.conn[id]; ok {
-			go conn.Close(errDeviceRemoved)
+		if conns, ok := m.deviceConnIDs[id]; ok {
+			for _, connID := range conns {
+				go m.connections[connID].Close(errDevicePaused)
+			}
 		}
 	}
 	m.pmut.RUnlock()
@@ -2986,6 +3145,17 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 	return true
 }

+func (m *model) setConnRequestLimitersPLocked(cfg config.DeviceConfiguration) {
+	// Touches connRequestLimiters which is protected by pmut.
+	// 0: default, <0: no limiting
+	switch {
+	case cfg.MaxRequestKiB > 0:
+		m.connRequestLimiters[cfg.DeviceID] = semaphore.New(1024 * cfg.MaxRequestKiB)
+	case cfg.MaxRequestKiB == 0:
+		m.connRequestLimiters[cfg.DeviceID] = semaphore.New(1024 * defaultPullerPendingKiB)
+	}
+}
+
 func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.DeviceConfiguration, existingFolders map[string]config.FolderConfiguration, ignoredDevices deviceIDSet, removedFolders map[string]struct{}) {
 	var removedPendingFolders []map[string]string
 	pendingFolders, err := m.db.PendingFolders()
@@ -3332,3 +3502,12 @@ type redactedError struct {
 	error
 	redacted error
 }
+
+func without[E comparable, S ~[]E](s S, e E) S {
+	for i, x := range s {
+		if x == e {
+			return append(s[:i], s[i+1:]...)
+		}
+	}
+	return s
+}